From 00c201e39ba5696fb2ea518a5da85d2ee1583073 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 15:51:32 +0000 Subject: [PATCH] feat(justdummies): add JD018-JD022 for the seeding long tail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the reproducibility family with the five remaining ways a seed fails to mean what the code says it means. JD018 reports a reproducibility scope nested inside another. The premise was verified before the rule was written: Any.Reproducibly takes its seed from Guid.NewGuid().GetHashCode(), not from the ambient source, so an inner scope ignores whatever the outer one pinned. The outer mechanism still reports its own seed, so the failure names a seed that reproduces nothing — a wrong instruction rather than a wrong result. The seeded overload is left alone: pinning a chosen seed inside is deliberate. JD019 reports a constant replay seed left in committed code, and ships opt-in on measurement rather than taste. Enabled across JustDummies.UnitTests it reports 238 sites, nearly all legitimate: this repository's own maintainer guide instructs "pin a seed for anything statistical", so a default-on rule would fight documented practice. It earns its keep as a pre-release sweep somebody turns on deliberately. JD020 reports an AnyContext held in a static field — the shape that looks maximally deterministic and is not, since interleaved draws make neither the sequence nor the multiset stable. Info, because a single-threaded consumer shares one harmlessly and the rule cannot see the suite's parallelism configuration. JD021 reports a blank replay snippet. The guard rejects it at run time, but from an adapter hook that runs before every test — so the throw fails the whole suite as an infrastructure error rather than one assertion. JD022 reports a parallel work item that draws without opening its own seed scope. It carries no code fix by design: the repair needs a run seed the developer must choose and record, and inventing one would produce exactly the committed-seed problem JD019 describes. All five were confirmed by a planted positive control in a real build, which also established that JD020 and JD022 are invisible at default verbosity because they are Info — worth knowing before reading a clean sweep as evidence of anything. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01GKqyhG9Y4AfdxEYekP2Evq --- .../Jd018ToJd022ReproducibilityTailTests.cs | 382 ++++++++++++++++++ .../AnalyzerReleases.Unshipped.md | 5 + .../BlankReplaySnippetAnalyzer.cs | 56 +++ .../CommittedReplaySeedAnalyzer.cs | 81 ++++ JustDummies.Analyzers/Descriptors.cs | 52 +++ JustDummies.Analyzers/DiagnosticIds.cs | 7 + .../NestedReproducibilityScopeAnalyzer.cs | 84 ++++ .../ParallelDrawWithoutPerItemSeedAnalyzer.cs | 83 ++++ .../SharedStaticAnyContextAnalyzer.cs | 50 +++ .../for-users/analyzers/JD018.en.md | 43 ++ .../for-users/analyzers/JD018.fr.md | 43 ++ .../for-users/analyzers/JD019.en.md | 50 +++ .../for-users/analyzers/JD019.fr.md | 50 +++ .../for-users/analyzers/JD020.en.md | 43 ++ .../for-users/analyzers/JD020.fr.md | 43 ++ .../for-users/analyzers/JD021.en.md | 41 ++ .../for-users/analyzers/JD021.fr.md | 41 ++ .../for-users/analyzers/JD022.en.md | 49 +++ .../for-users/analyzers/JD022.fr.md | 49 +++ .../for-users/analyzers/README.fr.md | 5 + doc/handwritten/for-users/analyzers/README.md | 5 + 21 files changed, 1262 insertions(+) create mode 100644 JustDummies.Analyzers.UnitTests/Jd018ToJd022ReproducibilityTailTests.cs create mode 100644 JustDummies.Analyzers/BlankReplaySnippetAnalyzer.cs create mode 100644 JustDummies.Analyzers/CommittedReplaySeedAnalyzer.cs create mode 100644 JustDummies.Analyzers/NestedReproducibilityScopeAnalyzer.cs create mode 100644 JustDummies.Analyzers/ParallelDrawWithoutPerItemSeedAnalyzer.cs create mode 100644 JustDummies.Analyzers/SharedStaticAnyContextAnalyzer.cs create mode 100644 doc/handwritten/for-users/analyzers/JD018.en.md create mode 100644 doc/handwritten/for-users/analyzers/JD018.fr.md create mode 100644 doc/handwritten/for-users/analyzers/JD019.en.md create mode 100644 doc/handwritten/for-users/analyzers/JD019.fr.md create mode 100644 doc/handwritten/for-users/analyzers/JD020.en.md create mode 100644 doc/handwritten/for-users/analyzers/JD020.fr.md create mode 100644 doc/handwritten/for-users/analyzers/JD021.en.md create mode 100644 doc/handwritten/for-users/analyzers/JD021.fr.md create mode 100644 doc/handwritten/for-users/analyzers/JD022.en.md create mode 100644 doc/handwritten/for-users/analyzers/JD022.fr.md diff --git a/JustDummies.Analyzers.UnitTests/Jd018ToJd022ReproducibilityTailTests.cs b/JustDummies.Analyzers.UnitTests/Jd018ToJd022ReproducibilityTailTests.cs new file mode 100644 index 00000000..75511e07 --- /dev/null +++ b/JustDummies.Analyzers.UnitTests/Jd018ToJd022ReproducibilityTailTests.cs @@ -0,0 +1,382 @@ +using System.Collections.Immutable; + +using Microsoft.CodeAnalysis; + +using NFluent; + +namespace JustDummies.Analyzers.UnitTests; + +public class Jd018NestedReproducibilityScopeTests { + + [Fact] + public async Task Reports_a_runner_nested_in_another_runner() { + const string source = """ + using JustDummies; + + public static class Sample { + public static void M() { + Any.Reproducibly(() => { + Any.Reproducibly(() => { }); + }); + } + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new NestedReproducibilityScopeAnalyzer(), source); + + Check.That(diagnostics.Length).IsEqualTo(1); + Check.That(diagnostics[0].Id).IsEqualTo("JD018"); + Check.That(diagnostics[0].GetMessage()).Contains("another Any.Reproducibly scope"); + } + + [Fact] + public async Task Reports_a_runner_inside_a_reproducible_test() { + const string source = """ + using JustDummies; + using JustDummies.Xunit; + using Xunit; + + public class Sample { + [Fact, Reproducible] + public void T() { + Any.Reproducibly(() => { }); + } + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new NestedReproducibilityScopeAnalyzer(), source); + + Check.That(diagnostics.Length).IsEqualTo(1); + Check.That(diagnostics[0].GetMessage()).Contains("[Reproducible] test"); + } + + [Fact] + public async Task Does_not_report_the_seeded_overload() { + // Pinning a chosen seed inside is deliberate; only the seedless form silently overrides the outer instruction. + const string source = """ + using JustDummies; + using JustDummies.Xunit; + using Xunit; + + public class Sample { + [Fact, Reproducible] + public void T() { + Any.Reproducibly(1234, () => { }); + } + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new NestedReproducibilityScopeAnalyzer(), source); + + Check.That(diagnostics.Length).IsEqualTo(0); + } + + [Fact] + public async Task Does_not_report_a_lone_runner() { + const string source = """ + using JustDummies; + using Xunit; + + public class Sample { + [Fact] + public void T() { + Any.Reproducibly(() => { }); + } + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new NestedReproducibilityScopeAnalyzer(), source); + + Check.That(diagnostics.Length).IsEqualTo(0); + } + +} + +public class Jd019CommittedReplaySeedTests { + + [Theory] + [InlineData("Any.Reproducibly(1234, () => { });")] + [InlineData("Any.ReproduciblyAsync(1234, () => System.Threading.Tasks.Task.CompletedTask);")] + [InlineData("_ = Any.WithSeed(1234);")] + public async Task Reports_a_pinned_constant_seed(string statement) { + string source = $$""" + using JustDummies; + + public static class Sample { + public static void M() { + {{statement}} + } + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new CommittedReplaySeedAnalyzer(), source, "JD019"); + + Check.That(diagnostics.Length).IsEqualTo(1); + Check.That(diagnostics[0].Id).IsEqualTo("JD019"); + Check.That(diagnostics[0].GetMessage()).Contains("1234"); + } + + [Fact] + public async Task Reports_a_pinned_seed_on_the_attribute() { + const string source = """ + using JustDummies.Xunit; + using Xunit; + + public class Sample { + [Fact, Reproducible(Seed = 1234)] + public void T() { } + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new CommittedReplaySeedAnalyzer(), source, "JD019"); + + Check.That(diagnostics.Length).IsEqualTo(1); + } + + [Fact] + public async Task Does_not_report_the_seedless_form() { + const string source = """ + using JustDummies; + + public static class Sample { + public static void M() { + Any.Reproducibly(() => { }); + } + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new CommittedReplaySeedAnalyzer(), source, "JD019"); + + Check.That(diagnostics.Length).IsEqualTo(0); + } + + [Fact] + public async Task Does_not_report_a_computed_seed() { + const string source = """ + using JustDummies; + + public static class Sample { + public static void M(int seed) { + Any.Reproducibly(seed, () => { }); + } + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new CommittedReplaySeedAnalyzer(), source, "JD019"); + + Check.That(diagnostics.Length).IsEqualTo(0); + } + + [Fact] + public async Task Is_disabled_by_default() { + // The maintainer guide instructs pinning a seed for anything statistical, so a default-on rule would fight + // documented practice. + DiagnosticDescriptor descriptor = new CommittedReplaySeedAnalyzer().SupportedDiagnostics[0]; + + Check.That(descriptor.IsEnabledByDefault).IsFalse(); + } + +} + +public class Jd020SharedStaticAnyContextTests { + + [Fact] + public async Task Reports_a_static_context_field() { + const string source = """ + using JustDummies; + + public static class Sample { + private static readonly AnyContext Context = Any.WithSeed(1234); + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new SharedStaticAnyContextAnalyzer(), source); + + Check.That(diagnostics.Length).IsEqualTo(1); + Check.That(diagnostics[0].Id).IsEqualTo("JD020"); + Check.That(diagnostics[0].GetMessage()).Contains("Context"); + } + + [Fact] + public async Task Does_not_report_an_instance_context() { + const string source = """ + using JustDummies; + + public class Sample { + private readonly AnyContext _context = Any.WithSeed(1234); + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new SharedStaticAnyContextAnalyzer(), source); + + Check.That(diagnostics.Length).IsEqualTo(0); + } + + [Fact] + public async Task Does_not_report_a_static_generator() { + // A shared generator is safe: the random source is resolved at Generate(), not at construction. + const string source = """ + using JustDummies; + + public static class Sample { + private static readonly IAny Reference = Any.String().NonEmpty(); + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new SharedStaticAnyContextAnalyzer(), source); + + Check.That(diagnostics.Length).IsEqualTo(0); + } + +} + +public class Jd021BlankReplaySnippetTests { + + [Theory] + [InlineData("\"\"")] + [InlineData("\" \"")] + public async Task Reports_a_blank_snippet(string literal) { + string source = $$""" + using System; + using JustDummies; + + public static class Sample { + public static void M() { + using IDisposable scope = Any.UseSeed(1234, {{literal}}); + } + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new BlankReplaySnippetAnalyzer(), source); + + Check.That(diagnostics.Length).IsEqualTo(1); + Check.That(diagnostics[0].Id).IsEqualTo("JD021"); + } + + [Fact] + public async Task Does_not_report_a_real_snippet() { + const string source = """ + using System; + using JustDummies; + + public static class Sample { + public static void M() { + using IDisposable scope = Any.UseSeed(1234, "[Reproducible(Seed = 1234)]"); + } + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new BlankReplaySnippetAnalyzer(), source); + + Check.That(diagnostics.Length).IsEqualTo(0); + } + + [Fact] + public async Task Does_not_report_the_overload_without_a_snippet() { + const string source = """ + using System; + using JustDummies; + + public static class Sample { + public static void M() { + using IDisposable scope = Any.UseSeed(1234); + } + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new BlankReplaySnippetAnalyzer(), source); + + Check.That(diagnostics.Length).IsEqualTo(0); + } + + [Fact] + public async Task Does_not_report_a_guard_asserting_negative_test() { + // JustDummies.PropertyTests asserts exactly this rejection. + const string source = """ + using System; + using JustDummies; + + public static class Expect { + public static bool Throws(Func code) where T : Exception => true; + } + + public static class Sample { + public static void M(int seed) { + Expect.Throws(() => Any.UseSeed(seed, " ")); + } + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new BlankReplaySnippetAnalyzer(), source); + + Check.That(diagnostics.Length).IsEqualTo(0); + } + +} + +public class Jd022ParallelDrawWithoutPerItemSeedTests { + + [Fact] + public async Task Reports_a_draw_in_a_parallel_body_with_no_scope() { + const string source = """ + using System.Threading.Tasks; + using JustDummies; + + public static class Sample { + public static void M() { + Parallel.For(0, 64, index => { + string reference = Any.String().NonEmpty().Generate(); + }); + } + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new ParallelDrawWithoutPerItemSeedAnalyzer(), source); + + Check.That(diagnostics.Length).IsEqualTo(1); + Check.That(diagnostics[0].Id).IsEqualTo("JD022"); + } + + [Fact] + public async Task Does_not_report_a_body_that_opens_its_own_scope() { + // The documented shape: a scope inside the loop body gives each unit of work its own sequence. + const string source = """ + using System.Threading.Tasks; + using JustDummies; + + public static class Sample { + public static void M() { + const int runSeed = 20240501; + + Parallel.For(0, 64, index => { + using (Any.UseSeed(unchecked(runSeed * 397 ^ index))) { + string reference = Any.String().NonEmpty().Generate(); + } + }); + } + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new ParallelDrawWithoutPerItemSeedAnalyzer(), source); + + Check.That(diagnostics.Length).IsEqualTo(0); + } + + [Fact] + public async Task Does_not_report_a_sequential_draw() { + const string source = """ + using JustDummies; + + public static class Sample { + public static void M() { + string reference = Any.String().NonEmpty().Generate(); + } + } + """; + + ImmutableArray diagnostics = await AnalyzerTestHarness.GetDiagnosticsAsync(new ParallelDrawWithoutPerItemSeedAnalyzer(), source); + + Check.That(diagnostics.Length).IsEqualTo(0); + } + +} diff --git a/JustDummies.Analyzers/AnalyzerReleases.Unshipped.md b/JustDummies.Analyzers/AnalyzerReleases.Unshipped.md index d07a120f..f1152b03 100644 --- a/JustDummies.Analyzers/AnalyzerReleases.Unshipped.md +++ b/JustDummies.Analyzers/AnalyzerReleases.Unshipped.md @@ -22,3 +22,8 @@ JD014 | JustDummies.Constraints | Warning | RejectedConstantArgumentAnaly JD015 | JustDummies.Constraints | Warning | StringConstraintsAdmitNoValueAnalyzer JD016 | JustDummies.Constraints | Warning | CollectionConstraintsAdmitNoValueAnalyzer JD017 | JustDummies.Constraints | Warning | EnumUniverseViolationAnalyzer +JD018 | JustDummies.Reproducibility | Warning | NestedReproducibilityScopeAnalyzer +JD019 | JustDummies.Reproducibility | Disabled | CommittedReplaySeedAnalyzer +JD020 | JustDummies.Reproducibility | Info | SharedStaticAnyContextAnalyzer +JD021 | JustDummies.Reproducibility | Warning | BlankReplaySnippetAnalyzer +JD022 | JustDummies.Reproducibility | Info | ParallelDrawWithoutPerItemSeedAnalyzer diff --git a/JustDummies.Analyzers/BlankReplaySnippetAnalyzer.cs b/JustDummies.Analyzers/BlankReplaySnippetAnalyzer.cs new file mode 100644 index 00000000..93b7b182 --- /dev/null +++ b/JustDummies.Analyzers/BlankReplaySnippetAnalyzer.cs @@ -0,0 +1,56 @@ +using System.Collections.Immutable; + +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Diagnostics; +using Microsoft.CodeAnalysis.Operations; + +namespace JustDummies.Analyzers; + +/// +/// JD021 — reports a blank replay snippet handed to Any.UseSeed(int, string). The guard rejects it at run +/// time, and because that scope is normally opened from a test-framework adapter's hook, the throw surfaces as an +/// infrastructure failure on every test in the suite rather than as one failing assertion — a +/// disproportionately expensive way to learn about a typo the compiler can already see. +/// +[DiagnosticAnalyzer(LanguageNames.CSharp)] +public sealed class BlankReplaySnippetAnalyzer : DiagnosticAnalyzer { + + /// + public override ImmutableArray SupportedDiagnostics { get; } = + ImmutableArray.Create(Descriptors.BlankReplaySnippet); + + /// + public override void Initialize(AnalysisContext context) { + context.EnableConcurrentExecution(); + context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.None); + context.RegisterCompilationStartAction(OnCompilationStart); + } + + private static void OnCompilationStart(CompilationStartAnalysisContext context) { + KnownSymbols symbols = KnownSymbols.From(context.Compilation); + if (symbols.Any is null) { return; } + + INamedTypeSymbol any = symbols.Any; + + context.RegisterOperationAction(operationContext => Analyze(operationContext, any), OperationKind.Invocation); + } + + private static void Analyze(OperationAnalysisContext context, INamedTypeSymbol any) { + IInvocationOperation invocation = (IInvocationOperation)context.Operation; + + if (invocation.TargetMethod.Name != "UseSeed") { return; } + if (!SymbolEqualityComparer.Default.Equals(invocation.TargetMethod.ContainingType, any)) { return; } + if (NegativeTestGuard.IsSoleBodyOfLambdaArgument(invocation.Syntax)) { return; } + + foreach (IArgumentOperation argument in invocation.Arguments) { + if (argument.Parameter?.Name != "replaySnippet") { continue; } + if (!ConstantFacts.TryGetString(argument.Value, out string snippet)) { continue; } + if (snippet.Trim().Length != 0) { continue; } + + context.ReportDiagnostic(Diagnostic.Create(Descriptors.BlankReplaySnippet, argument.Value.Syntax.GetLocation())); + + return; + } + } + +} diff --git a/JustDummies.Analyzers/CommittedReplaySeedAnalyzer.cs b/JustDummies.Analyzers/CommittedReplaySeedAnalyzer.cs new file mode 100644 index 00000000..a053e25e --- /dev/null +++ b/JustDummies.Analyzers/CommittedReplaySeedAnalyzer.cs @@ -0,0 +1,81 @@ +using System.Collections.Generic; +using System.Collections.Immutable; + +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Diagnostics; +using Microsoft.CodeAnalysis.Operations; + +namespace JustDummies.Analyzers; + +/// +/// JD019 — reports a replay seed pinned in committed code. The seeded overloads exist to replay a run a +/// failure reported: correct while you are reproducing, wrong the moment it is committed, because the test then +/// draws the same values for ever and stops surfacing the coupling the library exists to reveal. +/// +/// +/// Opt-in, and it must be. This repository's own maintainer guide instructs the opposite for a whole class of +/// tests — "Pin a seed for anything statistical" — so a rule enabled by default would fight documented practice. +/// It earns its keep as a pre-release sweep, not as a standing check. +/// +[DiagnosticAnalyzer(LanguageNames.CSharp)] +public sealed class CommittedReplaySeedAnalyzer : DiagnosticAnalyzer { + + private const string SeedPropertyName = "Seed"; + + /// + public override ImmutableArray SupportedDiagnostics { get; } = + ImmutableArray.Create(Descriptors.CommittedReplaySeed); + + /// + public override void Initialize(AnalysisContext context) { + context.EnableConcurrentExecution(); + context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.None); + context.RegisterCompilationStartAction(OnCompilationStart); + } + + private static void OnCompilationStart(CompilationStartAnalysisContext context) { + KnownSymbols symbols = KnownSymbols.From(context.Compilation); + if (symbols.Any is null) { return; } + + context.RegisterOperationAction(operationContext => AnalyzeInvocation(operationContext, symbols), OperationKind.Invocation); + + if (symbols.ReproducibleAttribute is not null) { + context.RegisterSymbolAction(symbolContext => AnalyzeAttribute(symbolContext, symbols.ReproducibleAttribute), SymbolKind.Method, SymbolKind.NamedType); + } + } + + private static void AnalyzeInvocation(OperationAnalysisContext context, KnownSymbols symbols) { + IInvocationOperation invocation = (IInvocationOperation)context.Operation; + + if (invocation.TargetMethod.Name is not ("Reproducibly" or "ReproduciblyAsync" or "UseSeed" or "WithSeed")) { return; } + if (!SymbolEqualityComparer.Default.Equals(invocation.TargetMethod.ContainingType, symbols.Any)) { return; } + + foreach (IArgumentOperation argument in invocation.Arguments) { + if (argument.Parameter?.Name != "seed") { continue; } + if (argument.ArgumentKind == ArgumentKind.DefaultValue) { continue; } + if (!ConstantFacts.TryGetInt32(argument.Value, out int seed)) { continue; } + + context.ReportDiagnostic(Diagnostic.Create(Descriptors.CommittedReplaySeed, argument.Value.Syntax.GetLocation(), seed)); + + return; + } + } + + private static void AnalyzeAttribute(SymbolAnalysisContext context, INamedTypeSymbol reproducibleAttribute) { + foreach (AttributeData attribute in context.Symbol.GetAttributes()) { + if (!SymbolEqualityComparer.Default.Equals(attribute.AttributeClass, reproducibleAttribute)) { continue; } + + foreach (KeyValuePair named in attribute.NamedArguments) { + if (named.Key != SeedPropertyName || named.Value.Value is not int seed) { continue; } + + Location location = attribute.ApplicationSyntaxReference?.GetSyntax(context.CancellationToken).GetLocation() + ?? context.Symbol.Locations[0]; + + context.ReportDiagnostic(Diagnostic.Create(Descriptors.CommittedReplaySeed, location, seed)); + + return; + } + } + } + +} diff --git a/JustDummies.Analyzers/Descriptors.cs b/JustDummies.Analyzers/Descriptors.cs index ef291b64..3676fdd1 100644 --- a/JustDummies.Analyzers/Descriptors.cs +++ b/JustDummies.Analyzers/Descriptors.cs @@ -182,4 +182,56 @@ internal static class Descriptors { description: "Any.Enum() draws uniformly across T's declared members and never an undeclared numeric value. That is deliberate and surprising: on a [Flags] enum, writing a combination in OneOf is the natural thing to do and the generator refuses it unless AllowingCombinations() is declared. An exclusion that removes every declared member is the same category error from the other side.", helpLinkUri: HelpLinks.For(DiagnosticIds.EnumUniverseViolation)); + public static readonly DiagnosticDescriptor NestedReproducibilityScope = new( + id: DiagnosticIds.NestedReproducibilityScope, + title: "A reproducibility scope is nested inside another", + messageFormat: "This Any.Reproducibly runs inside {0}, whose reported seed then replays nothing: the inner scope draws a fresh seed on every run", + category: DiagnosticCategories.Reproducibility, + defaultSeverity: DiagnosticSeverity.Warning, + isEnabledByDefault: true, + description: "Any.Reproducibly takes its seed from Guid.NewGuid().GetHashCode(), not from the ambient source, so an inner scope ignores whatever the outer one pinned and draws afresh every run. The outer mechanism still reports its own seed, so the failure names a seed that reproduces nothing — a wrong instruction rather than a wrong result. The seeded overload is left alone: pinning a chosen seed inside is deliberate.", + helpLinkUri: HelpLinks.For(DiagnosticIds.NestedReproducibilityScope)); + + public static readonly DiagnosticDescriptor CommittedReplaySeed = new( + id: DiagnosticIds.CommittedReplaySeed, + title: "A replay seed is pinned in committed code", + messageFormat: "Seed {0} is pinned: the values stop varying between runs, so the test no longer surfaces a dependency on one particular value", + category: DiagnosticCategories.Reproducibility, + defaultSeverity: DiagnosticSeverity.Info, + // Opt-in, and it must be: this repository's own maintainer guide instructs the opposite for a whole class of + // tests ("Pin a seed for anything statistical"), so a rule enabled by default would fight documented practice. + isEnabledByDefault: false, + description: "The seeded overloads exist to replay a run a failure reported — correct while reproducing, wrong once committed, because the test then draws the same values for ever and stops surfacing the coupling the library exists to reveal. Opt-in: a statistical test legitimately pins a seed, and this repository's maintainer guide says so, which makes the rule a pre-release sweep rather than a standing check.", + helpLinkUri: HelpLinks.For(DiagnosticIds.CommittedReplaySeed)); + + public static readonly DiagnosticDescriptor SharedStaticAnyContext = new( + id: DiagnosticIds.SharedStaticAnyContext, + title: "An AnyContext is shared through a static field", + messageFormat: "Give each unit of work its own context: '{0}' is shared, and interleaved draws make neither the sequence nor the multiset stable across runs", + category: DiagnosticCategories.Reproducibility, + defaultSeverity: DiagnosticSeverity.Info, + isEnabledByDefault: true, + description: "AnyContext's own documentation states the hazard: a context is safe to draw from concurrently, but sharing one across threads costs the replay rather than the values. A static context looks maximally deterministic — a literal seed, right there in the source — while a parallel suite gets a different value per test per run from it.", + helpLinkUri: HelpLinks.For(DiagnosticIds.SharedStaticAnyContext)); + + public static readonly DiagnosticDescriptor BlankReplaySnippet = new( + id: DiagnosticIds.BlankReplaySnippet, + title: "Any.UseSeed is given a blank replay snippet", + messageFormat: "Pass the code a reader copies to replay the run, or drop the argument: a blank snippet is rejected at run time", + category: DiagnosticCategories.Reproducibility, + defaultSeverity: DiagnosticSeverity.Warning, + isEnabledByDefault: true, + description: "Any.UseSeed(int, string) rejects a blank snippet. Because that scope is normally opened from a test-framework adapter's hook, the throw surfaces as an infrastructure failure on every test in the suite rather than as one failing assertion — a disproportionately expensive way to learn about a typo the compiler can already see.", + helpLinkUri: HelpLinks.For(DiagnosticIds.BlankReplaySnippet)); + + public static readonly DiagnosticDescriptor ParallelDrawWithoutPerItemSeed = new( + id: DiagnosticIds.ParallelDrawWithoutPerItemSeed, + title: "A parallel work item draws without its own seed scope", + messageFormat: "Open an Any.UseSeed scope inside the work item: the ambient scope reaches every worker, so the draws interleave and the run replays nothing", + category: DiagnosticCategories.Reproducibility, + defaultSeverity: DiagnosticSeverity.Info, + isEnabledByDefault: true, + description: "The ambient seed scope flows with the execution context, so a scope opened around a parallel loop reaches every worker and their draws interleave: neither the sequence nor the multiset is stable across runs. A scope opened inside the loop body gives each unit of work its own sequence, and the whole run replays — the shape the library's documentation names.", + helpLinkUri: HelpLinks.For(DiagnosticIds.ParallelDrawWithoutPerItemSeed)); + } diff --git a/JustDummies.Analyzers/DiagnosticIds.cs b/JustDummies.Analyzers/DiagnosticIds.cs index bec1e951..3e9f8924 100644 --- a/JustDummies.Analyzers/DiagnosticIds.cs +++ b/JustDummies.Analyzers/DiagnosticIds.cs @@ -33,4 +33,11 @@ internal static class DiagnosticIds { public const string CollectionConstraintsAdmitNoValue = "JD016"; public const string EnumUniverseViolation = "JD017"; + // Category: Reproducibility — the seeding long tail + public const string NestedReproducibilityScope = "JD018"; + public const string CommittedReplaySeed = "JD019"; + public const string SharedStaticAnyContext = "JD020"; + public const string BlankReplaySnippet = "JD021"; + public const string ParallelDrawWithoutPerItemSeed = "JD022"; + } diff --git a/JustDummies.Analyzers/NestedReproducibilityScopeAnalyzer.cs b/JustDummies.Analyzers/NestedReproducibilityScopeAnalyzer.cs new file mode 100644 index 00000000..04cd30f5 --- /dev/null +++ b/JustDummies.Analyzers/NestedReproducibilityScopeAnalyzer.cs @@ -0,0 +1,84 @@ +using System.Collections.Immutable; + +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Diagnostics; +using Microsoft.CodeAnalysis.Operations; + +namespace JustDummies.Analyzers; + +/// +/// JD018 — reports a reproducibility scope opened inside another one. Both mechanisms report a replay +/// instruction, and nesting makes the outer instruction false: Any.Reproducibly takes its seed from +/// Guid.NewGuid().GetHashCode(), not from the ambient source, so the inner scope draws a brand-new seed on +/// every run whatever the outer one pinned. The failure names a seed that reproduces nothing. +/// +[DiagnosticAnalyzer(LanguageNames.CSharp)] +public sealed class NestedReproducibilityScopeAnalyzer : DiagnosticAnalyzer { + + /// + public override ImmutableArray SupportedDiagnostics { get; } = + ImmutableArray.Create(Descriptors.NestedReproducibilityScope); + + /// + public override void Initialize(AnalysisContext context) { + context.EnableConcurrentExecution(); + context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.None); + context.RegisterCompilationStartAction(OnCompilationStart); + } + + private static void OnCompilationStart(CompilationStartAnalysisContext context) { + KnownSymbols symbols = KnownSymbols.From(context.Compilation); + if (symbols.Any is null) { return; } + + context.RegisterOperationAction(operationContext => Analyze(operationContext, symbols), OperationKind.Invocation); + } + + private static void Analyze(OperationAnalysisContext context, KnownSymbols symbols) { + IInvocationOperation invocation = (IInvocationOperation)context.Operation; + + if (!IsRunner(invocation, symbols)) { return; } + + // The seeded overload pins a chosen seed deliberately; only the seedless form silently overrides the outer + // instruction with one nobody recorded. + if (HasSeedArgument(invocation)) { return; } + + if (IsInsideAnotherRunner(invocation, symbols)) { + Report(context, invocation, "another Any.Reproducibly scope"); + + return; + } + + if (symbols.ReproducibleAttribute is null || symbols.FactAttribute is null) { return; } + if (context.ContainingSymbol is not IMethodSymbol method) { return; } + if (!XunitFacts.IsTestMethod(method, symbols.FactAttribute)) { return; } + if (!XunitFacts.IsCoveredByReproducible(method, symbols.ReproducibleAttribute)) { return; } + + Report(context, invocation, "a [Reproducible] test"); + } + + private static void Report(OperationAnalysisContext context, IInvocationOperation invocation, string outer) { + context.ReportDiagnostic(Diagnostic.Create(Descriptors.NestedReproducibilityScope, invocation.Syntax.GetLocation(), outer)); + } + + private static bool IsRunner(IInvocationOperation invocation, KnownSymbols symbols) { + return invocation.TargetMethod.Name is "Reproducibly" or "ReproduciblyAsync" + && SymbolEqualityComparer.Default.Equals(invocation.TargetMethod.ContainingType, symbols.Any); + } + + private static bool HasSeedArgument(IInvocationOperation invocation) { + foreach (IArgumentOperation argument in invocation.Arguments) { + if (argument.Parameter?.Name == "seed" && argument.ArgumentKind != ArgumentKind.DefaultValue) { return true; } + } + + return false; + } + + private static bool IsInsideAnotherRunner(IInvocationOperation invocation, KnownSymbols symbols) { + for (IOperation? current = invocation.Parent; current is not null; current = current.Parent) { + if (current is IInvocationOperation outer && !ReferenceEquals(outer, invocation) && IsRunner(outer, symbols)) { return true; } + } + + return false; + } + +} diff --git a/JustDummies.Analyzers/ParallelDrawWithoutPerItemSeedAnalyzer.cs b/JustDummies.Analyzers/ParallelDrawWithoutPerItemSeedAnalyzer.cs new file mode 100644 index 00000000..535e76c3 --- /dev/null +++ b/JustDummies.Analyzers/ParallelDrawWithoutPerItemSeedAnalyzer.cs @@ -0,0 +1,83 @@ +using System.Collections.Immutable; + +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Diagnostics; +using Microsoft.CodeAnalysis.Operations; + +namespace JustDummies.Analyzers; + +/// +/// JD022 — reports an ambient draw inside a Parallel work item that opens no seed scope of its own. The +/// ambient scope flows into every worker, so one shared scope reaches them all and the draws interleave: the +/// sequence is stable for nobody and the run replays nothing. The library's own documentation names this shape — +/// a scope opened inside the loop body gives each unit of work its own sequence, and the whole run replays. +/// +[DiagnosticAnalyzer(LanguageNames.CSharp)] +public sealed class ParallelDrawWithoutPerItemSeedAnalyzer : DiagnosticAnalyzer { + + private const string ParallelMetadataName = "System.Threading.Tasks.Parallel"; + + /// + public override ImmutableArray SupportedDiagnostics { get; } = + ImmutableArray.Create(Descriptors.ParallelDrawWithoutPerItemSeed); + + /// + public override void Initialize(AnalysisContext context) { + context.EnableConcurrentExecution(); + context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.None); + context.RegisterCompilationStartAction(OnCompilationStart); + } + + private static void OnCompilationStart(CompilationStartAnalysisContext context) { + KnownSymbols symbols = KnownSymbols.From(context.Compilation); + if (symbols.Any is null || symbols.IAny is null) { return; } + + INamedTypeSymbol? parallel = context.Compilation.GetTypeByMetadataName(ParallelMetadataName); + if (parallel is null) { return; } + + context.RegisterOperationAction(operationContext => Analyze(operationContext, symbols, parallel), OperationKind.Invocation); + } + + private static void Analyze(OperationAnalysisContext context, KnownSymbols symbols, INamedTypeSymbol parallel) { + IInvocationOperation invocation = (IInvocationOperation)context.Operation; + + if (!GeneratorFacts.IsGenerateCall(invocation, symbols.IAny!)) { return; } + if (!GeneratorFacts.RootsAtAmbientAny(invocation, symbols.Any!)) { return; } + + IAnonymousFunctionOperation? body = EnclosingParallelBody(invocation, parallel); + if (body is null) { return; } + if (OpensASeedScope(body, symbols.Any!)) { return; } + + context.ReportDiagnostic(Diagnostic.Create(Descriptors.ParallelDrawWithoutPerItemSeed, invocation.Syntax.GetLocation())); + } + + // The innermost lambda that is an argument to a Parallel.* call — the work item. + private static IAnonymousFunctionOperation? EnclosingParallelBody(IOperation operation, INamedTypeSymbol parallel) { + for (IOperation? current = operation.Parent; current is not null; current = current.Parent) { + if (current is not IAnonymousFunctionOperation lambda) { continue; } + + for (IOperation? outer = lambda.Parent; outer is not null; outer = outer.Parent) { + if (outer is IInvocationOperation call) { + return SymbolEqualityComparer.Default.Equals(call.TargetMethod.ContainingType, parallel) ? lambda : null; + } + + if (outer is IAnonymousFunctionOperation) { break; } + } + } + + return null; + } + + private static bool OpensASeedScope(IOperation node, INamedTypeSymbol any) { + if (node is IInvocationOperation call + && call.TargetMethod.Name == "UseSeed" + && SymbolEqualityComparer.Default.Equals(call.TargetMethod.ContainingType, any)) { return true; } + + foreach (IOperation child in node.ChildOperations) { + if (OpensASeedScope(child, any)) { return true; } + } + + return false; + } + +} diff --git a/JustDummies.Analyzers/SharedStaticAnyContextAnalyzer.cs b/JustDummies.Analyzers/SharedStaticAnyContextAnalyzer.cs new file mode 100644 index 00000000..007d548f --- /dev/null +++ b/JustDummies.Analyzers/SharedStaticAnyContextAnalyzer.cs @@ -0,0 +1,50 @@ +using System.Collections.Immutable; + +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Diagnostics; + +namespace JustDummies.Analyzers; + +/// +/// JD020 — reports an AnyContext held in a static field. It looks maximally deterministic — a literal seed, +/// right there in the source — and is not: the type's own documentation states that sharing one context across +/// threads "costs the replay rather than the values", because interleaved draws make neither the sequence nor the +/// multiset stable. A suite that runs its classes in parallel therefore gets a different value per test per run, +/// from a context that reads as pinned. +/// +[DiagnosticAnalyzer(LanguageNames.CSharp)] +public sealed class SharedStaticAnyContextAnalyzer : DiagnosticAnalyzer { + + /// + public override ImmutableArray SupportedDiagnostics { get; } = + ImmutableArray.Create(Descriptors.SharedStaticAnyContext); + + /// + public override void Initialize(AnalysisContext context) { + context.EnableConcurrentExecution(); + context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.None); + context.RegisterCompilationStartAction(OnCompilationStart); + } + + private static void OnCompilationStart(CompilationStartAnalysisContext context) { + KnownSymbols symbols = KnownSymbols.From(context.Compilation); + if (symbols.AnyContext is null) { return; } + + INamedTypeSymbol anyContext = symbols.AnyContext; + + context.RegisterSymbolAction(symbolContext => Analyze(symbolContext, anyContext), SymbolKind.Field, SymbolKind.Property); + } + + private static void Analyze(SymbolAnalysisContext context, INamedTypeSymbol anyContext) { + ITypeSymbol? type = context.Symbol switch { + IFieldSymbol { IsStatic: true } field => field.Type, + IPropertySymbol { IsStatic: true } property => property.Type, + _ => null, + }; + + if (type is null || !SymbolEqualityComparer.Default.Equals(type, anyContext)) { return; } + + context.ReportDiagnostic(Diagnostic.Create(Descriptors.SharedStaticAnyContext, context.Symbol.Locations[0], context.Symbol.Name)); + } + +} diff --git a/doc/handwritten/for-users/analyzers/JD018.en.md b/doc/handwritten/for-users/analyzers/JD018.en.md new file mode 100644 index 00000000..74649e8d --- /dev/null +++ b/doc/handwritten/for-users/analyzers/JD018.en.md @@ -0,0 +1,43 @@ +# JD018: NestedReproducibilityScope + +🌍 **Languages:** +🇬🇧 English (this file) | 🇫🇷 [Français](./JD018.fr.md) + +| | | +|---|---| +| **Category** | Reproducibility (`JustDummies.Reproducibility`) | +| **Severity** | 🟠 Warning | +| **Enabled by default** | Yes | + +Both mechanisms report a replay instruction, and nesting makes the outer one **false**. + +`Any.Reproducibly` takes its seed from `Guid.NewGuid().GetHashCode()` — not from the ambient source — so an inner scope ignores whatever the outer one pinned and draws afresh on every run. The outer runner, or `[Reproducible]`, still reports *its* seed. The failure therefore says "reproduce this run with seed N", and replaying with N reproduces nothing. + +That is a wrong instruction rather than a wrong result, which is what makes it worth reporting: the reader trusts it and loses time before doubting it. + +## Noncompliant + +```csharp +[Fact, Reproducible] +public void It_is_accepted() { + Any.Reproducibly(() => { /* ... */ }); // JD018: the attribute's seed governs nothing in here +} +``` + +## Compliant + +```csharp +[Fact, Reproducible] +public void It_is_accepted() { + /* ... */ // the attribute already pins the whole test +} +``` + +## What it does not flag + +* The **seeded** overload — `Any.Reproducibly(1234, ...)`. Pinning a chosen seed inside is deliberate, and the reader who wrote it knows which seed governs what. +* A lone runner in a test that carries no `[Reproducible]`. + +--- + +[← All analyzer rules](README.md) diff --git a/doc/handwritten/for-users/analyzers/JD018.fr.md b/doc/handwritten/for-users/analyzers/JD018.fr.md new file mode 100644 index 00000000..328fb488 --- /dev/null +++ b/doc/handwritten/for-users/analyzers/JD018.fr.md @@ -0,0 +1,43 @@ +# JD018 : NestedReproducibilityScope + +🌍 **Langues :** +🇫🇷 Français (ce fichier) | 🇬🇧 [English](./JD018.en.md) + +| | | +|---|---| +| **Catégorie** | Reproductibilité (`JustDummies.Reproducibility`) | +| **Sévérité** | 🟠 Avertissement | +| **Activée par défaut** | Oui | + +Les deux mécanismes rapportent une instruction de rejeu, et l'imbrication rend celle de l'extérieur **fausse**. + +`Any.Reproducibly` prend sa graine de `Guid.NewGuid().GetHashCode()` — et non de la source ambiante — de sorte qu'une portée interne ignore ce que l'externe a épinglé et tire à neuf à chaque exécution. Le lanceur externe, ou `[Reproducible]`, rapporte pourtant *sa* graine. L'échec dit donc « rejouez avec la graine N », et rejouer avec N ne reproduit rien. + +C'est une instruction fausse plutôt qu'un résultat faux, et c'est ce qui justifie de la signaler : le lecteur lui fait confiance et perd du temps avant d'en douter. + +## Non conforme + +```csharp +[Fact, Reproducible] +public void It_is_accepted() { + Any.Reproducibly(() => { /* ... */ }); // JD018 : la graine de l'attribut ne gouverne rien ici +} +``` + +## Conforme + +```csharp +[Fact, Reproducible] +public void It_is_accepted() { + /* ... */ // l'attribut épingle déjà tout le test +} +``` + +## Ce qui n'est pas signalé + +* La surcharge **avec graine** — `Any.Reproducibly(1234, ...)`. Épingler une graine choisie à l'intérieur est délibéré, et celui qui l'écrit sait quelle graine gouverne quoi. +* Un lanceur isolé dans un test qui ne porte aucun `[Reproducible]`. + +--- + +[← Toutes les règles d'analyse](README.fr.md) diff --git a/doc/handwritten/for-users/analyzers/JD019.en.md b/doc/handwritten/for-users/analyzers/JD019.en.md new file mode 100644 index 00000000..3c7b6d56 --- /dev/null +++ b/doc/handwritten/for-users/analyzers/JD019.en.md @@ -0,0 +1,50 @@ +# JD019: CommittedReplaySeed + +🌍 **Languages:** +🇬🇧 English (this file) | 🇫🇷 [Français](./JD019.fr.md) + +| | | +|---|---| +| **Category** | Reproducibility (`JustDummies.Reproducibility`) | +| **Severity** | 🔵 Info | +| **Enabled by default** | **No — opt-in** | + +The seeded overloads exist to **replay** a run a failure reported. That is correct while you are reproducing and wrong the moment it is committed: the test then draws the same values for ever, and stops surfacing the dependency on one particular value that arbitrary-by-default exists to reveal. + +It is the `fit`/`.only` of this library — right in the moment, wrong in the repository. + +## Enabling it + +```ini +dotnet_diagnostic.JD019.severity = warning +``` + +## Noncompliant + +```csharp +[Fact, Reproducible(Seed = 1234)] // JD019 +Any.Reproducibly(1234, () => { ... }); // JD019 +Any.WithSeed(1234) // JD019 +``` + +## Compliant + +```csharp +[Fact, Reproducible] +Any.Reproducibly(() => { ... }); +``` + +## Why it ships opt-in + +Because this repository's own maintainer guide instructs the opposite for a whole class of tests: *"Pin a seed for anything statistical."* A rule enabled by default would fight documented practice. + +The measurement is blunt. Enabled across `JustDummies.UnitTests`, it reports **238 sites** — nearly all of them legitimate, deliberate pins on distribution and coverage tests. It earns its keep as a **pre-release sweep** you turn on deliberately, not as a standing check. + +## What it does not flag + +* A seed read from configuration, computed, or passed as a parameter — only a compile-time constant is reported. +* The seedless overloads, which are the arbitrary-by-default path. + +--- + +[← All analyzer rules](README.md) diff --git a/doc/handwritten/for-users/analyzers/JD019.fr.md b/doc/handwritten/for-users/analyzers/JD019.fr.md new file mode 100644 index 00000000..b068b5ab --- /dev/null +++ b/doc/handwritten/for-users/analyzers/JD019.fr.md @@ -0,0 +1,50 @@ +# JD019 : CommittedReplaySeed + +🌍 **Langues :** +🇫🇷 Français (ce fichier) | 🇬🇧 [English](./JD019.en.md) + +| | | +|---|---| +| **Catégorie** | Reproductibilité (`JustDummies.Reproducibility`) | +| **Sévérité** | 🔵 Info | +| **Activée par défaut** | **Non — opt-in** | + +Les surcharges avec graine existent pour **rejouer** une exécution qu'un échec a rapportée. C'est correct pendant qu'on reproduit, et faux dès que c'est committé : le test tire alors les mêmes valeurs pour toujours, et cesse de révéler la dépendance à une valeur particulière que l'« arbitraire par défaut » existe pour mettre au jour. + +C'est le `fit`/`.only` de cette bibliothèque — juste sur le moment, faux dans le dépôt. + +## Comment l'activer + +```ini +dotnet_diagnostic.JD019.severity = warning +``` + +## Non conforme + +```csharp +[Fact, Reproducible(Seed = 1234)] // JD019 +Any.Reproducibly(1234, () => { ... }); // JD019 +Any.WithSeed(1234) // JD019 +``` + +## Conforme + +```csharp +[Fact, Reproducible] +Any.Reproducibly(() => { ... }); +``` + +## Pourquoi elle est livrée en opt-in + +Parce que le guide de maintenance de ce dépôt prescrit l'inverse pour toute une classe de tests : « *Pin a seed for anything statistical.* » Une règle activée par défaut combattrait une pratique documentée. + +La mesure est sans appel. Activée sur `JustDummies.UnitTests`, elle signale **238 sites** — presque tous légitimes, des épinglages délibérés sur des tests de distribution et de couverture. Elle gagne sa place comme **balayage d'avant-livraison** qu'on active exprès, pas comme vérification permanente. + +## Ce qui n'est pas signalé + +* Une graine lue depuis une configuration, calculée, ou passée en paramètre — seule une constante de compilation est signalée. +* Les surcharges sans graine, qui sont le chemin arbitraire par défaut. + +--- + +[← Toutes les règles d'analyse](README.fr.md) diff --git a/doc/handwritten/for-users/analyzers/JD020.en.md b/doc/handwritten/for-users/analyzers/JD020.en.md new file mode 100644 index 00000000..3eabc590 --- /dev/null +++ b/doc/handwritten/for-users/analyzers/JD020.en.md @@ -0,0 +1,43 @@ +# JD020: SharedStaticAnyContext + +🌍 **Languages:** +🇬🇧 English (this file) | 🇫🇷 [Français](./JD020.fr.md) + +| | | +|---|---| +| **Category** | Reproducibility (`JustDummies.Reproducibility`) | +| **Severity** | 🔵 Info | +| **Enabled by default** | Yes | + +A static `AnyContext` looks maximally deterministic — a literal seed, right there in the source — and is not. `AnyContext`'s own documentation states the hazard: + +> A context is safe to draw from concurrently, but sharing one across threads costs the **replay** rather than the values: interleaved draws make neither the sequence nor the multiset stable across runs. Keep a context to one thread at a time. + +So in a suite that runs its classes in parallel, each test gets whatever the interleaving happened to hand it. The seed is pinned and the run still does not replay. + +## Noncompliant + +```csharp +private static readonly AnyContext Context = Any.WithSeed(1234); // JD020 +``` + +## Compliant + +```csharp +// One context per unit of work… +AnyContext context = Any.WithSeed(1234); + +// …or the ambient scope, which flows with the execution context: +using IDisposable scope = Any.UseSeed(1234); +``` + +## What it does not flag + +* An **instance** context, which is per-test by construction. +* A static field holding a **generator**: the random source is resolved at `Generate()` time, never at construction, so a shared recipe is safe and idiomatic. + +Info rather than Warning: a single-threaded consumer — a console sample, a benchmark, a `[Collection]`-serialised class — shares one harmlessly, and the rule cannot see the suite's parallelism configuration. + +--- + +[← All analyzer rules](README.md) diff --git a/doc/handwritten/for-users/analyzers/JD020.fr.md b/doc/handwritten/for-users/analyzers/JD020.fr.md new file mode 100644 index 00000000..5406621a --- /dev/null +++ b/doc/handwritten/for-users/analyzers/JD020.fr.md @@ -0,0 +1,43 @@ +# JD020 : SharedStaticAnyContext + +🌍 **Langues :** +🇫🇷 Français (ce fichier) | 🇬🇧 [English](./JD020.en.md) + +| | | +|---|---| +| **Catégorie** | Reproductibilité (`JustDummies.Reproducibility`) | +| **Sévérité** | 🔵 Info | +| **Activée par défaut** | Oui | + +Un `AnyContext` statique paraît maximalement déterministe — une graine littérale, là, dans le source — et ne l'est pas. La documentation d'`AnyContext` énonce elle-même le risque : + +> Un contexte peut être tiré concurremment sans danger, mais en partager un entre threads coûte le **rejeu** plutôt que les valeurs : des tirages entrelacés ne rendent stables ni la séquence ni le multiensemble d'une exécution à l'autre. Gardez un contexte à un seul thread à la fois. + +Dans une suite qui exécute ses classes en parallèle, chaque test reçoit donc ce que l'entrelacement lui a donné. La graine est épinglée et l'exécution ne rejoue toujours pas. + +## Non conforme + +```csharp +private static readonly AnyContext Context = Any.WithSeed(1234); // JD020 +``` + +## Conforme + +```csharp +// Un contexte par unité de travail… +AnyContext context = Any.WithSeed(1234); + +// …ou la portée ambiante, qui suit le contexte d'exécution : +using IDisposable scope = Any.UseSeed(1234); +``` + +## Ce qui n'est pas signalé + +* Un contexte **d'instance**, qui est par test par construction. +* Un champ statique portant un **générateur** : la source aléatoire est résolue au moment de `Generate()`, jamais à la construction, donc une recette partagée est sûre et idiomatique. + +Info plutôt qu'avertissement : un consommateur monothread — un exemple console, un benchmark, une classe sérialisée par `[Collection]` — en partage un sans dommage, et la règle ne voit pas la configuration de parallélisme de la suite. + +--- + +[← Toutes les règles d'analyse](README.fr.md) diff --git a/doc/handwritten/for-users/analyzers/JD021.en.md b/doc/handwritten/for-users/analyzers/JD021.en.md new file mode 100644 index 00000000..23daf3ba --- /dev/null +++ b/doc/handwritten/for-users/analyzers/JD021.en.md @@ -0,0 +1,41 @@ +# JD021: BlankReplaySnippet + +🌍 **Languages:** +🇬🇧 English (this file) | 🇫🇷 [Français](./JD021.fr.md) + +| | | +|---|---| +| **Category** | Reproducibility (`JustDummies.Reproducibility`) | +| **Severity** | 🟠 Warning | +| **Enabled by default** | Yes | + +`Any.UseSeed(int, string)` supplies the **replay snippet** — the code a reader copies to replay the run — that generation-failure guidance quotes verbatim. It rejects a blank one at run time. + +What makes the compile-time check worth having is *where* that throw lands. This overload exists for a test-framework adapter, which opens the scope from a hook that runs before every test. A blank snippet therefore fails the whole suite as an infrastructure error, not one assertion — a disproportionately expensive way to learn about a typo the compiler can already see. + +## Noncompliant + +```csharp +using IDisposable scope = Any.UseSeed(1234, ""); // JD021 +using IDisposable scope = Any.UseSeed(1234, " "); // JD021 +``` + +## Compliant + +```csharp +using IDisposable scope = Any.UseSeed(1234, "[Reproducible(Seed = 1234)]"); + +// Or drop the argument: the default snippet names Any.Reproducibly(seed, ...). +using IDisposable scope = Any.UseSeed(1234); +``` + +Pass the **code** itself — an attribute with its seed argument, a runner setting — not a sentence about it. It is quoted verbatim into the failure message. + +## What it does not flag + +* A non-constant snippet. +* A test asserting the rejection, where the call is the whole body of a lambda argument. `JustDummies.PropertyTests` asserts exactly this guard. + +--- + +[← All analyzer rules](README.md) diff --git a/doc/handwritten/for-users/analyzers/JD021.fr.md b/doc/handwritten/for-users/analyzers/JD021.fr.md new file mode 100644 index 00000000..7e0de51f --- /dev/null +++ b/doc/handwritten/for-users/analyzers/JD021.fr.md @@ -0,0 +1,41 @@ +# JD021 : BlankReplaySnippet + +🌍 **Langues :** +🇫🇷 Français (ce fichier) | 🇬🇧 [English](./JD021.en.md) + +| | | +|---|---| +| **Catégorie** | Reproductibilité (`JustDummies.Reproducibility`) | +| **Sévérité** | 🟠 Avertissement | +| **Activée par défaut** | Oui | + +`Any.UseSeed(int, string)` fournit le **snippet de rejeu** — le code qu'un lecteur copie pour rejouer l'exécution — que les conseils d'échec de génération citent tels quels. Un snippet vide est rejeté à l'exécution. + +Ce qui justifie la vérification à la compilation, c'est *où* cette levée atterrit. Cette surcharge existe pour un adaptateur de framework de test, qui ouvre la portée depuis un hook s'exécutant avant chaque test. Un snippet vide fait donc échouer toute la suite comme erreur d'infrastructure, et non une seule assertion — une façon disproportionnellement coûteuse d'apprendre une faute de frappe que le compilateur voit déjà. + +## Non conforme + +```csharp +using IDisposable scope = Any.UseSeed(1234, ""); // JD021 +using IDisposable scope = Any.UseSeed(1234, " "); // JD021 +``` + +## Conforme + +```csharp +using IDisposable scope = Any.UseSeed(1234, "[Reproducible(Seed = 1234)]"); + +// Ou supprimez l'argument : le snippet par défaut nomme Any.Reproducibly(seed, ...). +using IDisposable scope = Any.UseSeed(1234); +``` + +Passez le **code** lui-même — un attribut avec son argument de graine, un réglage de lanceur — et non une phrase à son sujet. Il est cité tel quel dans le message d'échec. + +## Ce qui n'est pas signalé + +* Un snippet non constant. +* Un test qui vérifie le rejet, où l'appel constitue tout le corps d'une lambda passée en argument. `JustDummies.PropertyTests` vérifie exactement cette garde. + +--- + +[← Toutes les règles d'analyse](README.fr.md) diff --git a/doc/handwritten/for-users/analyzers/JD022.en.md b/doc/handwritten/for-users/analyzers/JD022.en.md new file mode 100644 index 00000000..12ef652f --- /dev/null +++ b/doc/handwritten/for-users/analyzers/JD022.en.md @@ -0,0 +1,49 @@ +# JD022: ParallelDrawWithoutPerItemSeed + +🌍 **Languages:** +🇬🇧 English (this file) | 🇫🇷 [Français](./JD022.fr.md) + +| | | +|---|---| +| **Category** | Reproducibility (`JustDummies.Reproducibility`) | +| **Severity** | 🔵 Info | +| **Enabled by default** | Yes | + +The ambient seed scope flows with the execution context, so a scope opened *around* a parallel loop reaches every worker — and their draws interleave. Neither the sequence nor the multiset is stable across runs, so the run replays nothing even though a seed was pinned. + +This is the shape the library's own documentation names, and the fix it prescribes: a scope opened **inside** the loop body gives each unit of work its own sequence, and the whole run replays. + +## Noncompliant + +```csharp +Parallel.For(0, 64, index => { + sut.Handle(Any.String().NonEmpty().Generate()); // JD022: one shared sequence, interleaved +}); +``` + +## Compliant + +```csharp +const int runSeed = 20240501; // recorded by hand: keep it to replay, change it to explore + +Parallel.For(0, 64, index => { + // a distinct, deterministic sub-seed per work item, floor-safe on netstandard2.0 + using (Any.UseSeed(unchecked(runSeed * 397 ^ index))) { + sut.Handle(Any.String().NonEmpty().Generate()); + } +}); +``` + +## No code fix + +The repair needs a run seed the developer must **choose and record**, plus a per-item derivation from the loop variable. An analyzer cannot invent a seed someone intends to keep — generating one would produce exactly the committed-replay-seed problem [JD019](JD019.en.md) describes. + +## What it does not flag + +* A body that already opens an `Any.UseSeed` scope. +* A draw from an isolated `Any.WithSeed(...)` context, or a generator reached through a local rather than written inline from `Any`. +* A per-item scope opened in a helper the body calls — a one-hop miss the rule deliberately accepts rather than attempt interprocedural analysis. + +--- + +[← All analyzer rules](README.md) diff --git a/doc/handwritten/for-users/analyzers/JD022.fr.md b/doc/handwritten/for-users/analyzers/JD022.fr.md new file mode 100644 index 00000000..c8a24ade --- /dev/null +++ b/doc/handwritten/for-users/analyzers/JD022.fr.md @@ -0,0 +1,49 @@ +# JD022 : ParallelDrawWithoutPerItemSeed + +🌍 **Langues :** +🇫🇷 Français (ce fichier) | 🇬🇧 [English](./JD022.en.md) + +| | | +|---|---| +| **Catégorie** | Reproductibilité (`JustDummies.Reproducibility`) | +| **Sévérité** | 🔵 Info | +| **Activée par défaut** | Oui | + +La portée de graine ambiante suit le contexte d'exécution : une portée ouverte *autour* d'une boucle parallèle atteint donc chaque worker — et leurs tirages s'entrelacent. Ni la séquence ni le multiensemble ne sont stables d'une exécution à l'autre : l'exécution ne rejoue rien alors qu'une graine a été épinglée. + +C'est la forme que la documentation de la bibliothèque nomme elle-même, avec le remède qu'elle prescrit : une portée ouverte **dans** le corps de boucle donne à chaque unité de travail sa propre séquence, et toute l'exécution rejoue. + +## Non conforme + +```csharp +Parallel.For(0, 64, index => { + sut.Handle(Any.String().NonEmpty().Generate()); // JD022 : une séquence partagée, entrelacée +}); +``` + +## Conforme + +```csharp +const int runSeed = 20240501; // noté à la main : gardez-le pour rejouer, changez-le pour explorer + +Parallel.For(0, 64, index => { + // une sous-graine distincte et déterministe par unité de travail, compatible netstandard2.0 + using (Any.UseSeed(unchecked(runSeed * 397 ^ index))) { + sut.Handle(Any.String().NonEmpty().Generate()); + } +}); +``` + +## Pas de correction automatique + +La réparation exige une graine d'exécution que le développeur doit **choisir et consigner**, plus une dérivation par item à partir de la variable de boucle. Un analyseur ne peut pas inventer une graine que quelqu'un compte conserver — en générer une produirait exactement le problème de graine committée que décrit [JD019](JD019.fr.md). + +## Ce qui n'est pas signalé + +* Un corps qui ouvre déjà une portée `Any.UseSeed`. +* Un tirage depuis un contexte isolé `Any.WithSeed(...)`, ou un générateur atteint par une variable locale plutôt qu'écrit en ligne depuis `Any`. +* Une portée par item ouverte dans un utilitaire appelé par le corps — un manque à un saut que la règle accepte délibérément plutôt que de tenter une analyse interprocédurale. + +--- + +[← Toutes les règles d'analyse](README.fr.md) diff --git a/doc/handwritten/for-users/analyzers/README.fr.md b/doc/handwritten/for-users/analyzers/README.fr.md index 4ae0e6a6..db0e8986 100644 --- a/doc/handwritten/for-users/analyzers/README.fr.md +++ b/doc/handwritten/for-users/analyzers/README.fr.md @@ -63,6 +63,11 @@ Ces règles sont incluses dans le package **`JustDummies`** (pas FirstClassError | [JD008 ArbitraryValueInTheoryData](JD008.fr.md) | 🟠 Avertissement | on | Le fournisseur de données d'une théorie tire une valeur à la découverte, avant tout épinglage ; tous les cas partagent cette unique valeur. | | [JD009 DrawInStaticInitializer](JD009.fr.md) | 🟠 Avertissement | on | Un initialiseur statique tire une seule fois pour toute la suite, sous le premier test exécuté, rendant les tests dépendants de l'ordre et rejouables depuis aucune graine. | | [JD010 ReproducibleOnNonTestMethod](JD010.fr.md) | 🟠 Avertissement | on | `[Reproducible]` sur une méthode qu'xUnit ne traite jamais comme un test ; il n'épingle rien, et ressemble exactement à la forme active. | +| [JD018 NestedReproducibilityScope](JD018.fr.md) | 🟠 Avertissement | on | Une portée de reproductibilité imbriquée dans une autre ; l'interne tire une graine neuve, donc la graine rapportée par l'externe ne rejoue rien. | +| [JD021 BlankReplaySnippet](JD021.fr.md) | 🟠 Avertissement | on | `Any.UseSeed` reçoit un snippet de rejeu vide, que la garde rejette — depuis un hook d'adaptateur, faisant échouer toute la suite. | +| [JD019 CommittedReplaySeed](JD019.fr.md) | 🔵 Info | opt-in | Une graine de rejeu constante est épinglée dans du code committé : le test cesse de varier d'une exécution à l'autre. | +| [JD020 SharedStaticAnyContext](JD020.fr.md) | 🔵 Info | on | Un `AnyContext` tenu dans un champ statique ; les tirages entrelacés ne rendent stables ni la séquence ni le multiensemble. | +| [JD022 ParallelDrawWithoutPerItemSeed](JD022.fr.md) | 🔵 Info | on | Une unité de travail parallèle tire sans sa propre portée de graine : les tirages s'entrelacent et l'exécution ne rejoue rien. | ## JustDummies — Usage diff --git a/doc/handwritten/for-users/analyzers/README.md b/doc/handwritten/for-users/analyzers/README.md index 84752612..6a65ab75 100644 --- a/doc/handwritten/for-users/analyzers/README.md +++ b/doc/handwritten/for-users/analyzers/README.md @@ -63,6 +63,11 @@ These rules ship in the **`JustDummies`** package (not FirstClassErrors) and kee | [JD008 ArbitraryValueInTheoryData](JD008.en.md) | 🟠 Warning | on | A theory's data provider draws a value at discovery, before any seed is pinned; every case shares the one value. | | [JD009 DrawInStaticInitializer](JD009.en.md) | 🟠 Warning | on | A static initializer draws once for the whole suite, under whichever test ran first, making the tests order-dependent and replayable from no seed. | | [JD010 ReproducibleOnNonTestMethod](JD010.en.md) | 🟠 Warning | on | [Reproducible] on a method xUnit never treats as a test; it pins nothing, and looks exactly like the working form. | +| [JD018 NestedReproducibilityScope](JD018.en.md) | 🟠 Warning | on | A reproducibility scope nested inside another; the inner one draws a fresh seed, so the outer's reported seed replays nothing. | +| [JD021 BlankReplaySnippet](JD021.en.md) | 🟠 Warning | on | Any.UseSeed is given a blank replay snippet, which the guard rejects — from an adapter hook, failing the whole suite. | +| [JD019 CommittedReplaySeed](JD019.en.md) | 🔵 Info | opt-in | A constant replay seed is pinned in committed code, so the test stops varying between runs. | +| [JD020 SharedStaticAnyContext](JD020.en.md) | 🔵 Info | on | An AnyContext held in a static field; interleaved draws make neither the sequence nor the multiset stable. | +| [JD022 ParallelDrawWithoutPerItemSeed](JD022.en.md) | 🔵 Info | on | A parallel work item draws without its own seed scope, so the draws interleave and the run replays nothing. | ## JustDummies — Usage