diff --git a/docs/counter.md b/docs/counter.md index 303be31fc..107053410 100644 --- a/docs/counter.md +++ b/docs/counter.md @@ -7,7 +7,7 @@ To change this file edit the source file and then run MarkdownSnippets. # Counter -The `Counter` class provides methods to attempt conversion of a `CharSpan` value into a string representation of supported types. +The `Counter` class provides methods to attempt conversion of a `ReadOnlySpan` value into a string representation of supported types. Supported types include @@ -18,7 +18,7 @@ It handles matching equivalent values and assigning a number to each match. It i ## TryConvert -Takes a CharSpan and attempts to parse it to one of the supported types, then return the tokenized scrubbed value for that value. +Takes a `ReadOnlySpan` and attempts to parse it to one of the supported types, then return the tokenized scrubbed value for that value. One example usage is inside a custom scrubber: diff --git a/docs/dates.md b/docs/dates.md index a3b17e3c5..cf7cdb72b 100644 --- a/docs/dates.md +++ b/docs/dates.md @@ -308,7 +308,7 @@ public Task InferredNamedDateFluent() .AddNamedDate(namedDate); } ``` -snippet source | anchor +snippet source | anchor Result: diff --git a/docs/guids.md b/docs/guids.md index a99d32809..2a3d9e87b 100644 --- a/docs/guids.md +++ b/docs/guids.md @@ -23,7 +23,7 @@ var target = new GuidTarget await Verify(target); ``` -snippet source | anchor +snippet source | anchor Results in the following: @@ -161,7 +161,7 @@ public Task NamedGuidInstance() settings); } ``` -snippet source | anchor +snippet source | anchor @@ -182,7 +182,7 @@ public Task NamedGuidFluent() .AddNamedGuid(guid, "instanceNamed"); } ``` -snippet source | anchor +snippet source | anchor @@ -217,7 +217,7 @@ public Task InferredNamedGuidFluent() .AddNamedGuid(namedGuid); } ``` -snippet source | anchor +snippet source | anchor Result: diff --git a/docs/mdsource/counter.source.md b/docs/mdsource/counter.source.md index 06854a0af..2f7f3012b 100644 --- a/docs/mdsource/counter.source.md +++ b/docs/mdsource/counter.source.md @@ -1,6 +1,6 @@ # Counter -The `Counter` class provides methods to attempt conversion of a `CharSpan` value into a string representation of supported types. +The `Counter` class provides methods to attempt conversion of a `ReadOnlySpan` value into a string representation of supported types. Supported types include @@ -11,7 +11,7 @@ It handles matching equivalent values and assigning a number to each match. It i ## TryConvert -Takes a CharSpan and attempts to parse it to one of the supported types, then return the tokenized scrubbed value for that value. +Takes a `ReadOnlySpan` and attempts to parse it to one of the supported types, then return the tokenized scrubbed value for that value. One example usage is inside a custom scrubber: diff --git a/docs/mdsource/scrubbers.source.md b/docs/mdsource/scrubbers.source.md index d14f95c39..a868bc904 100644 --- a/docs/mdsource/scrubbers.source.md +++ b/docs/mdsource/scrubbers.source.md @@ -4,9 +4,43 @@ Scrubbers run on the final string before doing the verification action. Multiple scrubbers [can be defined at multiple levels](#Scrubber-levels). -By default scrubber are executed in reverse order. So the most recent added method scrubber through to earliest added global scrubber. All scrubber APIs support a `ScrubberLocation location`. To execute a scrubber last use `ScrubberLocation.Last`. +Scrubbing is performed by two mechanisms: -Scrubbers can be added multiple times to have them execute multiple times. This can be helpful when compounding multiple scrubbers together. + * **The scrub engine**: a span based engine that performs all built-in scrubbing (`ScrubLinesContaining`, `ScrubInlineGuids`, `ScrubMachineName`, etc). + * **Legacy scrubbers**: `AddScrubber(Action)` overloads. These run after the engine, and only when at least one is registered. + + +## The scrub engine + +Each engine operation is available as a `Scrub*` method at any [level](#Scrubber-levels): + + * `ScrubReplace(find, replacement)`: replace every occurrence of a string. Supports an ordinal `StringComparison` (`Ordinal` or `OrdinalIgnoreCase`) and an optional word boundary requirement. A multi-pair overload replaces the longest matching find at any position. + * `ScrubWindow(minLength, maxLength, matcher)`: slide a window over the text; the matcher returns a replacement or null. Used by the inline guid and date scrubbers. + * `ScrubMatch(matcher, minLength, maxLength)`: custom search logic. The matcher locates the next match within a segment. + * `ScrubLinesContaining(...)`, `ScrubLines(...)`, `ScrubLinesWithReplace(...)`, `ScrubEmptyLines()`: line scoped scrubbing. + +snippet: ScrubEngine + +`ScrubLines` and `ScrubLinesWithReplace` also accept span based delegates (`LineMatch` / `LineReplace`) that avoid allocating a string per line. Use an explicitly typed lambda parameter to select them, e.g. `ScrubLines((ReadOnlySpan line) => ...)`; untyped lambdas bind the string overloads. + +Engine semantics: + + * **Quarantine**: text produced by a replacement is never re-examined by other engine scrubbers. Legacy scrubbers run afterwards and can still modify it. + * **Ordering** is engine determined, not registration determined: line removals run first, then line transforms (registration order), then inline scrubbers (unknown max length first, then longest max length first, ties broken by level then registration order). Directory replacements always run last, so scrubbers always see raw paths. + * **Length skip**: text shorter than a scrubber's minimum match length is never scanned by that scrubber. + * **Single line rule**: a match may never contain a line break. + * Text is newline normalized (`\r\n` and `\r` become `\n`) before scrubbers run. + +`ScrubberLocation` on the built-in scrub methods is obsolete and ignored; it still applies to legacy `AddScrubber` overloads (default `First` executes in reverse registration order, `Last` in registration order). + + +## Legacy scrubbers + +Instead of being executed by the engine, `AddScrubber(Action)` mutates the full text directly. Overloads also accept a `Counter`, and the context dictionary: + +snippet: AddScrubber + +Since these run after every engine scrubber, they can also modify text that the engine has already replaced. ## Available Scrubbers @@ -147,6 +181,15 @@ snippet: ScrubbersSampleTUnit snippet: Verify.XunitV3.Tests/Scrubbers/ScrubbersSample.Lines.verified.txt +## Extension specific scrubbers + +Scrubbers can be scoped to verified files with a matching extension by passing the extension as the first argument. The extension is specified without a leading dot: + +snippet: ScrubEngineExtension + +A scrubber registered this way runs only for verified files with that extension, while a scrubber registered without an extension runs for all of them. Extension scoping is available at every [level](#Scrubber-levels), and for the legacy `AddScrubber(Action)` overloads. + + ## Scrubber levels Scrubbers can be defined at three levels: diff --git a/docs/members-throw.md b/docs/members-throw.md index 847c1f8d6..fea107e64 100644 --- a/docs/members-throw.md +++ b/docs/members-throw.md @@ -35,7 +35,7 @@ public Task CustomExceptionPropFluent() .IgnoreMembersThatThrow(); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -45,7 +45,7 @@ Or globally: ```cs VerifierSettings.IgnoreMembersThatThrow(); ``` -snippet source | anchor +snippet source | anchor Result: @@ -82,7 +82,7 @@ public Task ExceptionMessagePropFluent() .IgnoreMembersThatThrow(_ => _.Message == "Ignore"); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -92,7 +92,7 @@ Or globally: ```cs VerifierSettings.IgnoreMembersThatThrow(_ => _.Message == "Ignore"); ``` -snippet source | anchor +snippet source | anchor Result: diff --git a/docs/named-tuples.md b/docs/named-tuples.md index 2e3654434..cbc2a40f2 100644 --- a/docs/named-tuples.md +++ b/docs/named-tuples.md @@ -19,7 +19,7 @@ Given a method that returns a named tuple: static (bool Member1, string Member2, string Member3) MethodWithNamedTuple() => (true, "A", "B"); ``` -snippet source | anchor +snippet source | anchor Can be verified: @@ -29,7 +29,7 @@ Can be verified: ```cs await VerifyTuple(() => MethodWithNamedTuple()); ``` -snippet source | anchor +snippet source | anchor Resulting in: diff --git a/docs/obsolete-members.md b/docs/obsolete-members.md index bd3f9d2bc..9a8412f1a 100644 --- a/docs/obsolete-members.md +++ b/docs/obsolete-members.md @@ -31,7 +31,7 @@ public Task WithObsoleteProp() return Verify(target); } ``` -snippet source | anchor +snippet source | anchor Result: @@ -79,7 +79,7 @@ public Task WithObsoletePropIncludedFluent() .IncludeObsoletes(); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -89,7 +89,7 @@ Or globally: ```cs VerifierSettings.IncludeObsoletes(); ``` -snippet source | anchor +snippet source | anchor Result: diff --git a/docs/scrubbers.md b/docs/scrubbers.md index 9ea9c2573..ccbd5d869 100644 --- a/docs/scrubbers.md +++ b/docs/scrubbers.md @@ -11,9 +11,68 @@ Scrubbers run on the final string before doing the verification action. Multiple scrubbers [can be defined at multiple levels](#Scrubber-levels). -By default scrubber are executed in reverse order. So the most recent added method scrubber through to earliest added global scrubber. All scrubber APIs support a `ScrubberLocation location`. To execute a scrubber last use `ScrubberLocation.Last`. +Scrubbing is performed by two mechanisms: -Scrubbers can be added multiple times to have them execute multiple times. This can be helpful when compounding multiple scrubbers together. + * **The scrub engine**: a span based engine that performs all built-in scrubbing (`ScrubLinesContaining`, `ScrubInlineGuids`, `ScrubMachineName`, etc). + * **Legacy scrubbers**: `AddScrubber(Action)` overloads. These run after the engine, and only when at least one is registered. + + +## The scrub engine + +Each engine operation is available as a `Scrub*` method at any [level](#Scrubber-levels): + + * `ScrubReplace(find, replacement)`: replace every occurrence of a string. Supports an ordinal `StringComparison` (`Ordinal` or `OrdinalIgnoreCase`) and an optional word boundary requirement. A multi-pair overload replaces the longest matching find at any position. + * `ScrubWindow(minLength, maxLength, matcher)`: slide a window over the text; the matcher returns a replacement or null. Used by the inline guid and date scrubbers. + * `ScrubMatch(matcher, minLength, maxLength)`: custom search logic. The matcher locates the next match within a segment. + * `ScrubLinesContaining(...)`, `ScrubLines(...)`, `ScrubLinesWithReplace(...)`, `ScrubEmptyLines()`: line scoped scrubbing. + + + +```cs +verifySettings.ScrubReplace("abc", "xyz"); + +verifySettings.ScrubWindow( + minLength: 3, + maxLength: 10, + matcher: (window, _, _) => + { + if (window.StartsWith("id-")) + { + return "{Id}"; + } + + return null; + }); +``` +snippet source | anchor + + +`ScrubLines` and `ScrubLinesWithReplace` also accept span based delegates (`LineMatch` / `LineReplace`) that avoid allocating a string per line. Use an explicitly typed lambda parameter to select them, e.g. `ScrubLines((ReadOnlySpan line) => ...)`; untyped lambdas bind the string overloads. + +Engine semantics: + + * **Quarantine**: text produced by a replacement is never re-examined by other engine scrubbers. Legacy scrubbers run afterwards and can still modify it. + * **Ordering** is engine determined, not registration determined: line removals run first, then line transforms (registration order), then inline scrubbers (unknown max length first, then longest max length first, ties broken by level then registration order). Directory replacements always run last, so scrubbers always see raw paths. + * **Length skip**: text shorter than a scrubber's minimum match length is never scanned by that scrubber. + * **Single line rule**: a match may never contain a line break. + * Text is newline normalized (`\r\n` and `\r` become `\n`) before scrubbers run. + +`ScrubberLocation` on the built-in scrub methods is obsolete and ignored; it still applies to legacy `AddScrubber` overloads (default `First` executes in reverse registration order, `Last` in registration order). + + +## Legacy scrubbers + +Instead of being executed by the engine, `AddScrubber(Action)` mutates the full text directly. Overloads also accept a `Counter`, and the context dictionary: + + + +```cs +verifySettings.AddScrubber(_ => _.Remove(0, 100)); +``` +snippet source | anchor + + +Since these run after every engine scrubber, they can also modify text that the engine has already replaced. ## Available Scrubbers @@ -59,7 +118,7 @@ For example remove lines containing `text`: ```cs verifySettings.ScrubLines(line => line.Contains("text")); ``` -snippet source | anchor +snippet source | anchor @@ -74,7 +133,7 @@ For example remove lines containing `text1` or `text2` ```cs verifySettings.ScrubLinesContaining("text1", "text2"); ``` -snippet source | anchor +snippet source | anchor Case insensitive by default (`StringComparison.OrdinalIgnoreCase`). @@ -86,7 +145,7 @@ Case insensitive by default (`StringComparison.OrdinalIgnoreCase`). ```cs verifySettings.ScrubLinesContaining(StringComparison.Ordinal, "text1", "text2"); ``` -snippet source | anchor +snippet source | anchor @@ -101,7 +160,7 @@ For example converts lines to upper case: ```cs verifySettings.ScrubLinesWithReplace(line => line.ToUpper()); ``` -snippet source | anchor +snippet source | anchor @@ -114,7 +173,7 @@ Replaces `Environment.MachineName` with `TheMachineName`. ```cs verifySettings.ScrubMachineName(); ``` -snippet source | anchor +snippet source | anchor @@ -127,7 +186,7 @@ Replaces `Environment.UserName` with `TheUserName`. ```cs verifySettings.ScrubUserName(); ``` -snippet source | anchor +snippet source | anchor @@ -720,6 +779,21 @@ LineI +## Extension specific scrubbers + +Scrubbers can be scoped to verified files with a matching extension by passing the extension as the first argument. The extension is specified without a leading dot: + + + +```cs +verifySettings.ScrubReplace("json", "abc", "xyz"); +``` +snippet source | anchor + + +A scrubber registered this way runs only for verified files with that extension, while a scrubber registered without an extension runs for all of them. Extension scoping is available at every [level](#Scrubber-levels), and for the legacy `AddScrubber(Action)` overloads. + + ## Scrubber levels Scrubbers can be defined at three levels: diff --git a/docs/serializer-settings.md b/docs/serializer-settings.md index 5b074b7c4..f7cda568a 100644 --- a/docs/serializer-settings.md +++ b/docs/serializer-settings.md @@ -495,7 +495,7 @@ public Task ScopedSerializerFluent() .AddExtraSettings(_ => _.TypeNameHandling = TypeNameHandling.All); } ``` -snippet source | anchor +snippet source | anchor Result: @@ -623,7 +623,7 @@ public Task IgnoreTypeFluent() .IgnoreMembersWithType(); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -633,7 +633,7 @@ Or globally: ```cs VerifierSettings.IgnoreMembersWithType(); ``` -snippet source | anchor +snippet source | anchor Result: @@ -770,7 +770,7 @@ public Task ScrubTypeFluent() .ScrubMembersWithType(); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -780,7 +780,7 @@ Or globally: ```cs VerifierSettings.ScrubMembersWithType(); ``` -snippet source | anchor +snippet source | anchor Result: @@ -859,7 +859,7 @@ public Task AddIgnoreInstanceFluent() .IgnoreInstance(_ => _.Property == "Ignore"); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -869,7 +869,7 @@ Or globally: ```cs VerifierSettings.IgnoreInstance(_ => _.Property == "Ignore"); ``` -snippet source | anchor +snippet source | anchor Result: @@ -931,7 +931,7 @@ public Task AddScrubInstanceFluent() .ScrubInstance(_ => _.Property == "Ignore"); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -941,7 +941,7 @@ Or globally: ```cs VerifierSettings.ScrubInstance(_ => _.Property == "Ignore"); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1004,7 +1004,7 @@ public Task IgnoreMemberByExpressionFluent() _ => _.PropertyThatThrows); } ``` -snippet source | anchor +snippet source | anchor Or globally @@ -1019,7 +1019,7 @@ VerifierSettings.IgnoreMembers( _ => _.GetOnlyProperty, _ => _.PropertyThatThrows); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1079,7 +1079,7 @@ public Task ScrubMemberByExpressionFluent() _ => _.PropertyThatThrows); } ``` -snippet source | anchor +snippet source | anchor Or globally @@ -1094,7 +1094,7 @@ VerifierSettings.ScrubMembers( _ => _.GetOnlyProperty, _ => _.PropertyThatThrows); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1173,7 +1173,7 @@ public Task IgnoreMemberByNameFluent() .IgnoreMember(_ => _.PropertyThatThrows); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -1193,7 +1193,7 @@ VerifierSettings.IgnoreMember("Field"); // For a specific type with expression VerifierSettings.IgnoreMember(_ => _.PropertyThatThrows); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1268,7 +1268,7 @@ public Task ScrubMemberByNameFluent() .ScrubMember(_ => _.PropertyThatThrows); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -1288,7 +1288,7 @@ VerifierSettings.ScrubMember("Field"); // For a specific type with expression VerifierSettings.ScrubMember(_ => _.PropertyThatThrows); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1381,7 +1381,7 @@ public Task IgnoreDictionaryByPredicate() return Verify(target, settings); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -1393,7 +1393,7 @@ VerifierSettings.IgnoreMembers( _=>_.DeclaringType == typeof(TargetClass) && _.Name == "Proprty"); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1482,7 +1482,7 @@ public Task ScrubDictionaryByPredicate() return Verify(target, settings); } ``` -snippet source | anchor +snippet source | anchor Or globally: @@ -1494,7 +1494,7 @@ VerifierSettings.ScrubMembers( _=>_.DeclaringType == typeof(TargetClass) && _.Name == "Proprty"); ``` -snippet source | anchor +snippet source | anchor Result: @@ -1548,7 +1548,7 @@ public Task MemberConverterByExpression() return Verify(input); } ``` -snippet source | anchor +snippet source | anchor diff --git a/src/ApplyScrubbersTests/ApplyScrubbersTests.csproj b/src/ApplyScrubbersTests/ApplyScrubbersTests.csproj new file mode 100644 index 000000000..23383b3f2 --- /dev/null +++ b/src/ApplyScrubbersTests/ApplyScrubbersTests.csproj @@ -0,0 +1,15 @@ + + + net11.0 + false + Exe + + + + + + + + + + diff --git a/src/ApplyScrubbersTests/ComparisonTests.cs b/src/ApplyScrubbersTests/ComparisonTests.cs new file mode 100644 index 000000000..2b62dd347 --- /dev/null +++ b/src/ApplyScrubbersTests/ComparisonTests.cs @@ -0,0 +1,44 @@ +// The engine splices out exactly Find.Length chars and skips text shorter than +// Find. Both only hold for ordinal comparisons. +public class ComparisonTests +{ + [Fact] + public void LinguisticComparisonRejectedForReplace() + { + var exception = Assert.Throws( + () => Scrubber.Replace("ab", "X", StringComparison.InvariantCulture)); + Assert.Contains("Ordinal", exception.Message); + + Assert.Throws( + () => Scrubber.Replace("ab", "X", StringComparison.CurrentCulture)); + Assert.Throws( + () => Scrubber.Replace("ab", "X", StringComparison.CurrentCultureIgnoreCase)); + Assert.Throws( + () => Scrubber.Replace(StringComparison.InvariantCultureIgnoreCase, false, ("ab", "X"))); + } + + [Fact] + public void OrdinalComparisonsAccepted() + { + Assert.Equal("X!", EngineRunner.Run("ab!", Scrubber.Replace("ab", "X"))); + Assert.Equal("X!", EngineRunner.Run("AB!", Scrubber.Replace("ab", "X", StringComparison.OrdinalIgnoreCase))); + } + + [Fact] + public void LinguisticNeedleDropsShorterLine() + { + // Soft hyphen is ignorable, so the 6 char line matches the 7 char needle. + // The line may not be skipped for being shorter than the needle. + var scrubber = Scrubber.RemoveLinesContaining(StringComparison.InvariantCultureIgnoreCase, "sec­ret"); + var result = EngineRunner.Run("keep\nsecret\nkeep2", scrubber); + Assert.Equal("keep\nkeep2", result); + } + + [Fact] + public void OrdinalNeedleDropsMatchingLine() + { + var scrubber = Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "secret"); + var result = EngineRunner.Run("keep\nsecret\nkeep2", scrubber); + Assert.Equal("keep\nkeep2", result); + } +} diff --git a/src/ApplyScrubbersTests/CultureTests.cs b/src/ApplyScrubbersTests/CultureTests.cs new file mode 100644 index 000000000..4c6b73198 --- /dev/null +++ b/src/ApplyScrubbersTests/CultureTests.cs @@ -0,0 +1,101 @@ +// Inline date scrubbers registered without an explicit culture must follow the +// culture in effect when the scrub runs, not the one in effect at registration. +public class CultureTests +{ + static readonly CultureInfo enUs = new("en-US"); + static readonly CultureInfo deDe = new("de-DE"); + + static string RunInCulture(CultureInfo culture, string input, params Scrubber[] scrubbers) + { + var original = CultureInfo.CurrentCulture; + try + { + CultureInfo.CurrentCulture = culture; + return EngineRunner.Run(input, scrubbers); + } + finally + { + CultureInfo.CurrentCulture = original; + } + } + + [Fact] + public void ScrubTimeCultureIsUsedWhenNoneSupplied() + { + // Registered under en-US, as a module initializer would be + var original = CultureInfo.CurrentCulture; + Scrubber[] scrubbers; + try + { + CultureInfo.CurrentCulture = enUs; + scrubbers = DateMatchers.DateTimes("dd MMMM yyyy", null); + } + finally + { + CultureInfo.CurrentCulture = original; + } + + var german = new DateTime(2024, 12, 5).ToString("dd MMMM yyyy", deDe); + var american = new DateTime(2024, 12, 5).ToString("dd MMMM yyyy", enUs); + + Assert.Equal("[DateTime_1]", RunInCulture(deDe, $"[{german}]", scrubbers)); + Assert.Equal("[DateTime_1]", RunInCulture(enUs, $"[{american}]", scrubbers)); + } + + [Fact] + public void ExplicitCultureIsNotAffectedByScrubTimeCulture() + { + var scrubbers = DateMatchers.DateTimes("dd MMMM yyyy", enUs); + + var american = new DateTime(2024, 12, 5).ToString("dd MMMM yyyy", enUs); + var german = new DateTime(2024, 12, 5).ToString("dd MMMM yyyy", deDe); + + // The explicit culture wins even while another culture is current + Assert.Equal("[DateTime_1]", RunInCulture(deDe, $"[{american}]", scrubbers)); + Assert.Equal($"[{german}]", RunInCulture(deDe, $"[{german}]", scrubbers)); + } + + [Fact] + public void CultureBoundsFollowScrubTimeCulture() + { + // Month name lengths differ between cultures, so the window bounds have to + // be rebuilt per culture and not just the parse + var original = CultureInfo.CurrentCulture; + Scrubber[] scrubbers; + try + { + CultureInfo.CurrentCulture = enUs; + scrubbers = DateMatchers.DateTimes("MMMM d yyyy", null); + } + finally + { + CultureInfo.CurrentCulture = original; + } + + var french = new CultureInfo("fr-FR"); + var rendered = new DateTime(2024, 2, 1).ToString("MMMM d yyyy", french); + Assert.Equal("[DateTime_1]", RunInCulture(french, $"[{rendered}]", scrubbers)); + } + + [Fact] + public void RepeatedScrubsReuseTheCachedCultureScrubber() + { + var original = CultureInfo.CurrentCulture; + Scrubber[] scrubbers; + try + { + CultureInfo.CurrentCulture = enUs; + scrubbers = DateMatchers.DateTimes("dd MMMM yyyy", null); + } + finally + { + CultureInfo.CurrentCulture = original; + } + + var german = new DateTime(2024, 12, 5).ToString("dd MMMM yyyy", deDe); + for (var index = 0; index < 3; index++) + { + Assert.Equal("[DateTime_1]", RunInCulture(deDe, $"[{german}]", scrubbers)); + } + } +} diff --git a/src/ApplyScrubbersTests/DeletionSeamTests.cs b/src/ApplyScrubbersTests/DeletionSeamTests.cs new file mode 100644 index 000000000..34a816798 --- /dev/null +++ b/src/ApplyScrubbersTests/DeletionSeamTests.cs @@ -0,0 +1,71 @@ +// An empty replacement quarantines nothing, so the text on either side of it is +// adjacent document text and must stay matchable by later scrubbers. +public class DeletionSeamTests +{ + static DeletionSeamTests() => + EngineRunner.UseFakeDirectories(); + + [Fact] + public void LaterScrubberMatchesAcrossDeletion() + { + var result = EngineRunner.Run( + "a--b", + Scrubber.Replace("--", ""), + Scrubber.Replace("ab", "X")); + Assert.Equal("X", result); + } + + [Fact] + public void LongerFindMatchesAcrossDeletion() + { + var result = EngineRunner.Run( + "daREDACT-ta", + Scrubber.Replace("REDACT-", ""), + Scrubber.Replace("data", "X")); + Assert.Equal("X", result); + } + + [Fact] + public void MultipleDeletionsAllJoined() + { + // Equal max lengths, so these run in registration order and the deletion + // happens first (inline scrubbers otherwise order by descending max length) + var result = EngineRunner.Run( + "aXXbXXcXXd", + Scrubber.Replace("XX", ""), + Scrubber.Replace("bc", "Y")); + Assert.Equal("aYd", result); + } + + [Fact] + public void DeletionAfterLinePhaseStillJoins() + { + var result = EngineRunner.Run( + "drop\na--b\nkeep", + Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "drop"), + Scrubber.Replace("--", ""), + Scrubber.Replace("ab", "X")); + Assert.Equal("X\nkeep", result); + } + + [Fact] + public void DirectoryReplacementMatchesAcrossDeletion() + { + var result = EngineRunner.RunWithDirectoryReplacements( + "C:/Co[DEL]de/TheSolution/TheProject/file.txt", + Scrubber.Replace("[DEL]", "")); + Assert.Equal("{ProjectDirectory}file.txt", result); + } + + [Fact] + public void QuarantineSurvivesDeletion() + { + var result = EngineRunner.Run( + "a-b", + Scrubber.Replace("-", ""), + Scrubber.Replace("b", "a"), + Scrubber.Replace("aa", "BOOM")); + // The 'a' produced by the second scrubber is quarantined, so "aa" must not form + Assert.Equal("aa", result); + } +} diff --git a/src/ApplyScrubbersTests/DigitAnchorTests.cs b/src/ApplyScrubbersTests/DigitAnchorTests.cs new file mode 100644 index 000000000..ab3b6dab5 --- /dev/null +++ b/src/ApplyScrubbersTests/DigitAnchorTests.cs @@ -0,0 +1,58 @@ +// The digit anchor lets the engine jump between digit positions instead of +// probing every one. It may only be used when the format really does render an +// ASCII digit first, otherwise candidate windows are never probed. +public class DigitAnchorTests +{ + [Fact] + public void LeadingOptionalFractionIsScrubbed() + { + // Upper case F renders nothing when the fraction is zero, so the value + // starts with the literal space and a digit anchor would never probe it + var rendered = new DateTime(2020, 1, 1, 0, 30, 0).ToString("FF mm", CultureInfo.InvariantCulture); + Assert.Equal(" 30", rendered); + + using var counter = Counter.Start(); + var scrubbers = DateMatchers.DateTimes("FF mm", CultureInfo.InvariantCulture); + Assert.Equal("[DateTime_1]", EngineRunner.Run($"[{rendered}]", counter, scrubbers)); + } + + [Fact] + public void NonZeroFractionIsScrubbed() + { + var date = new DateTime(2020, 1, 1, 0, 30, 0).AddMilliseconds(250); + var rendered = date.ToString("FF mm", CultureInfo.InvariantCulture); + Assert.Equal("25 30", rendered); + + using var counter = Counter.Start(); + var scrubbers = DateMatchers.DateTimes("FF mm", CultureInfo.InvariantCulture); + Assert.Equal("[DateTime_1]", EngineRunner.Run($"[{rendered}]", counter, scrubbers)); + } + + [Fact] + public void DigitLeadingFormatIsScrubbed() + { + using var counter = Counter.Start(); + var scrubbers = DateMatchers.DateTimes("yyyy-MM-dd", CultureInfo.InvariantCulture); + Assert.Equal("[DateTime_1]", EngineRunner.Run("[2024-01-02]", counter, scrubbers)); + } + + [Fact] + public void NameLeadingFormatIsScrubbed() + { + using var counter = Counter.Start(); + var scrubbers = DateMatchers.DateTimes("MMMM d yyyy", CultureInfo.InvariantCulture); + Assert.Equal("[DateTime_1]", EngineRunner.Run("[January 2 2024]", counter, scrubbers)); + } + + [Fact] + public void LowerCaseFractionIsScrubbed() + { + // Lower case f always renders digits, so the anchor stays available + var rendered = new DateTime(2020, 1, 1, 0, 30, 0).ToString("ff mm", CultureInfo.InvariantCulture); + Assert.Equal("00 30", rendered); + + using var counter = Counter.Start(); + var scrubbers = DateMatchers.DateTimes("ff mm", CultureInfo.InvariantCulture); + Assert.Equal("[DateTime_1]", EngineRunner.Run($"[{rendered}]", counter, scrubbers)); + } +} diff --git a/src/ApplyScrubbersTests/DirectoryGateTests.cs b/src/ApplyScrubbersTests/DirectoryGateTests.cs new file mode 100644 index 000000000..73424fcff --- /dev/null +++ b/src/ApplyScrubbersTests/DirectoryGateTests.cs @@ -0,0 +1,30 @@ +// Path replacement is pinned last, so it must run against the scrubbed document +// rather than being gated on the length of the pre-scrub source. +public class DirectoryGateTests +{ + static DirectoryGateTests() => + EngineRunner.UseFakeDirectories(); + + [Fact] + public void TransformExpandedTextIsPathScrubbed() + { + var result = EngineRunner.RunWithDirectoryReplacements( + "here", + Scrubber.ReplaceLines((string line) => line == "here" ? "C:/Code/TheSolution/TheProject/file.txt" : line)); + Assert.Equal("{ProjectDirectory}file.txt", result); + } + + [Fact] + public void ReplacementTextStaysQuarantined() + { + var result = EngineRunner.RunWithDirectoryReplacements( + "x", + Scrubber.Replace("x", "C:/Code/TheSolution/TheProject/f.txt")); + // Replacement text is quarantined, so it is deliberately not path scrubbed + Assert.Equal("C:/Code/TheSolution/TheProject/f.txt", result); + } + + [Fact] + public void ShortSourceWithNoScrubbersIsUnchanged() => + Assert.Equal("here", EngineRunner.RunWithDirectoryReplacements("here")); +} diff --git a/src/ApplyScrubbersTests/DirectorySnapshotTests.cs b/src/ApplyScrubbersTests/DirectorySnapshotTests.cs new file mode 100644 index 000000000..5913ec5dc --- /dev/null +++ b/src/ApplyScrubbersTests/DirectorySnapshotTests.cs @@ -0,0 +1,44 @@ +// The anchors and the shortest find are derived from the pairs, so UseAssembly has +// to keep them in step. A scan that used anchors not covering a pair's first char +// would never land on that path. +public class DirectorySnapshotTests +{ + static DirectorySnapshotTests() => + EngineRunner.UseFakeDirectories(); + + [Fact] + public void AnchorsCoverEveryPair() + { + var snapshot = DirectoryReplacements.Current; + Assert.NotEmpty(snapshot.Pairs); + + foreach (var pair in snapshot.Pairs) + { + Assert.Contains(pair.Find[0], snapshot.Anchors); + } + } + + [Fact] + public void ShortestFindLengthMatchesThePairs() + { + var snapshot = DirectoryReplacements.Current; + var shortest = int.MaxValue; + foreach (var pair in snapshot.Pairs) + { + shortest = Math.Min(shortest, pair.Find.Length); + } + + Assert.Equal(shortest, snapshot.ShortestFindLength); + } + + [Fact] + public void PairsAreOrderedLongestFirst() + { + // The most specific path has to win at any given position + var snapshot = DirectoryReplacements.Current; + for (var index = 1; index < snapshot.Pairs.Count; index++) + { + Assert.True(snapshot.Pairs[index - 1].Find.Length >= snapshot.Pairs[index].Find.Length); + } + } +} diff --git a/src/ApplyScrubbersTests/EngineRunner.cs b/src/ApplyScrubbersTests/EngineRunner.cs new file mode 100644 index 000000000..738b2b4c3 --- /dev/null +++ b/src/ApplyScrubbersTests/EngineRunner.cs @@ -0,0 +1,35 @@ +static class EngineRunner +{ + public static Dictionary EmptyContext { get; } = []; + + public static string Run(string input, params Scrubber[] scrubbers) + { + using var counter = Counter.Start(); + return Run(input, counter, scrubbers); + } + + public static string Run(string input, Counter counter, params Scrubber[] scrubbers) => + ScrubEngine.Run( + input, + EngineScrubberSet.ForScrubbers([.. scrubbers]), + counter, + EmptyContext, + applyDirectoryReplacements: false); + + public static string RunWithDirectoryReplacements(string input, params Scrubber[] scrubbers) + { + using var counter = Counter.Start(); + return ScrubEngine.Run( + input, + EngineScrubberSet.ForScrubbers([.. scrubbers]), + counter, + EmptyContext, + applyDirectoryReplacements: true); + } + + // Deterministic path replacements, matching the DisableScrubbersTests pattern. + // Called from static ctors of test classes that exercise path replacement, so it + // runs after the adapter has assigned the real target assembly paths. + public static void UseFakeDirectories() => + DirectoryReplacements.UseAssembly("C:/Code/TheSolution", "C:/Code/TheSolution/TheProject"); +} diff --git a/src/ApplyScrubbersTests/EngineTests.cs b/src/ApplyScrubbersTests/EngineTests.cs new file mode 100644 index 000000000..6f9f60296 --- /dev/null +++ b/src/ApplyScrubbersTests/EngineTests.cs @@ -0,0 +1,155 @@ +public class EngineTests +{ + [Fact] + public void Quarantine_ReplacementNotSeenByOtherScrubbers() + { + // Equal max lengths, so registration order applies: first replaces abc + // with xyz, and the quarantined xyz must not be re-matched by the second + var result = EngineRunner.Run( + "abc", + Scrubber.Replace("abc", "xyz"), + Scrubber.Replace("xyz", "!!!")); + Assert.Equal("xyz", result); + } + + [Fact] + public void Quarantine_ReplacementNotRescannedBySameScrubber() + { + // A replacement containing its own find must not recurse + var result = EngineRunner.Run("aa", Scrubber.Replace("aa", "aaaa")); + Assert.Equal("aaaa", result); + } + + [Fact] + public void ZeroCopy_NoMatches() + { + var input = "no matches here"; + var result = EngineRunner.Run(input, Scrubber.Replace("missing", "x")); + Assert.Same(input, result); + } + + [Fact] + public void Newlines_CrLfCollapsed() + { + var result = EngineRunner.Run("a\r\nb\rc", Scrubber.Replace("missing", "x")); + Assert.Equal("a\nb\nc", result); + } + + [Fact] + public void Newlines_ReplacementNormalized() + { + var result = EngineRunner.Run( + "token", + Scrubber.Match((CharSpan segment, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + index = segment.IndexOf("token".AsSpan()); + if (index < 0) + { + length = 0; + replacement = null; + return false; + } + + length = 5; + replacement = "x\r\ny"; + return true; + })); + Assert.Equal("x\ny", result); + } + + [Fact] + public void EmptyInput() + { + var input = string.Empty; + var result = EngineRunner.Run(input, Scrubber.Replace("a", "b")); + Assert.Same(input, result); + } + + [Fact] + public void Split_PrefixAndSuffixPreserved() + { + var result = EngineRunner.Run("xxabcxx", Scrubber.Replace("abc", "Y")); + Assert.Equal("xxYxx", result); + } + + [Fact] + public void EmptyReplacement_RemovesText() + { + var result = EngineRunner.Run("a-remove-b", Scrubber.Replace("-remove-", "")); + Assert.Equal("ab", result); + } + + [Fact] + public void MultipleMatches_SameScrubber() + { + var result = EngineRunner.Run("abc abc abc", Scrubber.Replace("abc", "Y")); + Assert.Equal("Y Y Y", result); + } + + [Fact] + public void WordBoundary_RejectsAdjacentLetterOrDigit() + { + var result = EngineRunner.Run( + "name names 1name name", + Scrubber.Replace("name", "X", requireWordBoundary: true)); + Assert.Equal("X names 1name X", result); + } + + [Fact] + public void WordBoundary_ReplacementChunkIsNeighbor() + { + // First scrubber replaces ab with 12. The second requires a word boundary; + // the preceding chunk now ends with '2' (a digit), so cd must not match. + var result = EngineRunner.Run( + "abcd", + Scrubber.Replace("ab", "12"), + Scrubber.Replace("cd", "YZ", requireWordBoundary: true)); + Assert.Equal("12cd", result); + } + + [Fact] + public void MinLength_SkipsShortValues() + { + var input = "ab"; + var result = EngineRunner.Run(input, Scrubber.Replace("abc", "x")); + Assert.Same(input, result); + } + + [Fact] + public void MultiPair_LongestWinsAtSamePosition() + { + var result = EngineRunner.Run( + "abc", + Scrubber.Replace(StringComparison.Ordinal, false, ("ab", "SHORT"), ("abc", "LONG"))); + Assert.Equal("LONG", result); + } + + [Fact] + public void MultiPair_EarlierPositionWins() + { + var result = EngineRunner.Run( + "abc", + Scrubber.Replace(StringComparison.Ordinal, false, ("bc", "B"), ("c", "C"))); + Assert.Equal("aB", result); + } + + [Fact] + public void Comparison_OrdinalIgnoreCase() + { + var result = EngineRunner.Run( + "ABC abc", + Scrubber.Replace("abc", "x", StringComparison.OrdinalIgnoreCase)); + Assert.Equal("x x", result); + } + + [Fact] + public void Validation_FindWithNewlineThrows() => + Assert.Throws(() => Scrubber.Replace("a\nb", "x")); + + [Fact] + public void Validation_WindowBoundsThrow() + { + Assert.Throws(() => Scrubber.Window(0, 5, (CharSpan _, Counter _, IReadOnlyDictionary _) => null)); + Assert.Throws(() => Scrubber.Window(5, 4, (CharSpan _, Counter _, IReadOnlyDictionary _) => null)); + } +} diff --git a/src/ApplyScrubbersTests/FastPathTests.cs b/src/ApplyScrubbersTests/FastPathTests.cs new file mode 100644 index 000000000..31007c007 --- /dev/null +++ b/src/ApplyScrubbersTests/FastPathTests.cs @@ -0,0 +1,107 @@ +public class FastPathTests +{ + static FastPathTests() => + EngineRunner.UseFakeDirectories(); + + static string Apply(string value, Action? configure = null) + { + var settings = new VerifySettings(); + configure?.Invoke(settings); + using var counter = Counter.Start(); + return ApplyScrubbers.ApplyForPropertyValue(value, settings, counter); + } + + [Fact] + public void ShortValue_NoScrubbers_SameInstance() + { + var value = "short"; + Assert.Same(value, Apply(value)); + } + + [Fact] + public void CarriageReturn_ForcesNormalization() + { + var value = "a\r\nb"; + var result = Apply(value); + Assert.Equal("a\nb", result); + } + + [Fact] + public void ValueShorterThanScrubberMin_SameInstance() + { + var value = "ab"; + var result = Apply(value, settings => settings.AddScrubber(Scrubber.Replace("abcdef", "x"))); + Assert.Same(value, result); + } + + [Fact] + public void MatchingValue_Scrubbed() + { + var result = Apply("abcdef!", settings => settings.AddScrubber(Scrubber.Replace("abcdef", "x"))); + Assert.Equal("x!", result); + } + + [Fact] + public void UnknownMinScrubber_DisablesFastPath() + { + var invoked = false; + Apply( + "a", + settings => settings.AddScrubber(Scrubber.Match( + (CharSpan _, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + invoked = true; + index = 0; + length = 0; + replacement = null; + return false; + }))); + Assert.True(invoked); + } + + [Fact] + public void PathValue_Replaced() + { + var result = Apply("C:/Code/TheSolution/TheProject/file.txt"); + Assert.Equal("{ProjectDirectory}file.txt", result); + } + + [Fact] + public void LegacyScrubber_ForcesFullPath() + { + var result = Apply( + "ab", + settings => settings.AddScrubber(builder => builder.Append('!'))); + Assert.Equal("ab!", result); + } + + [Fact] + public void ExtensionMappedScrubbers_ExcludedFromPropertyPath() + { + var value = "abc"; + var result = Apply(value, settings => settings.AddScrubber("txt", Scrubber.Replace("abc", "xyz"))); + Assert.Same(value, result); + } + + [Fact] + public void ScrubbersDisabled_OnlyNormalizes() + { + var result = Apply( + "abc\r\n", + settings => + { + settings.AddScrubber(Scrubber.Replace("abc", "xyz")); + settings.DisableScrubbers(); + }); + Assert.Equal("abc\n", result); + } + + [Fact] + public void NoMatch_EngineStillZeroCopy() + { + // Long enough to enter the engine, but nothing matches: same instance back + var value = new string('x', 200); + var result = Apply(value, settings => settings.AddScrubber(Scrubber.Replace("missing", "y"))); + Assert.Same(value, result); + } +} diff --git a/src/ApplyScrubbersTests/GlobalUsings.cs b/src/ApplyScrubbersTests/GlobalUsings.cs new file mode 100644 index 000000000..16b875289 --- /dev/null +++ b/src/ApplyScrubbersTests/GlobalUsings.cs @@ -0,0 +1,2 @@ +global using System.Text; +global using CharSpan = System.ReadOnlySpan; diff --git a/src/ApplyScrubbersTests/LegacyInteropTests.cs b/src/ApplyScrubbersTests/LegacyInteropTests.cs new file mode 100644 index 000000000..65b897380 --- /dev/null +++ b/src/ApplyScrubbersTests/LegacyInteropTests.cs @@ -0,0 +1,147 @@ +public class LegacyInteropTests +{ + static LegacyInteropTests() => + EngineRunner.UseFakeDirectories(); + + static string RunForExtension(string input, Action configure) + { + var settings = new VerifySettings(); + configure(settings); + using var counter = Counter.Start(); + var builder = new StringBuilder(input); + ApplyScrubbers.ApplyForExtension("txt", builder, settings, counter); + return builder.ToString(); + } + + [Fact] + public void LegacyRunsAfterEngine() + { + // The span scrubber quarantines its replacement from other span scrubbers, + // but the legacy pass still sees and may modify it + var result = RunForExtension( + "a", + settings => + { + settings.AddScrubber(Scrubber.Replace("a", "1")); + settings.AddScrubber(builder => builder.Replace("1", "2")); + }); + Assert.Equal("2", result); + } + + [Fact] + public void LegacyLocations_StillHonored() + { + var result = RunForExtension( + "value", + settings => + { + settings.AddScrubber(builder => builder.Append(" one"), ScrubberLocation.Last); + settings.AddScrubber(builder => builder.Append(" two"), ScrubberLocation.Last); + }); + Assert.Equal("value one two", result); + } + + [Fact] + public void LegacyDoesNotSeeCarriageReturns() + { + // The engine normalizes newlines before the legacy pass, so a legacy + // scrubber matching a literal "\r\n" no longer fires. Declared as a + // breaking change in the 32.0.0 release notes. + var result = RunForExtension( + "a\r\nb", + settings => settings.AddScrubber(builder => builder.Replace("a\r\nb", "MATCHED"))); + Assert.Equal("a\nb", result); + } + + [Fact] + public void FixNewlines_AfterLegacy() + { + var result = RunForExtension( + "value", + settings => settings.AddScrubber(builder => builder.Append("\r\nline"))); + Assert.Equal("value\nline", result); + } + + [Fact] + public void PathReplacements_AfterLegacy() + { + // A legacy scrubber writes a raw path; the trailing path replacement pass + // still converts it + var result = RunForExtension( + "value ", + settings => settings.AddScrubber(builder => builder.Append("C:/Code/TheSolution/TheProject/file.txt"))); + Assert.Equal("value {ProjectDirectory}file.txt", result); + } + + [Fact] + public void LegacyUserScrubber_BeatsPathReplacements() + { + // MoreSpecificScrubberShouldOverride semantics: the legacy scrubber sees the + // raw path before the path replacement pass + var result = RunForExtension( + "C:/Code/TheSolution/TheProject/data", + settings => settings.AddScrubber(builder => builder.Replace("C:/Code/TheSolution/TheProject/data", "{custom}"))); + Assert.Equal("{custom}", result); + } + + [Fact] + public void NoLegacy_EngineOnlyPath() + { + var result = RunForExtension( + "C:/Code/TheSolution/TheProject/file.txt abc", + settings => settings.AddScrubber(Scrubber.Replace("abc", "xyz"))); + Assert.Equal("{ProjectDirectory}file.txt xyz", result); + } + + [Fact] + public void ExtensionMapped_SpanScrubberApplies() + { + var result = RunForExtension( + "abc", + settings => settings.AddScrubber("txt", Scrubber.Replace("abc", "xyz"))); + Assert.Equal("xyz", result); + } + + [Fact] + public void ExtensionMapped_OtherExtensionIgnored() + { + var result = RunForExtension( + "abc", + settings => settings.AddScrubber("json", Scrubber.Replace("abc", "xyz"))); + Assert.Equal("abc", result); + } + + [Fact] + public void DisableScrubbers_BypassesEverything() + { + var result = RunForExtension( + "abc\r\n", + settings => + { + settings.AddScrubber(Scrubber.Replace("abc", "xyz")); + settings.DisableScrubbers(); + }); + Assert.Equal("abc\n", result); + } + + [Fact] + public void SettingsClone_CopiesSpanScrubbers() + { + var settings = new VerifySettings(); + settings.AddScrubber(Scrubber.Replace("abc", "xyz")); + settings.AddScrubber("txt", Scrubber.Replace("def", "uvw")); + + var clone = new VerifySettings(settings); + using var counter = Counter.Start(); + var builder = new StringBuilder("abc def"); + ApplyScrubbers.ApplyForExtension("txt", builder, clone, counter); + Assert.Equal("xyz uvw", builder.ToString()); + + // Mutating the clone must not affect the original + clone.AddScrubber(Scrubber.Replace("ghi", "rst")); + using var secondCounter = Counter.Start(); + var second = new StringBuilder("ghi"); + ApplyScrubbers.ApplyForExtension("txt", second, settings, secondCounter); + Assert.Equal("ghi", second.ToString()); + } +} diff --git a/src/ApplyScrubbersTests/LegacyPropertyValueTests.cs b/src/ApplyScrubbersTests/LegacyPropertyValueTests.cs new file mode 100644 index 000000000..36977ddfe --- /dev/null +++ b/src/ApplyScrubbersTests/LegacyPropertyValueTests.cs @@ -0,0 +1,74 @@ +// The property value path when a legacy StringBuilder scrubber is registered: +// span scrubbers run, then the legacy pass, then path replacement and the newline +// fix. Covers ApplyScrubbers.ApplyWithLegacy, which ApplyForExtension does not. +public class LegacyPropertyValueTests +{ + static LegacyPropertyValueTests() => + EngineRunner.UseFakeDirectories(); + + static string Run(string input, Action configure) + { + var settings = new VerifySettings(); + configure(settings); + using var counter = Counter.Start(); + return ApplyScrubbers.ApplyForPropertyValue(input, settings, counter); + } + + [Fact] + public void LegacyRunsAfterSpanScrubbers() + { + var result = Run( + "a", + settings => + { + settings.AddScrubber(Scrubber.Replace("a", "1")); + settings.AddScrubber(builder => builder.Replace("1", "2")); + }); + Assert.Equal("2", result); + } + + [Fact] + public void PathWrittenByLegacyIsScrubbed() + { + var result = Run( + "value ", + settings => settings.AddScrubber(builder => builder.Append("C:/Code/TheSolution/TheProject/file.txt"))); + Assert.Equal("value {ProjectDirectory}file.txt", result); + } + + [Fact] + public void LegacySeesRawPath() + { + var result = Run( + "C:/Code/TheSolution/TheProject/data", + settings => settings.AddScrubber(builder => builder.Replace("C:/Code/TheSolution/TheProject/data", "{custom}"))); + Assert.Equal("{custom}", result); + } + + [Fact] + public void NewlinesInjectedByLegacyAreNormalized() + { + var result = Run( + "value", + settings => settings.AddScrubber(builder => builder.Append("\r\nline\rmore"))); + Assert.Equal("value\nline\nmore", result); + } + + [Fact] + public void PathAdjacentToInjectedCarriageReturnIsStillScrubbed() + { + // Newline normalization now runs before path replacement rather than after, + // so a path sitting next to an injected '\r' must still be replaced + var result = Run( + "value", + settings => settings.AddScrubber(builder => builder.Append("\r\nC:/Code/TheSolution/TheProject/f.txt\r\n"))); + Assert.Equal("value\n{ProjectDirectory}f.txt\n", result); + } + + [Fact] + public void UnchangedValueIsReturnedAsIs() + { + var result = Run("plain value", settings => settings.AddScrubber(_ => { })); + Assert.Equal("plain value", result); + } +} diff --git a/src/ApplyScrubbersTests/LinePhaseTests.cs b/src/ApplyScrubbersTests/LinePhaseTests.cs new file mode 100644 index 000000000..df5bbe1fc --- /dev/null +++ b/src/ApplyScrubbersTests/LinePhaseTests.cs @@ -0,0 +1,175 @@ +public class LinePhaseTests +{ + [Fact] + public void Transforms_ComposeInRegistrationOrder() + { + var result = EngineRunner.Run( + "a", + Scrubber.ReplaceLines(_ => _ == "a" ? "b" : _), + Scrubber.ReplaceLines(_ => _ == "b" ? "c" : _)); + Assert.Equal("c", result); + } + + [Fact] + public void Transform_LineResultVariants() + { + var result = EngineRunner.Run( + "keep\nremove\nreplace", + Scrubber.ReplaceLines(line => + { + if (line.SequenceEqual("remove".AsSpan())) + { + return LineResult.Remove; + } + + if (line.SequenceEqual("replace".AsSpan())) + { + return LineResult.Replace("replaced"); + } + + return LineResult.Keep; + })); + Assert.Equal("keep\nreplaced", result); + } + + [Fact] + public void Transform_NullDrops() + { + var result = EngineRunner.Run( + "keep\ndrop me", + Scrubber.ReplaceLines(_ => _.Contains("drop") ? null : _)); + Assert.Equal("keep", result); + } + + [Fact] + public void TransformedLine_RescannedByInlineScrubbers() + { + var result = EngineRunner.Run( + "line", + Scrubber.ReplaceLines(_ => _ == "line" ? "token here" : _), + Scrubber.Replace("token", "X")); + Assert.Equal("X here", result); + } + + [Fact] + public void Drops_SeeRawLines_NotTransformOutput() + { + // The transform would rewrite the marker, but drops run first on the raw line + var result = EngineRunner.Run( + "clean\nsecret data", + Scrubber.ReplaceLines(_ => _.Replace("secret", "public")), + Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "secret")); + Assert.Equal("clean", result); + } + + [Fact] + public void NeedleDrop_LengthSkip() + { + // Lines shorter than the needle cannot match and are kept + var result = EngineRunner.Run( + "ab\nlongneedle here\ncd", + Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "longneedle")); + Assert.Equal("ab\ncd", result); + } + + [Fact] + public void ComparisonBucketing() + { + var result = EngineRunner.Run( + "foo\nFOO\nbar\nBAR\nkeep", + Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "foo"), + Scrubber.RemoveLinesContaining("BAR")); + Assert.Equal("FOO\nkeep", result); + } + + [Fact] + public void RemoveEmptyLines_DropsWhitespaceLines() + { + var result = EngineRunner.Run( + "a\n\n \t\nb", + Scrubber.RemoveEmptyLines()); + Assert.Equal("a\nb", result); + } + + [Fact] + public void RemoveEmptyLines_TrimsTrailingNewline() + { + // Matches legacy RemoveEmptyLines: the trailing newline is trimmed even when + // no line was dropped + var result = EngineRunner.Run( + "a\nb\n", + Scrubber.RemoveEmptyLines()); + Assert.Equal("a\nb", result); + } + + [Fact] + public void SpanPredicateOverload() + { + var result = EngineRunner.Run( + "keep\nremove", + Scrubber.RemoveLines((LineMatch) (line => line.StartsWith("remove".AsSpan())))); + Assert.Equal("keep", result); + } + + [Fact] + public void UntypedLambda_BindsStringOverload() + { + // _.Contains('D') is valid for both string and span, so overload + // resolution priority must pick the string overload instead of + // reporting an ambiguity + var result = EngineRunner.Run( + "a\nD\nb", + Scrubber.RemoveLines(_ => _.Contains('D'))); + Assert.Equal("a\nb", result); + } + + [Fact] + public void SpanSugarOverloads() + { + var settings = new VerifySettings(); + // Explicitly typed span lambdas select the LineMatch/LineReplace overloads + settings.ScrubLines((CharSpan line) => line.StartsWith("remove".AsSpan())); + settings.ScrubLinesWithReplace((CharSpan line) => + { + if (line.StartsWith("replace".AsSpan())) + { + return LineResult.Replace("replaced"); + } + + return LineResult.Keep; + }); + // Untyped lambda still binds the string overload without ambiguity + settings.ScrubLines(_ => _.Contains("drop")); + + using var counter = Counter.Start(); + var builder = new StringBuilder("keep\nremove\nreplace me\ndrop me"); + ApplyScrubbers.ApplyForExtension("txt", builder, settings, counter); + Assert.Equal("keep\nreplaced", builder.ToString()); + } + + [Fact] + public void DropFirstMiddleLast() + { + Assert.Equal("b\nc", EngineRunner.Run("x\nb\nc", Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "x"))); + Assert.Equal("a\nc", EngineRunner.Run("a\nx\nc", Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "x"))); + Assert.Equal("a\nb", EngineRunner.Run("a\nb\nx", Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "x"))); + Assert.Equal("", EngineRunner.Run("x\nx\nx", Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "x"))); + } + + [Fact] + public void TrailingNewlinePreservedOnDrop() + { + Assert.Equal("a\n", EngineRunner.Run("a\nx\n", Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "x"))); + Assert.Equal("a", EngineRunner.Run("a\nx", Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "x"))); + } + + [Fact] + public void LineResultReplace_NormalizesNewlines() + { + var replaced = LineResult.Replace("a\r\nb"); + var result = EngineRunner.Run( + "line", + Scrubber.ReplaceLines(_ => replaced)); + Assert.Equal("a\nb", result); + } +} diff --git a/src/ApplyScrubbersTests/MatchTests.cs b/src/ApplyScrubbersTests/MatchTests.cs new file mode 100644 index 000000000..39051ed65 --- /dev/null +++ b/src/ApplyScrubbersTests/MatchTests.cs @@ -0,0 +1,143 @@ +public class MatchTests +{ + static SegmentMatch DigitRunMatcher { get; } = + (CharSpan segment, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + for (index = 0; index < segment.Length; index++) + { + if (!char.IsDigit(segment[index])) + { + continue; + } + + var end = index; + while (end < segment.Length && + char.IsDigit(segment[end])) + { + end++; + } + + length = end - index; + replacement = "#"; + return true; + } + + length = 0; + replacement = null; + return false; + }; + + [Fact] + public void ReinvokedPastEachMatch() + { + var result = EngineRunner.Run("a1b22c333", Scrubber.Match(DigitRunMatcher)); + Assert.Equal("a#b#c#", result); + } + + [Fact] + public void MinLength_SkipsShortSegments() + { + var invoked = false; + EngineRunner.Run( + "abc", + Scrubber.Match( + (CharSpan _, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + invoked = true; + index = 0; + length = 0; + replacement = null; + return false; + }, + minLength: 5)); + Assert.False(invoked); + } + + [Fact] + public void InvalidRange_Throws() + { + var exception = Assert.ThrowsAny(() => + EngineRunner.Run( + "abc", + Scrubber.Match((CharSpan _, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + index = 0; + length = 10; + replacement = "x"; + return true; + }))); + Assert.Contains("invalid range", exception.Message); + } + + [Fact] + public void NewlineSpanningMatch_Throws() + { + var exception = Assert.ThrowsAny(() => + EngineRunner.Run( + "ab\ncd", + Scrubber.Match((CharSpan _, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + index = 0; + length = 5; + replacement = "x"; + return true; + }))); + Assert.Contains("line break", exception.Message); + } + + [Fact] + public void ContextAndCounter_FlowToMatcher() + { + using var counter = Counter.Start(); + Counter? seenCounter = null; + IReadOnlyDictionary? seenContext = null; + EngineRunner.Run( + "abc", + counter, + Scrubber.Match((CharSpan _, Counter matcherCounter, IReadOnlyDictionary matcherContext, out int index, out int length, out string? replacement) => + { + seenCounter = matcherCounter; + seenContext = matcherContext; + index = 0; + length = 0; + replacement = null; + return false; + })); + Assert.Same(counter, seenCounter); + Assert.Same(EngineRunner.EmptyContext, seenContext); + } + + [Fact] + public void UlidStyle_CounterNumbering() + { + // A fixed length token matcher using counter numbering, the Verify.Ulid shape + using var counter = Counter.Start(); + var result = EngineRunner.Run( + "01ARZ3NDEKTSV4RRFFQ69G5FAV then 01BX5ZZKBKACTAV9WEVGEMMVRZ", + counter, + Scrubber.Window(26, 26, (window, windowCounter, _) => + { + foreach (var ch in window) + { + if (!char.IsLetterOrDigit(ch)) + { + return null; + } + } + + return $"Ulid_{windowCounter.Next(new Guid(HashUlid(window), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))}"; + }, requireWordBoundary: true)); + Assert.Equal("Ulid_1 then Ulid_2", result); + } + + static int HashUlid(CharSpan window) + { + var hash = 17; + foreach (var ch in window) + { + hash = unchecked((hash * 31) + ch); + } + + return hash; + } +} diff --git a/src/ApplyScrubbersTests/MultiLineTransformTests.cs b/src/ApplyScrubbersTests/MultiLineTransformTests.cs new file mode 100644 index 000000000..ed608220c --- /dev/null +++ b/src/ApplyScrubbersTests/MultiLineTransformTests.cs @@ -0,0 +1,85 @@ +// A line transform is defined over a single line, so when one produces line +// breaks the remaining transforms must see each produced line on its own. +public class MultiLineTransformTests +{ + [Fact] + public void LaterTransformSeesProducedLines() + { + var result = EngineRunner.Run( + "a", + Scrubber.ReplaceLines((string line) => line == "a" ? "x\ny" : line), + Scrubber.ReplaceLines((string line) => line == "x" ? "z" : line)); + Assert.Equal("z\ny", result); + } + + [Fact] + public void LaterTransformSeesEveryProducedLine() + { + var result = EngineRunner.Run( + "a", + Scrubber.ReplaceLines((string line) => line == "a" ? "x\ny\nz" : line), + Scrubber.ReplaceLines((string line) => line.ToUpperInvariant())); + Assert.Equal("X\nY\nZ", result); + } + + [Fact] + public void LaterTransformCanRemoveAProducedLine() + { + var result = EngineRunner.Run( + "a", + Scrubber.ReplaceLines((string line) => line == "a" ? "keep\ndrop\nkeep2" : line), + Scrubber.ReplaceLines((string line) => line == "drop" ? null : line)); + Assert.Equal("keep\nkeep2", result); + } + + [Fact] + public void RemovingEveryProducedLineRemovesTheLine() + { + var result = EngineRunner.Run( + "a\nb", + Scrubber.ReplaceLines((string line) => line == "a" ? "x\ny" : line), + Scrubber.ReplaceLines((string line) => line is "x" or "y" ? null : line)); + Assert.Equal("b", result); + } + + [Fact] + public void ProducedLinesCanExpandAgain() + { + var result = EngineRunner.Run( + "a", + Scrubber.ReplaceLines((string line) => line == "a" ? "x\ny" : line), + Scrubber.ReplaceLines((string line) => line == "y" ? "1\n2" : line), + Scrubber.ReplaceLines((string line) => line == "2" ? "end" : line)); + Assert.Equal("x\n1\nend", result); + } + + [Fact] + public void SpanTransformSeesProducedLines() + { + var result = EngineRunner.Run( + "a", + Scrubber.ReplaceLines((CharSpan line) => line.SequenceEqual("a".AsSpan()) ? LineResult.Replace("x\ny") : LineResult.Keep), + Scrubber.ReplaceLines((CharSpan line) => line.SequenceEqual("x".AsSpan()) ? LineResult.Replace("z") : LineResult.Keep)); + Assert.Equal("z\ny", result); + } + + [Fact] + public void TrailingTransformLeavesTextExactlyAsProduced() + { + // Nothing follows the expanding transform, so the text is untouched + var result = EngineRunner.Run( + "a", + Scrubber.ReplaceLines((string line) => line == "a" ? "x\ny\n" : line)); + Assert.Equal("x\ny\n", result); + } + + [Fact] + public void SurroundingLinesArePreserved() + { + var result = EngineRunner.Run( + "before\na\nafter", + Scrubber.ReplaceLines((string line) => line == "a" ? "x\ny" : line), + Scrubber.ReplaceLines((string line) => line == "x" ? "z" : line)); + Assert.Equal("before\nz\ny\nafter", result); + } +} diff --git a/src/ApplyScrubbersTests/OrderingTests.cs b/src/ApplyScrubbersTests/OrderingTests.cs new file mode 100644 index 000000000..efcf8333d --- /dev/null +++ b/src/ApplyScrubbersTests/OrderingTests.cs @@ -0,0 +1,81 @@ +public class OrderingTests +{ + static OrderingTests() => + EngineRunner.UseFakeDirectories(); + + [Fact] + public void UnknownMaxRunsFirst() + { + // The Match scrubber has no max length, so it runs before the known length + // Replace even though the Replace was registered first + var result = EngineRunner.Run( + "abcdef", + Scrubber.Replace("abcdef", "R"), + Scrubber.Match((CharSpan segment, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + index = segment.IndexOf("abc".AsSpan()); + if (index < 0) + { + length = 0; + replacement = null; + return false; + } + + length = 3; + replacement = "M"; + return true; + })); + Assert.Equal("Mdef", result); + } + + [Fact] + public void LongestMaxRunsFirst() + { + // Registered shortest first; the longer max must still win the overlap + var result = EngineRunner.Run( + "abc", + Scrubber.Replace("bc", "X"), + Scrubber.Replace("abc", "Y")); + Assert.Equal("Y", result); + } + + [Fact] + public void EqualMax_RegistrationOrderWins() + { + var result = EngineRunner.Run( + "ab", + Scrubber.Replace("ab", "1"), + Scrubber.Replace("ab", "2")); + Assert.Equal("1", result); + } + + [Fact] + public void PathReplacementsApply() + { + var result = EngineRunner.RunWithDirectoryReplacements("C:/Code/TheSolution/TheProject/file.txt"); + Assert.Equal("{ProjectDirectory}file.txt", result); + } + + [Fact] + public void UserScrubberBeatsPathReplacements() + { + // Path replacements are pinned last: a user scrubber matching the raw path + // wins, mirroring MoreSpecificScrubberShouldOverride + var result = EngineRunner.RunWithDirectoryReplacements( + "C:/Code/TheSolution/TheProject/file.txt", + Scrubber.Replace("C:/Code/TheSolution/TheProject/file.txt", "{custom}")); + Assert.Equal("{custom}", result); + } + + [Fact] + public void PathReplacements_RespectQuarantine() + { + // The user scrubber consumes the project segment, so the full project path + // can no longer match; the remaining raw prefix still matches the solution + // directory (greedily absorbing the trailing separator) + var result = EngineRunner.RunWithDirectoryReplacements( + "C:/Code/TheSolution/TheProject", + Scrubber.Replace("TheProject", "{p}")); + Assert.Equal("{SolutionDirectory}{p}", result); + } +} diff --git a/src/ApplyScrubbersTests/WindowTests.cs b/src/ApplyScrubbersTests/WindowTests.cs new file mode 100644 index 000000000..9a180879a --- /dev/null +++ b/src/ApplyScrubbersTests/WindowTests.cs @@ -0,0 +1,88 @@ +public class WindowTests +{ + [Fact] + public void FixedLength() + { + var result = EngineRunner.Run( + "xx ABC xx", + Scrubber.Window(3, 3, (window, _, _) => window.SequenceEqual("ABC".AsSpan()) ? "match" : null)); + Assert.Equal("xx match xx", result); + } + + [Fact] + public void VariableLength_LongestTriedFirst() + { + var lengths = new List(); + EngineRunner.Run( + "abcdefgh", + Scrubber.Window(2, 4, (window, _, _) => + { + lengths.Add(window.Length); + return null; + })); + + // At the first position all lengths fit and are probed longest first + Assert.Equal([4, 3, 2], lengths.Take(3)); + } + + [Fact] + public void WindowsNeverContainNewlines() + { + var windows = new List(); + EngineRunner.Run( + "ab\ncd", + Scrubber.Window(2, 4, (window, _, _) => + { + windows.Add(window.ToString()); + return null; + })); + + Assert.NotEmpty(windows); + Assert.All(windows, _ => Assert.DoesNotContain('\n', _)); + } + + [Fact] + public void WordBoundary_DocumentEdgesAreValid() + { + var result = EngineRunner.Run( + "ABC", + Scrubber.Window(3, 3, (window, _, _) => window.SequenceEqual("ABC".AsSpan()) ? "match" : null, requireWordBoundary: true)); + Assert.Equal("match", result); + } + + [Fact] + public void WordBoundary_AdjacentLetterRejects() + { + var result = EngineRunner.Run( + "xABC ABCx ABC", + Scrubber.Window(3, 3, (window, _, _) => window.SequenceEqual("ABC".AsSpan()) ? "match" : null, requireWordBoundary: true)); + Assert.Equal("xABC ABCx match", result); + } + + [Fact] + public void Counter_FlowsToMatcher() + { + using var counter = Counter.Start(); + Counter? seen = null; + EngineRunner.Run( + "abc", + counter, + Scrubber.Window(3, 3, (_, matcherCounter, _) => + { + seen = matcherCounter; + return null; + })); + Assert.Same(counter, seen); + } + + [Fact] + public void GuidWindow_EndToEnd() + { + using var counter = Counter.Start(); + var result = EngineRunner.Run( + "id: 173535ae-995b-4cc6-a74e-8cd4be57039c done", + counter, + GuidMatcher.Instance); + Assert.Equal("id: Guid_1 done", result); + } +} diff --git a/src/Benchmarks/FilterLinesBenchmarks.cs b/src/Benchmarks/FilterLinesBenchmarks.cs index 714372038..a3e45795c 100644 --- a/src/Benchmarks/FilterLinesBenchmarks.cs +++ b/src/Benchmarks/FilterLinesBenchmarks.cs @@ -1,10 +1,19 @@ -[MemoryDiagnoser] +// Predicate based line removal (ScrubLines). Legacy_* rows run the pre-engine +// StringReader implementation; Engine_* rows run the span engine line phase with a +// string predicate (the engine materializes each line lazily for the predicate). +[MemoryDiagnoser] [SimpleJob(iterationCount: 10, warmupCount: 3)] public class FilterLinesBenchmarks { - StringBuilder smallInput = null!; - StringBuilder mediumInput = null!; - StringBuilder largeInput = null!; + string smallInput = null!; + string mediumInput = null!; + string largeInput = null!; + + EngineScrubberSet removeEvenSet = null!; + EngineScrubberSet neverMatchesSet = null!; + EngineScrubberSet spanRemoveEvenSet = null!; + EngineScrubberSet spanNeverMatchesSet = null!; + static Dictionary emptyContext = []; [GlobalSetup] public void Setup() @@ -17,9 +26,14 @@ public void Setup() // Large: ~500KB, 10000 lines largeInput = CreateTestData(10000, 50); + + removeEvenSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveLines(RemoveEvenLines)]); + neverMatchesSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveLines(NeverMatches)]); + spanRemoveEvenSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveLines((LineMatch) RemoveEvenLinesSpan)]); + spanNeverMatchesSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveLines((LineMatch) NeverMatchesSpan)]); } - static StringBuilder CreateTestData(int lineCount, int charsPerLine) + static string CreateTestData(int lineCount, int charsPerLine) { var builder = new StringBuilder(); for (var i = 0; i < lineCount; i++) @@ -28,7 +42,8 @@ static StringBuilder CreateTestData(int lineCount, int charsPerLine) builder.Append(i); builder.AppendLine(); } - return builder; + + return builder.ToString(); } // Remove every other line @@ -38,45 +53,93 @@ static bool RemoveEvenLines(string line) => // Never matches - no lines removed static bool NeverMatches(string line) => false; + // Span variants: no per-line string is materialized for these + static bool RemoveEvenLinesSpan(ReadOnlySpan line) => + line.Length > 0 && char.IsDigit(line[^1]) && (line[^1] - '0') % 2 == 0; + + static bool NeverMatchesSpan(ReadOnlySpan line) => false; + + static string Engine(EngineScrubberSet set, string content) + { + using var counter = Counter.Start(); + return ScrubEngine.Run(content, set, counter, emptyContext, applyDirectoryReplacements: false); + } + [Benchmark(Baseline = true)] - public void Small() + public void Legacy_Small() { - var builder = new StringBuilder(smallInput.ToString()); + var builder = new StringBuilder(smallInput); builder.FilterLines(RemoveEvenLines); } [Benchmark] - public void Medium() + public void Legacy_Medium() { - var builder = new StringBuilder(mediumInput.ToString()); + var builder = new StringBuilder(mediumInput); builder.FilterLines(RemoveEvenLines); } [Benchmark] - public void Large() + public void Legacy_Large() { - var builder = new StringBuilder(largeInput.ToString()); + var builder = new StringBuilder(largeInput); builder.FilterLines(RemoveEvenLines); } [Benchmark] - public void Small_NoMatches() + public void Legacy_Small_NoMatches() { - var builder = new StringBuilder(smallInput.ToString()); + var builder = new StringBuilder(smallInput); builder.FilterLines(NeverMatches); } [Benchmark] - public void Medium_NoMatches() + public void Legacy_Medium_NoMatches() { - var builder = new StringBuilder(mediumInput.ToString()); + var builder = new StringBuilder(mediumInput); builder.FilterLines(NeverMatches); } [Benchmark] - public void Large_NoMatches() + public void Legacy_Large_NoMatches() { - var builder = new StringBuilder(largeInput.ToString()); + var builder = new StringBuilder(largeInput); builder.FilterLines(NeverMatches); } -} \ No newline at end of file + + [Benchmark] + public string Engine_Small() => Engine(removeEvenSet, smallInput); + + [Benchmark] + public string Engine_Medium() => Engine(removeEvenSet, mediumInput); + + [Benchmark] + public string Engine_Large() => Engine(removeEvenSet, largeInput); + + [Benchmark] + public string Engine_Small_NoMatches() => Engine(neverMatchesSet, smallInput); + + [Benchmark] + public string Engine_Medium_NoMatches() => Engine(neverMatchesSet, mediumInput); + + [Benchmark] + public string Engine_Large_NoMatches() => Engine(neverMatchesSet, largeInput); + + [Benchmark] + public string EngineSpan_Small() => Engine(spanRemoveEvenSet, smallInput); + + [Benchmark] + public string EngineSpan_Medium() => Engine(spanRemoveEvenSet, mediumInput); + + [Benchmark] + public string EngineSpan_Large() => Engine(spanRemoveEvenSet, largeInput); + + [Benchmark] + public string EngineSpan_Small_NoMatches() => Engine(spanNeverMatchesSet, smallInput); + + [Benchmark] + public string EngineSpan_Medium_NoMatches() => Engine(spanNeverMatchesSet, mediumInput); + + [Benchmark] + public string EngineSpan_Large_NoMatches() => Engine(spanNeverMatchesSet, largeInput); +} diff --git a/src/Benchmarks/GlobalUsings.cs b/src/Benchmarks/GlobalUsings.cs index 34862b298..961afb644 100644 --- a/src/Benchmarks/GlobalUsings.cs +++ b/src/Benchmarks/GlobalUsings.cs @@ -2,3 +2,4 @@ global using BenchmarkDotNet.Attributes; global using BenchmarkDotNet.Running; global using VerifyTests; +global using Culture = System.Globalization.CultureInfo; diff --git a/src/Benchmarks/InlineDateScrubberBenchmarks.cs b/src/Benchmarks/InlineDateScrubberBenchmarks.cs index 7d5de2a33..796065daa 100644 --- a/src/Benchmarks/InlineDateScrubberBenchmarks.cs +++ b/src/Benchmarks/InlineDateScrubberBenchmarks.cs @@ -3,15 +3,11 @@ // density p50 6.5 per 1000 chars); DateTimeOffset 2.3%, DateOnly 1.6%. File sizes: // p50=260 chars, p90=2.9KB, p99=31KB. // -// DateScrubber ToString()s the whole builder (via AsSpan) and then tests every window -// of the format's length, so the scan cost dominates and is paid whether or not a date -// is present. A fixed-length format (ISO "yyyy-MM-ddTHH:mm:ss", min==max) takes the -// ReplaceFixedLength path; a variable-length format (short-date "d", min emptyContext = []; - Action dateTimeScrubber = null!; - Action dateScrubber = null!; - Action dateTimeOffsetScrubber = null!; + Action legacyDateTimeScrubber = null!; + Action legacyDateScrubber = null!; + Action legacyDateTimeOffsetScrubber = null!; + + EngineScrubberSet dateTimeSet = null!; + EngineScrubberSet dateSet = null!; + EngineScrubberSet dateTimeOffsetSet = null!; // Plain no-match content shared across the fixed (DateTime) and variable (Date) paths string smallNone = null!; @@ -40,9 +41,13 @@ public class InlineDateScrubberBenchmarks [GlobalSetup] public void Setup() { - dateTimeScrubber = DateScrubber.BuildDateTimeScrubber(dateTimeFormat, null); - dateScrubber = DateScrubber.BuildDateScrubber(dateFormat, null); - dateTimeOffsetScrubber = DateScrubber.BuildDateTimeOffsetScrubber(dateTimeOffsetFormat, null); + legacyDateTimeScrubber = LegacyDateScrubber.BuildDateTimeScrubber(dateTimeFormat, null); + legacyDateScrubber = LegacyDateScrubber.BuildDateScrubber(dateFormat, null); + legacyDateTimeOffsetScrubber = LegacyDateScrubber.BuildDateTimeOffsetScrubber(dateTimeOffsetFormat, null); + + dateTimeSet = EngineScrubberSet.ForScrubbers([.. DateMatchers.DateTimes(dateTimeFormat, null)]); + dateSet = EngineScrubberSet.ForScrubbers([.. DateMatchers.Dates(dateFormat, null)]); + dateTimeOffsetSet = EngineScrubberSet.ForScrubbers([.. DateMatchers.DateTimeOffsets(dateTimeOffsetFormat, null)]); smallNone = Build(260, 0, DateTimeToken); // p50 mediumNone = Build(2_900, 0, DateTimeToken); // p90 @@ -96,33 +101,69 @@ static string Build(int targetChars, int tokenEveryLines, Func toke return builder.ToString(); } + static void Legacy(Action scrubber, string content) + { + using var counter = Counter.Start(); + scrubber(new(content), counter); + } + + static string Engine(EngineScrubberSet set, string content) + { + using var counter = Counter.Start(); + return ScrubEngine.Run(content, set, counter, emptyContext, applyDirectoryReplacements: false); + } + // DateTime, fixed-length (ISO) path [Benchmark(Baseline = true)] - public void DateTime_Small_None() => dateTimeScrubber(new(smallNone), Counter.Start()); + public void Legacy_DateTime_Small_None() => Legacy(legacyDateTimeScrubber, smallNone); + + [Benchmark] + public void Legacy_DateTime_Medium_None() => Legacy(legacyDateTimeScrubber, mediumNone); + + [Benchmark] + public void Legacy_DateTime_Large_None() => Legacy(legacyDateTimeScrubber, largeNone); [Benchmark] - public void DateTime_Medium_None() => dateTimeScrubber(new(mediumNone), Counter.Start()); + public void Legacy_DateTime_Medium_Typical() => Legacy(legacyDateTimeScrubber, dateTimeMediumTypical); [Benchmark] - public void DateTime_Large_None() => dateTimeScrubber(new(largeNone), Counter.Start()); + public void Legacy_DateTime_Large_Typical() => Legacy(legacyDateTimeScrubber, dateTimeLargeTypical); [Benchmark] - public void DateTime_Medium_Typical() => dateTimeScrubber(new(dateTimeMediumTypical), Counter.Start()); + public string Engine_DateTime_Small_None() => Engine(dateTimeSet, smallNone); [Benchmark] - public void DateTime_Large_Typical() => dateTimeScrubber(new(dateTimeLargeTypical), Counter.Start()); + public string Engine_DateTime_Medium_None() => Engine(dateTimeSet, mediumNone); + + [Benchmark] + public string Engine_DateTime_Large_None() => Engine(dateTimeSet, largeNone); + + [Benchmark] + public string Engine_DateTime_Medium_Typical() => Engine(dateTimeSet, dateTimeMediumTypical); + + [Benchmark] + public string Engine_DateTime_Large_Typical() => Engine(dateTimeSet, dateTimeLargeTypical); // DateOnly, variable-length (short-date) path - same no-match input as DateTime_Medium_None [Benchmark] - public void Date_Medium_None() => dateScrubber(new(mediumNone), Counter.Start()); + public void Legacy_Date_Medium_None() => Legacy(legacyDateScrubber, mediumNone); [Benchmark] - public void Date_Medium_Typical() => dateScrubber(new(dateMediumTypical), Counter.Start()); + public void Legacy_Date_Medium_Typical() => Legacy(legacyDateScrubber, dateMediumTypical); + + [Benchmark] + public string Engine_Date_Medium_None() => Engine(dateSet, mediumNone); + + [Benchmark] + public string Engine_Date_Medium_Typical() => Engine(dateSet, dateMediumTypical); // DateTimeOffset [Benchmark] - public void DateTimeOffset_Medium_Typical() => dateTimeOffsetScrubber(new(dateTimeOffsetMediumTypical), Counter.Start()); + public void Legacy_DateTimeOffset_Medium_Typical() => Legacy(legacyDateTimeOffsetScrubber, dateTimeOffsetMediumTypical); + + [Benchmark] + public string Engine_DateTimeOffset_Medium_Typical() => Engine(dateTimeOffsetSet, dateTimeOffsetMediumTypical); } diff --git a/src/Benchmarks/InlineGuidScrubberBenchmarks.cs b/src/Benchmarks/InlineGuidScrubberBenchmarks.cs index 2577bc5b4..0f0f3b0a2 100644 --- a/src/Benchmarks/InlineGuidScrubberBenchmarks.cs +++ b/src/Benchmarks/InlineGuidScrubberBenchmarks.cs @@ -1,13 +1,10 @@ // Calibrated from a scan of 33,707 *.verified.* text files across D:\Code (2026-07): // size p50=260 chars, p90=2.9KB, p99=31KB. Guid placeholders appear in 5.1% of files // (density p50 1.5, p90 23 per 1000 chars; the densest single file held 1,057). -// GuidScrubber.ReplaceGuids walks StringBuilder.GetChunks() testing every 36-char -// window, so the scan cost is paid on every scrubbed file even when no guid is present -// (the common case). Builders produced by serialization span many chunks and exercise -// the cross-chunk carryover branch (recently bug-fixed); builders from raw-string -// verification are a single chunk. Both shapes are covered. -// A fresh Counter is created per invocation to mirror one-Counter-per-verify; its -// allocation (a handful of empty dictionaries) is a constant floor across all rows. +// Legacy_* rows run the pre-engine chunk-walking implementation (LegacyScrubbers.cs); +// Engine_* rows run the span engine. The Composition rows show the whole-pipeline win: +// guids + dates + line removal as one engine pass vs sequential legacy passes. +// A fresh Counter is created per invocation to mirror one-Counter-per-verify. [MemoryDiagnoser] [SimpleJob(iterationCount: 10, warmupCount: 3)] public class InlineGuidScrubberBenchmarks @@ -18,6 +15,12 @@ public class InlineGuidScrubberBenchmarks string mediumTypical = null!; string largeTypical = null!; string largeDense = null!; + string composition = null!; + + EngineScrubberSet guidSet = null!; + EngineScrubberSet compositionSet = null!; + static Dictionary emptyContext = []; + const string isoFormat = "yyyy-MM-ddTHH:mm:ss"; [GlobalSetup] public void Setup() @@ -28,10 +31,16 @@ public void Setup() mediumTypical = Build(2_900, 12); // ~1 guid / 12 lines ~= p50 density largeTypical = Build(31_000, 12); largeDense = Build(31_000, 1); // a guid on every line - worst case + composition = BuildComposition(31_000); + + guidSet = EngineScrubberSet.ForScrubbers([GuidMatcher.Instance]); + List compositionScrubbers = [GuidMatcher.Instance, Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "SECRET")]; + compositionScrubbers.AddRange(DateMatchers.DateTimes(isoFormat, null)); + compositionSet = EngineScrubberSet.ForScrubbers(compositionScrubbers); } // Lines of ~40 chars; every guidEveryLines'th line embeds a distinct, quote-delimited - // guid in canonical "D" format so GuidScrubber.ReplaceGuids parses and replaces it. + // guid in canonical "D" format. static string Build(int targetChars, int guidEveryLines) { var builder = new StringBuilder(); @@ -61,48 +70,112 @@ static string Build(int targetChars, int guidEveryLines) return builder.ToString(); } - // Rebuilds the same content as many ~200-char chunks, the way an incrementally - // appended serialization buffer arrives, forcing the chunk-carryover branch. - static StringBuilder MultiChunk(string content) + // Mixed content: a guid every 12 lines, an ISO timestamp every 4, a SECRET line every 10 + static string BuildComposition(int targetChars) { var builder = new StringBuilder(); - for (var start = 0; start < content.Length; start += 200) + var line = 0; + var seed = 1; + var baseDate = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Unspecified); + while (builder.Length < targetChars) { - var length = Math.Min(200, content.Length - start); - builder.Append(content.AsSpan(start, length)); + if (line % 12 == 0) + { + builder.Append(" \"id\": \""); + builder.Append($"{seed:x8}-0000-4000-8000-000000000000"); + builder.Append("\","); + } + else if (line % 10 == 0) + { + builder.Append(" [trace] SECRET session token for request "); + builder.Append(line); + } + else if (line % 4 == 0) + { + builder.Append(" \"timestamp\": \""); + builder.Append(baseDate.AddSeconds(seed).ToString(isoFormat, Culture.InvariantCulture)); + builder.Append("\","); + } + else + { + builder.Append(" \"name\": \"item "); + builder.Append(line); + builder.Append(" description text\","); + } + + seed++; + builder.Append('\n'); + line++; } - return builder; + return builder.ToString(); } - static void Scrub(StringBuilder builder) => - GuidScrubber.ReplaceGuids(builder, Counter.Start()); + static void LegacyScrub(string content) + { + using var counter = Counter.Start(); + var builder = new StringBuilder(content); + LegacyGuidScrubber.ReplaceGuids(builder, counter); + } - // Single chunk (raw-string verification shape) + string EngineScrub(string content) + { + using var counter = Counter.Start(); + return ScrubEngine.Run(content, guidSet, counter, emptyContext, applyDirectoryReplacements: false); + } [Benchmark(Baseline = true)] - public void Small_None() => Scrub(new(smallNone)); + public void Legacy_Small_None() => LegacyScrub(smallNone); + + [Benchmark] + public void Legacy_Medium_None() => LegacyScrub(mediumNone); + + [Benchmark] + public void Legacy_Large_None() => LegacyScrub(largeNone); + + [Benchmark] + public void Legacy_Medium_Typical() => LegacyScrub(mediumTypical); + + [Benchmark] + public void Legacy_Large_Typical() => LegacyScrub(largeTypical); [Benchmark] - public void Medium_None() => Scrub(new(mediumNone)); + public void Legacy_Large_Dense() => LegacyScrub(largeDense); [Benchmark] - public void Large_None() => Scrub(new(largeNone)); + public string Engine_Small_None() => EngineScrub(smallNone); [Benchmark] - public void Medium_Typical() => Scrub(new(mediumTypical)); + public string Engine_Medium_None() => EngineScrub(mediumNone); [Benchmark] - public void Large_Typical() => Scrub(new(largeTypical)); + public string Engine_Large_None() => EngineScrub(largeNone); [Benchmark] - public void Large_Dense() => Scrub(new(largeDense)); + public string Engine_Medium_Typical() => EngineScrub(mediumTypical); - // Multi chunk (serialized-output shape) - exercises the cross-chunk carryover branch + [Benchmark] + public string Engine_Large_Typical() => EngineScrub(largeTypical); [Benchmark] - public void Large_MultiChunk_None() => Scrub(MultiChunk(largeNone)); + public string Engine_Large_Dense() => EngineScrub(largeDense); + + // Whole-pipeline comparison: three scrubbers over mixed content [Benchmark] - public void Large_MultiChunk_Typical() => Scrub(MultiChunk(largeTypical)); + public void Legacy_Composition() + { + using var counter = Counter.Start(); + var builder = new StringBuilder(composition); + LegacyGuidScrubber.ReplaceGuids(builder, counter); + LegacyDateScrubber.ReplaceDateTimes(builder, isoFormat, counter, Culture.CurrentCulture); + builder.RemoveLinesContaining(StringComparison.Ordinal, "SECRET"); + } + + [Benchmark] + public string Engine_Composition() + { + using var counter = Counter.Start(); + return ScrubEngine.Run(composition, compositionSet, counter, emptyContext, applyDirectoryReplacements: false); + } } diff --git a/src/Verify/Serialization/Scrubbers/DateScrubber.cs b/src/Benchmarks/LegacyDateScrubber.cs similarity index 70% rename from src/Verify/Serialization/Scrubbers/DateScrubber.cs rename to src/Benchmarks/LegacyDateScrubber.cs index bfdafc9b7..ac3dc24bb 100644 --- a/src/Verify/Serialization/Scrubbers/DateScrubber.cs +++ b/src/Benchmarks/LegacyDateScrubber.cs @@ -1,23 +1,20 @@ -// ReSharper disable ReturnValueOfPureMethodIsNotUsed -static class DateScrubber +static class LegacyDateScrubber { delegate bool TryConvert( - CharSpan span, + ReadOnlySpan span, string format, Counter counter, Culture culture, [NotNullWhen(true)] out string? result); -#if NET6_0_OR_GREATER - static bool TryConvertDate( - CharSpan span, - [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, + ReadOnlySpan span, + string format, Counter counter, Culture culture, [NotNullWhen(true)] out string? result) { - if (Date.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) + if (DateOnly.TryParseExact(span, format, culture, DateTimeStyles.None, out var date)) { result = counter.Convert(date); return true; @@ -27,54 +24,23 @@ static bool TryConvertDate( return false; } - public static Action BuildDateScrubber( - [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, - Culture? culture) - { - try - { - Date.MaxValue.ToString(format, culture); - } - catch (FormatException exception) - { - throw new($"Format '{format}' is not valid for DateOnly.ToString(format, culture).", exception); - } - - return (builder, counter) => ReplaceDates(builder, format, counter, culture ?? Culture.CurrentCulture); - } + public static Action BuildDateScrubber(string format, Culture? culture) => + (builder, counter) => ReplaceDates(builder, format, counter, culture ?? Culture.CurrentCulture); - public static void ReplaceDates( - StringBuilder builder, - [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, - Counter counter, - Culture culture) => + public static void ReplaceDates(StringBuilder builder, string format, Counter counter, Culture culture) => ReplaceInner( builder, format, counter, culture, TryConvertDate); -#endif - public static Action BuildDateTimeOffsetScrubber( - [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture) - { - try - { - DateTimeOffset.MaxValue.ToString(format, culture); - } - catch (FormatException exception) - { - throw new($"Format '{format}' is not valid for DateTimeOffset.ToString(format, culture).", exception); - } - - return (builder, counter) => ReplaceDateTimeOffsets(builder, format, counter, culture ?? Culture.CurrentCulture); - } + public static Action BuildDateTimeOffsetScrubber(string format, Culture? culture) => + (builder, counter) => ReplaceDateTimeOffsets(builder, format, counter, culture ?? Culture.CurrentCulture); static bool TryConvertDateTimeOffset( - CharSpan span, - [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, + ReadOnlySpan span, + string format, Counter counter, Culture culture, [NotNullWhen(true)] out string? result) @@ -89,11 +55,7 @@ static bool TryConvertDateTimeOffset( return false; } - public static void ReplaceDateTimeOffsets( - StringBuilder builder, - [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Counter counter, - Culture culture) + public static void ReplaceDateTimeOffsets(StringBuilder builder, string format, Counter counter, Culture culture) { ReplaceInner( builder, @@ -113,8 +75,8 @@ public static void ReplaceDateTimeOffsets( } static bool TryConvertDateTime( - CharSpan span, - [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, + ReadOnlySpan span, + string format, Counter counter, Culture culture, [NotNullWhen(true)] out string? result) @@ -129,19 +91,8 @@ static bool TryConvertDateTime( return false; } - public static Action BuildDateTimeScrubber(string format, Culture? culture) - { - try - { - DateTime.MaxValue.ToString(format, culture); - } - catch (FormatException exception) - { - throw new($"Format '{format}' is not valid for DateTime.ToString(format, culture).", exception); - } - - return (builder, counter) => ReplaceDateTimes(builder, format, counter, culture ?? Culture.CurrentCulture); - } + public static Action BuildDateTimeScrubber(string format, Culture? culture) => + (builder, counter) => ReplaceDateTimes(builder, format, counter, culture ?? Culture.CurrentCulture); public static void ReplaceDateTimes(StringBuilder builder, string format, Counter counter, Culture culture) { diff --git a/src/Verify/Serialization/Scrubbers/LinesScrubber.cs b/src/Benchmarks/LegacyLineScrubber.cs similarity index 50% rename from src/Verify/Serialization/Scrubbers/LinesScrubber.cs rename to src/Benchmarks/LegacyLineScrubber.cs index 56d480410..ee597dd4f 100644 --- a/src/Verify/Serialization/Scrubbers/LinesScrubber.cs +++ b/src/Benchmarks/LegacyLineScrubber.cs @@ -1,8 +1,45 @@ -static class LinesScrubber +static class LegacyLineScrubber { - public static void RemoveLinesContaining(this StringBuilder input, StringComparison comparison, params string[] stringToMatch) + public static void FilterLines(this StringBuilder input, Func removeLine) { - Ensure.NotNullOrEmpty(stringToMatch); + var theString = input.ToString(); + using var reader = new StringReader(theString); + input.Clear(); + + while (reader.ReadLine() is { } line) + { + if (removeLine(line)) + { + continue; + } + + input.AppendLineN(line); + } + + var endsWithNewLine = theString.EndsWith('\n'); + if (input.Length > 0 && !endsWithNewLine) + { + input.Length -= 1; + } + } + + public static void RemoveEmptyLines(this StringBuilder builder) + { + builder.FilterLines(string.IsNullOrWhiteSpace); + if (builder.Length > 0 && + builder[0] is '\n') + { + builder.Remove(0, 1); + } + + if (builder.Length > 0 && + builder[^1] is '\n') + { + builder.Length--; + } + } + + public static void RemoveLinesContaining(this StringBuilder input, StringComparison comparison, params string[] stringToMatch) => input.FilterLines(_ => { foreach (var toMatch in stringToMatch) @@ -15,7 +52,6 @@ public static void RemoveLinesContaining(this StringBuilder input, StringCompari return false; }); - } public static void ReplaceLines(this StringBuilder input, Func replaceLine) { diff --git a/src/Benchmarks/LegacyScrubberPathBenchmarks.cs b/src/Benchmarks/LegacyScrubberPathBenchmarks.cs new file mode 100644 index 000000000..26fa911ef --- /dev/null +++ b/src/Benchmarks/LegacyScrubberPathBenchmarks.cs @@ -0,0 +1,65 @@ +// When any legacy AddScrubber(Action) is registered, the span engine +// runs first and a StringBuilder pass follows. The trailing path replacement and +// newline fix then run over that builder. +// PropertyValue_* is the hot one: it runs once per serialized string value, and it +// materialized the builder twice, once inside the path pass and once to return. +// Document_* runs once per file and materializes once. +[MemoryDiagnoser] +[SimpleJob(iterationCount: 10, warmupCount: 3)] +public class LegacyScrubberPathBenchmarks +{ + string small = null!; + string medium = null!; + string large = null!; + + VerifySettings settings = null!; + Counter counter = null!; + + [GlobalSetup] + public void Setup() + { + // Deterministic path replacements, so the trailing pass has real work to scan + DirectoryReplacements.UseAssembly("C:/Code/TheSolution", "C:/Code/TheSolution/TheProject"); + + small = Build(260); + medium = Build(2_900); + large = Build(31_000); + + settings = new(); + // A no-op legacy scrubber, enough to select the legacy pipeline + settings.AddScrubber(_ => { }); + + counter = Counter.Start(); + } + + [GlobalCleanup] + public void Cleanup() => counter.Dispose(); + + static string Build(int targetChars) + { + var builder = new StringBuilder(); + while (builder.Length < targetChars) + { + builder.Append(" \"name\": \"some value here\",\n"); + } + + return builder.ToString(); + } + + [Benchmark] + public string PropertyValue_Small() => ApplyScrubbers.ApplyForPropertyValue(small, settings, counter); + + [Benchmark] + public string PropertyValue_Medium() => ApplyScrubbers.ApplyForPropertyValue(medium, settings, counter); + + [Benchmark] + public string PropertyValue_Large() => ApplyScrubbers.ApplyForPropertyValue(large, settings, counter); + + [Benchmark] + public int Document_Large() + { + var builder = new StringBuilder(large); + ApplyScrubbers.ApplyForExtension("txt", builder, settings, counter); + return builder.Length; + } +} diff --git a/src/Verify/Serialization/Scrubbers/GuidScrubber.cs b/src/Benchmarks/LegacyScrubbers.cs similarity index 93% rename from src/Verify/Serialization/Scrubbers/GuidScrubber.cs rename to src/Benchmarks/LegacyScrubbers.cs index 84162387c..0d4a7ce9f 100644 --- a/src/Verify/Serialization/Scrubbers/GuidScrubber.cs +++ b/src/Benchmarks/LegacyScrubbers.cs @@ -1,4 +1,8 @@ -static class GuidScrubber +// Copies of the pre-engine StringBuilder based scrubber implementations, preserved so the +// benchmarks can report before/after rows over identical corpora. Sourced from +// src/Verify/Serialization/Scrubbers (deleted when the span engine replaced them). + +static class LegacyGuidScrubber { public static void ReplaceGuids(StringBuilder builder, Counter counter) { @@ -117,10 +121,7 @@ static List FindMatches(StringBuilder builder, Counter counter) } } - // Roll the carryover forward: keep the last 35 chars of everything seen - // so far. Rebuilding it from the current chunk alone drops the prefix - // when a chunk is shorter than 35, so a guid spanning three or more - // chunks would never be found. + // Roll the carryover forward: keep the last 35 chars of everything seen so far if (chunk.Length >= 35) { chunkSpan.Slice(chunk.Length - 35, 35).CopyTo(carryoverBuffer); @@ -155,7 +156,7 @@ static bool IsInvalidStartingChar(char ch) => ch != '{' && ch != '('; - internal readonly struct Match(int index, string value) + readonly struct Match(int index, string value) { public readonly int Index = index; public readonly string Value = value; diff --git a/src/Benchmarks/LineScrubberBenchmarks.cs b/src/Benchmarks/LineScrubberBenchmarks.cs index 5c9d16112..b42d37cb2 100644 --- a/src/Benchmarks/LineScrubberBenchmarks.cs +++ b/src/Benchmarks/LineScrubberBenchmarks.cs @@ -1,10 +1,9 @@ // Calibrated from a scan of 33,707 *.verified.* text files across D:\Code (2026-07): -// line counts p50=15, p90=88, p99=508. The line scrubbers round-trip the whole builder -// through StringReader.ReadLine (one string allocation per line) and rebuild it, so cost -// and allocations scale with line count rather than raw byte size. -// FilterLines (the predicate path used by ScrubLines) is covered by FilterLinesBenchmarks; -// this covers the Contains-matching (ScrubLinesContaining), empty-line (ScrubEmptyLines) -// and replace (ScrubLinesWithReplace) paths. +// line counts p50=15, p90=88, p99=508. ScrubLinesContaining is the most common hand-written +// scrubber in the wild (56 call sites over 15 repos, 1-2 needles, OrdinalIgnoreCase default). +// Legacy_* rows run the pre-engine implementation: whole-document ToString + StringReader +// (one string allocation per line) + rebuild. Engine_* rows run the span engine line phase +// (per-line span checks; kept lines never allocate). [MemoryDiagnoser] [SimpleJob(iterationCount: 10, warmupCount: 3)] public class LineScrubberBenchmarks @@ -14,6 +13,12 @@ public class LineScrubberBenchmarks string large = null!; string mediumWithBlanks = null!; + EngineScrubberSet containsOrdinalSet = null!; + EngineScrubberSet containsIgnoreCaseSet = null!; + EngineScrubberSet emptyLinesSet = null!; + EngineScrubberSet replaceLinesSet = null!; + static Dictionary emptyContext = []; + [GlobalSetup] public void Setup() { @@ -21,6 +26,11 @@ public void Setup() medium = Build(88, false); // p90 large = Build(508, false); // p99 mediumWithBlanks = Build(88, true); + + containsOrdinalSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "SECRET")]); + containsIgnoreCaseSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveLinesContaining("secret")]); + emptyLinesSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveEmptyLines()]); + replaceLinesSet = EngineScrubberSet.ForScrubbers([Scrubber.ReplaceLines(Keep)]); } // Lines of ~55 chars; every 10th line carries a "SECRET" marker (the kind of noise @@ -58,29 +68,65 @@ static string Build(int lineCount, bool withBlanks) static string? Keep(string line) => line; + static string Engine(EngineScrubberSet set, string content) + { + using var counter = Counter.Start(); + return ScrubEngine.Run(content, set, counter, emptyContext, applyDirectoryReplacements: false); + } + // ScrubLinesContaining - Contains match on every line [Benchmark(Baseline = true)] - public void RemoveLinesContaining_Small() => + public void Legacy_RemoveLinesContaining_Small() => new StringBuilder(small).RemoveLinesContaining(StringComparison.Ordinal, "SECRET"); [Benchmark] - public void RemoveLinesContaining_Medium() => + public void Legacy_RemoveLinesContaining_Medium() => new StringBuilder(medium).RemoveLinesContaining(StringComparison.Ordinal, "SECRET"); [Benchmark] - public void RemoveLinesContaining_Large() => + public void Legacy_RemoveLinesContaining_Large() => new StringBuilder(large).RemoveLinesContaining(StringComparison.Ordinal, "SECRET"); + [Benchmark] + public string Engine_RemoveLinesContaining_Small() => + Engine(containsOrdinalSet, small); + + [Benchmark] + public string Engine_RemoveLinesContaining_Medium() => + Engine(containsOrdinalSet, medium); + + [Benchmark] + public string Engine_RemoveLinesContaining_Large() => + Engine(containsOrdinalSet, large); + + // OrdinalIgnoreCase is the wild default (44 of 56 call sites) + + [Benchmark] + public void Legacy_RemoveLinesContaining_Large_IgnoreCase() => + new StringBuilder(large).RemoveLinesContaining(StringComparison.OrdinalIgnoreCase, "secret"); + + [Benchmark] + public string Engine_RemoveLinesContaining_Large_IgnoreCase() => + Engine(containsIgnoreCaseSet, large); + // ScrubEmptyLines [Benchmark] - public void RemoveEmptyLines_Medium() => + public void Legacy_RemoveEmptyLines_Medium() => new StringBuilder(mediumWithBlanks).RemoveEmptyLines(); + [Benchmark] + public string Engine_RemoveEmptyLines_Medium() => + Engine(emptyLinesSet, mediumWithBlanks); + // ScrubLinesWithReplace - identity replace isolates the read/rewrite round-trip [Benchmark] - public void ReplaceLines_Medium() => + public void Legacy_ReplaceLines_Medium() => new StringBuilder(medium).ReplaceLines(Keep); + + [Benchmark] + public string Engine_ReplaceLines_Medium() => + Engine(replaceLinesSet, medium); } diff --git a/src/Benchmarks/NoMatchAllocationBenchmarks.cs b/src/Benchmarks/NoMatchAllocationBenchmarks.cs new file mode 100644 index 000000000..6116805c6 --- /dev/null +++ b/src/Benchmarks/NoMatchAllocationBenchmarks.cs @@ -0,0 +1,81 @@ +// The no-match pass is the dominant real case: scrubbers are registered globally, +// but most documents (and most serialized property values) contain nothing they +// match. Those passes should allocate nothing. +// Inline_* registers a Replace whose find never occurs. Line_* registers a +// RemoveLinesContaining whose needle never occurs, which routes through the line +// phase. Both_* registers both. +// Allocated is the metric that matters here, not Mean. +[MemoryDiagnoser] +[SimpleJob(iterationCount: 10, warmupCount: 3)] +public class NoMatchAllocationBenchmarks +{ + static Dictionary emptyContext = []; + + string small = null!; + string medium = null!; + string large = null!; + + EngineScrubberSet inlineSet = null!; + EngineScrubberSet lineSet = null!; + EngineScrubberSet bothSet = null!; + + // One Counter for the whole run: Counter.Start allocates several dictionaries, + // which would swamp the engine allocation being measured here. A verify creates + // one Counter and scrubs many values through it. + Counter counter = null!; + + [GlobalCleanup] + public void Cleanup() => counter.Dispose(); + + [GlobalSetup] + public void Setup() + { + counter = Counter.Start(); + small = Build(260); // p50 + medium = Build(2_900); // p90 + large = Build(31_000); // p99 + + inlineSet = EngineScrubberSet.ForScrubbers([Scrubber.Replace("NEVER_OCCURS_TOKEN", "{X}")]); + lineSet = EngineScrubberSet.ForScrubbers([Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "NEVER_OCCURS_TOKEN")]); + bothSet = EngineScrubberSet.ForScrubbers( + [ + Scrubber.Replace("NEVER_OCCURS_TOKEN", "{X}"), + Scrubber.RemoveLinesContaining(StringComparison.Ordinal, "ALSO_NEVER") + ]); + } + + static string Build(int targetChars) + { + var builder = new StringBuilder(); + while (builder.Length < targetChars) + { + builder.Append(" \"name\": \"some value here\",\n"); + } + + return builder.ToString(); + } + + string Run(string input, EngineScrubberSet set) => + ScrubEngine.Run(input, set, counter, emptyContext, applyDirectoryReplacements: false); + + [Benchmark] + public string Inline_Small() => Run(small, inlineSet); + + [Benchmark] + public string Inline_Medium() => Run(medium, inlineSet); + + [Benchmark] + public string Inline_Large() => Run(large, inlineSet); + + [Benchmark] + public string Line_Small() => Run(small, lineSet); + + [Benchmark] + public string Line_Medium() => Run(medium, lineSet); + + [Benchmark] + public string Line_Large() => Run(large, lineSet); + + [Benchmark] + public string Both_Large() => Run(large, bothSet); +} diff --git a/src/Benchmarks/SerializationConvertBenchmarks.cs b/src/Benchmarks/SerializationConvertBenchmarks.cs index df6d6564b..f81be4ee9 100644 --- a/src/Benchmarks/SerializationConvertBenchmarks.cs +++ b/src/Benchmarks/SerializationConvertBenchmarks.cs @@ -18,7 +18,7 @@ public class SerializationConvertBenchmarks { const int count = 100; - static readonly CultureInfo culture = CultureInfo.CurrentCulture; + static readonly Culture culture = Culture.CurrentCulture; Guid[] guids = null!; DateTime[] dateTimes = null!; diff --git a/src/Benchmarks/SpliceBenchmarks.cs b/src/Benchmarks/SpliceBenchmarks.cs new file mode 100644 index 000000000..a45195bd4 --- /dev/null +++ b/src/Benchmarks/SpliceBenchmarks.cs @@ -0,0 +1,82 @@ +// Splice removes and re-inserts in the shared chunk list, so every match shifts the +// elements after it. That is free while the list is short or matches land at the +// tail, but a second scrubber running over a list a first scrubber already +// fragmented splices near the front of a long list, giving O(matches x chunks). +// Fragmented_* rows are that shape: scrubber A (longer find, so it runs first) +// splits the document into ~2N chunks, then scrubber B matches N times across them. +// Single_* rows are the control: B alone over an unfragmented document, where +// splices land at the tail and cost nothing. +// Sizes step by 4x so a quadratic term shows as a ~16x step. +[MemoryDiagnoser] +[SimpleJob(iterationCount: 10, warmupCount: 3)] +public class SpliceBenchmarks +{ + const string longToken = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; // 40 chars + const string shortToken = "BBBBBBBBBB"; // 10 chars + + static Dictionary emptyContext = []; + + string small = null!; + string medium = null!; + string large = null!; + + EngineScrubberSet fragmentingSet = null!; + EngineScrubberSet singleSet = null!; + + [GlobalSetup] + public void Setup() + { + small = Build(200); + medium = Build(800); + large = Build(3_200); + + // A has the longer find so the engine orders it first, fragmenting the list + // before B runs over it + fragmentingSet = EngineScrubberSet.ForScrubbers( + [ + Scrubber.Replace(longToken, "{A}"), + Scrubber.Replace(shortToken, "{B}") + ]); + singleSet = EngineScrubberSet.ForScrubbers([Scrubber.Replace(shortToken, "{B}")]); + } + + // Each line holds one long token and one short token + static string Build(int lines) + { + var builder = new StringBuilder(); + for (var line = 0; line < lines; line++) + { + builder.Append(" \"a\": \""); + builder.Append(longToken); + builder.Append("\", \"b\": \""); + builder.Append(shortToken); + builder.Append("\",\n"); + } + + return builder.ToString(); + } + + static string Run(string input, EngineScrubberSet set) + { + using var counter = Counter.Start(); + return ScrubEngine.Run(input, set, counter, emptyContext, applyDirectoryReplacements: false); + } + + [Benchmark] + public string Fragmented_200() => Run(small, fragmentingSet); + + [Benchmark] + public string Fragmented_800() => Run(medium, fragmentingSet); + + [Benchmark] + public string Fragmented_3200() => Run(large, fragmentingSet); + + [Benchmark] + public string Single_200() => Run(small, singleSet); + + [Benchmark] + public string Single_800() => Run(medium, singleSet); + + [Benchmark] + public string Single_3200() => Run(large, singleSet); +} diff --git a/src/Benchmarks/TempPathScrubberBenchmarks.cs b/src/Benchmarks/TempPathScrubberBenchmarks.cs new file mode 100644 index 000000000..039fba741 --- /dev/null +++ b/src/Benchmarks/TempPathScrubberBenchmarks.cs @@ -0,0 +1,153 @@ +// TempDirectory and TempFile each held a verbatim copy of the tracked-path matcher. +// Duplicated_* rows run that copy, built inline so it closes over statics only. +// Shared_* rows run the extracted TempPathScrubber.Build, whose matcher closes over +// its parameters instead. The question is whether that extra closure indirection +// costs anything on the scrub path. +// Document sizes follow the *.verified.* scan used by the other scrubber benchmarks: +// p50=260 chars, p90=2.9KB, p99=31KB. NoPaths is the dominant real case, since the +// async local is null unless a test actually created a temp path. +[MemoryDiagnoser] +[SimpleJob(iterationCount: 10, warmupCount: 3)] +public class TempPathScrubberBenchmarks +{ + static AsyncLocal?> asyncPaths = new(); + static readonly object pathsLock = new(); + + // A realistic root, matching [System.Temp]\VerifyTempDirectory + static string rootDirectory = Path.Combine(Path.GetTempPath(), "VerifyTempDirectory"); + static string trackedPath = Path.Combine(rootDirectory, "abcd1234.xyz"); + + static Dictionary emptyContext = []; + + string small = null!; + string medium = null!; + string large = null!; + string largeWithPaths = null!; + + EngineScrubberSet duplicatedSet = null!; + EngineScrubberSet sharedSet = null!; + + [GlobalSetup] + public void Setup() + { + small = Build(260, false); + medium = Build(2_900, false); + large = Build(31_000, false); + largeWithPaths = Build(31_000, true); + + duplicatedSet = EngineScrubberSet.ForScrubbers([BuildDuplicated()]); + sharedSet = EngineScrubberSet.ForScrubbers([TempPathScrubber.Build(asyncPaths, pathsLock, "{TempDirectory}", rootDirectory.Length)]); + } + + // Lines of ~40 chars; when withPaths, every 12th line embeds the tracked path + static string Build(int targetChars, bool withPaths) + { + var builder = new StringBuilder(); + var line = 0; + while (builder.Length < targetChars) + { + if (withPaths && + line % 12 == 0) + { + builder.Append(" \"path\": \""); + builder.Append(trackedPath); + builder.Append("\","); + } + else + { + builder.Append(" \"name\": \"some value here\","); + } + + builder.Append('\n'); + line++; + } + + return builder.ToString(); + } + + // The matcher as it was duplicated in TempDirectory.Init and TempFile.Init + static Scrubber BuildDuplicated() => + Scrubber.Match( + (ReadOnlySpan segment, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + index = -1; + length = 0; + replacement = "{TempDirectory}"; + var pathsValue = asyncPaths.Value; + if (pathsValue == null) + { + return false; + } + + lock (pathsLock) + { + foreach (var path in pathsValue) + { + var found = segment.IndexOf(path.AsSpan(), StringComparison.Ordinal); + if (found < 0) + { + continue; + } + + if (index < 0 || + found < index) + { + index = found; + length = path.Length; + } + } + } + + return index >= 0; + }, + minLength: rootDirectory.Length); + + static string Run(string input, EngineScrubberSet set) + { + using var counter = Counter.Start(); + return ScrubEngine.Run(input, set, counter, emptyContext, applyDirectoryReplacements: false); + } + + static string RunWithPaths(string input, EngineScrubberSet set) + { + asyncPaths.Value = [trackedPath]; + try + { + return Run(input, set); + } + finally + { + asyncPaths.Value = null; + } + } + + [Benchmark] + public string Duplicated_NoPaths_Small() => Run(small, duplicatedSet); + + [Benchmark] + public string Shared_NoPaths_Small() => Run(small, sharedSet); + + [Benchmark] + public string Duplicated_NoPaths_Medium() => Run(medium, duplicatedSet); + + [Benchmark] + public string Shared_NoPaths_Medium() => Run(medium, sharedSet); + + [Benchmark] + public string Duplicated_NoPaths_Large() => Run(large, duplicatedSet); + + [Benchmark] + public string Shared_NoPaths_Large() => Run(large, sharedSet); + + [Benchmark] + public string Duplicated_PathsNoMatch_Large() => RunWithPaths(large, duplicatedSet); + + [Benchmark] + public string Shared_PathsNoMatch_Large() => RunWithPaths(large, sharedSet); + + [Benchmark] + public string Duplicated_PathsMatch_Large() => RunWithPaths(largeWithPaths, duplicatedSet); + + [Benchmark] + public string Shared_PathsMatch_Large() => RunWithPaths(largeWithPaths, sharedSet); +} diff --git a/src/StaticSettingsTests/UserMachineScrubberTests.cs b/src/StaticSettingsTests/UserMachineScrubberTests.cs index 2133fb5bb..8e6ac57ad 100644 --- a/src/StaticSettingsTests/UserMachineScrubberTests.cs +++ b/src/StaticSettingsTests/UserMachineScrubberTests.cs @@ -5,13 +5,12 @@ public UserMachineScrubberTests() => UserMachineScrubber.ResetReplacements("FakeMachineName", "FakeUserName"); [Fact] - public void MultipleChunks() + public void Replace() { - var builder = new StringBuilder(capacity: 8, maxCapacity: int.MaxValue); - builder.Append("FakeUserName"); using var counter = Counter.Start(); - UserMachineScrubber.PerformReplacements(builder, "FakeUserName", "TheUserName"); - Assert.Equal("TheUserName", builder.ToString()); + var set = EngineScrubberSet.ForScrubbers([UserMachineScrubber.UserScrubber()]); + var result = ScrubEngine.Run("FakeUserName", set, counter, new Dictionary(), applyDirectoryReplacements: false); + Assert.Equal("TheUserName", result); } [Fact] diff --git a/src/Verify.Tests/DateScrubberTests.cs b/src/Verify.Tests/DateScrubberTests.cs index 223b1903a..33f3449b4 100644 --- a/src/Verify.Tests/DateScrubberTests.cs +++ b/src/Verify.Tests/DateScrubberTests.cs @@ -19,6 +19,14 @@ public static void Init() public Task GetCultureDates() => Verify(DateFormatLengthCalculator.GetCultureLengthInfo(CultureInfo.InvariantCulture)); + static Dictionary emptyContext = []; + + static string Scrub(string value, Scrubber[] scrubbers, Counter counter) + { + var set = EngineScrubberSet.ForScrubbers([.. scrubbers]); + return ScrubEngine.Run(value, set, counter, emptyContext, applyDirectoryReplacements: false); + } + [Theory] [InlineData("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "no match")] [InlineData("aaaa", "no match short")] @@ -31,9 +39,8 @@ public Task GetCultureDates() => public async Task DateTimeOffsets(string value, string name) { using var counter = Counter.Start(); - var builder = new StringBuilder(value); - DateScrubber.ReplaceDateTimeOffsets(builder, "yyyy-MM-dd", counter, CultureInfo.InvariantCulture); - await Verify(builder) + var result = Scrub(value, DateMatchers.DateTimeOffsets("yyyy-MM-dd", CultureInfo.InvariantCulture), counter); + await Verify(result) .UseTextForParameters(name); } @@ -49,9 +56,8 @@ await Verify(builder) public async Task VariableLengthDateTimeOffsets(string value, string name) { using var counter = Counter.Start(); - var builder = new StringBuilder(value); - DateScrubber.ReplaceDateTimeOffsets(builder, "yyyy MMMM dd dddd", counter, CultureInfo.InvariantCulture); - await Verify(builder) + var result = Scrub(value, DateMatchers.DateTimeOffsets("yyyy MMMM dd dddd", CultureInfo.InvariantCulture), counter); + await Verify(result) .UseTextForParameters(name); } @@ -79,19 +85,43 @@ public Task NamedDateTimeOffsets(string value, string name) => public async Task DateTimes(string value, string name) { using var counter = Counter.Start(); - var builder = new StringBuilder(value); - DateScrubber.ReplaceDateTimes(builder, "yyyy-MM-dd", counter, CultureInfo.InvariantCulture); - await Verify(builder) + var result = Scrub(value, DateMatchers.DateTimes("yyyy-MM-dd", CultureInfo.InvariantCulture), counter); + await Verify(result) .UseTextForParameters(name); } + [Fact] + public void StandardFormat_AllCultures() + { + // Single char standard formats expand per culture; the digit prefilter + // analysis runs on the expanded pattern and must never suppress a real match + foreach (var culture in CultureInfo.GetCultures(CultureTypes.AllCultures)) + { + using var counter = Counter.Start(); + var dateTime = DateTime.Now; + var value = dateTime.ToString("d", culture); + var result = Scrub($"a {value} b", DateMatchers.DateTimes("d", culture), counter); + if (result == "a DateTime_1 b") + { + continue; + } + + throw new( + $""" + {culture.DisplayName} {culture.Name} + {result} + {value} + """); + } + } + [Fact] public void ReplaceDateTimes_AllCultures() { foreach (var culture in CultureInfo.GetCultures(CultureTypes.AllCultures)) { var format = "yyyy MMMM MMM MM dddd ddd dd d HH H mm m ss s fffff tt"; - var counter = Counter.Start(); + using var counter = Counter.Start(); var dateTime = DateTime.Now; var dateFormat = culture.DateTimeFormat; @@ -102,9 +132,7 @@ public void ReplaceDateTimes_AllCultures() } var value = dateTime.ToString(format, culture); - var builder = new StringBuilder(value); - DateScrubber.ReplaceDateTimes(builder, format, counter, culture); - var result = builder.ToString(); + var result = Scrub(value, DateMatchers.DateTimes(format, culture), counter); if (result == "DateTime_1") { continue; @@ -131,9 +159,8 @@ public void ReplaceDateTimes_AllCultures() public async Task VariableLengthDateTimes(string value, string name) { using var counter = Counter.Start(); - var builder = new StringBuilder(value); - DateScrubber.ReplaceDateTimes(builder, "yyyy MMMM dd dddd", counter, CultureInfo.InvariantCulture); - await Verify(builder) + var result = Scrub(value, DateMatchers.DateTimes("yyyy MMMM dd dddd", CultureInfo.InvariantCulture), counter); + await Verify(result) .UseTextForParameters(name); } @@ -162,9 +189,8 @@ public Task NamedDateTimes(string value, string name) => public async Task Dates(string value, string name) { using var counter = Counter.Start(); - var builder = new StringBuilder(value); - DateScrubber.ReplaceDates(builder, "yyyy-MM-dd", counter, CultureInfo.InvariantCulture); - await Verify(builder) + var result = Scrub(value, DateMatchers.Dates("yyyy-MM-dd", CultureInfo.InvariantCulture), counter); + await Verify(result) .UseTextForParameters(name); } @@ -180,9 +206,8 @@ await Verify(builder) public async Task VariableLengthDates(string value, string name) { using var counter = Counter.Start(); - var builder = new StringBuilder(value); - DateScrubber.ReplaceDates(builder, "yyyy MMMM dd dddd", counter, CultureInfo.InvariantCulture); - await Verify(builder) + var result = Scrub(value, DateMatchers.Dates("yyyy MMMM dd dddd", CultureInfo.InvariantCulture), counter); + await Verify(result) .UseTextForParameters(name); } diff --git a/src/Verify.Tests/GuidScrubberTests.cs b/src/Verify.Tests/GuidScrubberTests.cs index 531b80d04..3c179b8da 100644 --- a/src/Verify.Tests/GuidScrubberTests.cs +++ b/src/Verify.Tests/GuidScrubberTests.cs @@ -7,6 +7,14 @@ public class GuidScrubberTests #endregion + static Dictionary emptyContext = []; + + static string ReplaceGuids(string value, Counter counter) + { + var set = EngineScrubberSet.ForScrubbers([GuidMatcher.Instance]); + return ScrubEngine.Run(value, set, counter, emptyContext, applyDirectoryReplacements: false); + } + [Theory] [InlineData("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "no match")] [InlineData("aaaaaaaaaaaaaaaaaaaaaaaa", "no match short")] @@ -43,9 +51,8 @@ public class GuidScrubberTests public async Task Run(string guid, string name) { using var counter = Counter.Start(); - var builder = new StringBuilder(guid); - GuidScrubber.ReplaceGuids(builder, counter); - await Verify(builder) + var result = ReplaceGuids(guid, counter); + await Verify(result) .DontScrubGuids() .UseTextForParameters(name); } @@ -91,31 +98,11 @@ public Task NamedGuidTopLevelInstance() } [Fact] - public void MultipleChunks() - { - var builder = new StringBuilder(capacity: 8, maxCapacity: int.MaxValue); - builder.Append("[2e6bddf7-fcf7-4b09-bb6f-a7948e1eecf3]"); - builder.Append("[c2eeaf99-d5c4-4341-8543-4597c3fd40d9]"); - using var counter = Counter.Start(); - GuidScrubber.ReplaceGuids(builder, counter); - Assert.Equal("[Guid_1][Guid_2]", builder.ToString()); - } - - [Fact] - public void GuidSpanningThreeChunks() + public void Multiple() { - // Capacity 4 forces small chunks [4][4][rest]; the 36-char guid spans all - // three, and the 4-char middle chunk is shorter than the 35-char carryover. - // The carryover must accumulate across chunks or the prefix is dropped and - // the guid is never found (silent leak). - var guid = "173535ae-995b-4cc6-a74e-8cd4be57039c"; - var builder = new StringBuilder(capacity: 4); - builder.Append(guid[..4]); // chunk0 - builder.Append(guid[4..8]); // chunk1 (short middle chunk) - builder.Append(guid[8..]); // chunk2 using var counter = Counter.Start(); - GuidScrubber.ReplaceGuids(builder, counter); - Assert.Equal("Guid_1", builder.ToString()); + var result = ReplaceGuids("[2e6bddf7-fcf7-4b09-bb6f-a7948e1eecf3][c2eeaf99-d5c4-4341-8543-4597c3fd40d9]", counter); + Assert.Equal("[Guid_1][Guid_2]", result); } #region NamedGuidFluent @@ -161,4 +148,4 @@ public Task NamedGuidTopLevelFluent() [Fact] public Task NamedGuidTopLevelGlobal() => Verify(new Guid("c8eeaf99-d5c4-4341-8543-4597c3fd40c9")); -} \ No newline at end of file +} diff --git a/src/Verify.Tests/LinesScrubberTests.cs b/src/Verify.Tests/LinesScrubberTests.cs index 8bf274259..4aaa6f82a 100644 --- a/src/Verify.Tests/LinesScrubberTests.cs +++ b/src/Verify.Tests/LinesScrubberTests.cs @@ -1,4 +1,4 @@ -public class LinesScrubberTests +public class LinesScrubberTests { [Fact] public Task ScrubLinesContaining() @@ -78,317 +78,158 @@ public Task ScrubLinesContaining_case_sensitive() """); } - [Fact] - public void ReplaceLines_AllRemovedNoTrailingNewline() - { - // Single line, no trailing newline, and every line replaced with null. - // The trailing-newline trim must guard on the rebuilt builder length, not - // the original string length, otherwise Length -= 1 underflows. - var builder = new StringBuilder("single line"); - - builder.ReplaceLines(_ => null); - - Assert.Equal(string.Empty, builder.ToString()); - } + static Dictionary emptyContext = []; - [Fact] - public void ReplaceLines_ReplacesLines() + static string Run(string input, Scrubber scrubber) { - var builder = new StringBuilder("a\nb\nc"); - - builder.ReplaceLines(line => line == "b" ? "B" : line); - - Assert.Equal("a\nB\nc", builder.ToString()); + using var counter = Counter.Start(); + var set = EngineScrubberSet.ForScrubbers([scrubber]); + return ScrubEngine.Run(input, set, counter, emptyContext, applyDirectoryReplacements: false); } - [Fact] - public void FilterLines_RemovesSingleLine() - { - var builder = new StringBuilder("line1\nline2\nline3"); + static string Filter(string input, Func removeLine) => + Run(input, Scrubber.RemoveLines(removeLine)); - builder.FilterLines(line => line == "line2"); - - Assert.Equal("line1\nline3", builder.ToString()); - } + static string Replace(string input, Func replaceLine) => + Run(input, Scrubber.ReplaceLines(replaceLine)); [Fact] - public void FilterLines_RemovesMultipleLines() - { - var builder = new StringBuilder("keep1\nremove1\nkeep2\nremove2\nkeep3"); - - builder.FilterLines(line => line.StartsWith("remove")); - - Assert.Equal("keep1\nkeep2\nkeep3", builder.ToString()); - } + public void ReplaceLines_AllRemovedNoTrailingNewline() => + // Single line, no trailing newline, and every line replaced with null + Assert.Equal(string.Empty, Replace("single line", _ => null)); [Fact] - public void FilterLines_RemovesAllLines() - { - var builder = new StringBuilder("line1\nline2\nline3"); - - builder.FilterLines(_ => true); - - Assert.Equal(string.Empty, builder.ToString()); - } + public void ReplaceLines_ReplacesLines() => + Assert.Equal("a\nB\nc", Replace("a\nb\nc", _ => _ == "b" ? "B" : _)); [Fact] - public void FilterLines_RemovesNoLines() - { - var builder = new StringBuilder("line1\nline2\nline3"); - var original = builder.ToString(); - - builder.FilterLines(_ => false); - - Assert.Equal(original, builder.ToString()); - } + public void FilterLines_RemovesSingleLine() => + Assert.Equal("line1\nline3", Filter("line1\nline2\nline3", _ => _ == "line2")); [Fact] - public void FilterLines_EmptyStringBuilder() - { - var builder = new StringBuilder(); - - builder.FilterLines(_ => true); - - Assert.Equal(string.Empty, builder.ToString()); - } + public void FilterLines_RemovesMultipleLines() => + Assert.Equal("keep1\nkeep2\nkeep3", Filter("keep1\nremove1\nkeep2\nremove2\nkeep3", _ => _.StartsWith("remove"))); [Fact] - public void FilterLines_SingleLineNoNewline() - { - var builder = new StringBuilder("single line"); - - builder.FilterLines(_ => false); - - Assert.Equal("single line", builder.ToString()); - } + public void FilterLines_RemovesAllLines() => + Assert.Equal(string.Empty, Filter("line1\nline2\nline3", _ => true)); [Fact] - public void FilterLines_SingleLineWithNewline() - { - var builder = new StringBuilder("single line\n"); - - builder.FilterLines(_ => false); - - Assert.Equal("single line\n", builder.ToString()); - } + public void FilterLines_RemovesNoLines() => + Assert.Equal("line1\nline2\nline3", Filter("line1\nline2\nline3", _ => false)); [Fact] - public void FilterLines_PreservesTrailingNewline() - { - var builder = new StringBuilder("line1\nline2\n"); - - builder.FilterLines(line => line == "line2"); - - Assert.Equal("line1\n", builder.ToString()); - } + public void FilterLines_EmptyString() => + Assert.Equal(string.Empty, Filter(string.Empty, _ => true)); [Fact] - public void FilterLines_DoesNotAddTrailingNewlineWhenOriginalLacked() - { - var builder = new StringBuilder("line1\nline2"); - - builder.FilterLines(line => line == "line1"); - - Assert.Equal("line2", builder.ToString()); - } + public void FilterLines_SingleLineNoNewline() => + Assert.Equal("single line", Filter("single line", _ => false)); [Fact] - public void FilterLines_HandlesEmptyLines() - { - var builder = new StringBuilder("line1\n\nline3"); - - builder.FilterLines(string.IsNullOrEmpty); - - Assert.Equal("line1\nline3", builder.ToString()); - } + public void FilterLines_SingleLineWithNewline() => + Assert.Equal("single line\n", Filter("single line\n", _ => false)); [Fact] - public void FilterLines_KeepsEmptyLines() - { - var builder = new StringBuilder("line1\n\nline3"); - - builder.FilterLines(line => line == "line1"); - - Assert.Equal("\nline3", builder.ToString()); - } + public void FilterLines_PreservesTrailingNewline() => + Assert.Equal("line1\n", Filter("line1\nline2\n", _ => _ == "line2")); [Fact] - public void FilterLines_OnlyEmptyLines() - { - var builder = new StringBuilder("\n\n"); - - builder.FilterLines(_ => false); - - Assert.Equal("\n\n", builder.ToString()); - } + public void FilterLines_DoesNotAddTrailingNewlineWhenOriginalLacked() => + Assert.Equal("line2", Filter("line1\nline2", _ => _ == "line1")); [Fact] - public void FilterLines_RemovesFirstLine() - { - var builder = new StringBuilder("remove\nkeep1\nkeep2"); - - builder.FilterLines(line => line == "remove"); - - Assert.Equal("keep1\nkeep2", builder.ToString()); - } + public void FilterLines_HandlesEmptyLines() => + Assert.Equal("line1\nline3", Filter("line1\n\nline3", string.IsNullOrEmpty)); [Fact] - public void FilterLines_RemovesLastLine() - { - var builder = new StringBuilder("keep1\nkeep2\nremove"); - - builder.FilterLines(line => line == "remove"); - - Assert.Equal("keep1\nkeep2", builder.ToString()); - } + public void FilterLines_KeepsEmptyLines() => + Assert.Equal("\nline3", Filter("line1\n\nline3", _ => _ == "line1")); [Fact] - public void FilterLines_RemovesLastLineWithTrailingNewline() - { - var builder = new StringBuilder("keep1\nkeep2\nremove\n"); - - builder.FilterLines(line => line == "remove"); - - Assert.Equal("keep1\nkeep2\n", builder.ToString()); - } + public void FilterLines_OnlyEmptyLines() => + Assert.Equal("\n\n", Filter("\n\n", _ => false)); [Fact] - public void FilterLines_ComplexPredicate() - { - var builder = new StringBuilder("abc123\ndef456\nghi789\njkl012"); - - builder.FilterLines(line => line.Any(char.IsDigit) && line.Contains('4')); - - Assert.Equal("abc123\nghi789\njkl012", builder.ToString()); - } + public void FilterLines_RemovesFirstLine() => + Assert.Equal("keep1\nkeep2", Filter("remove\nkeep1\nkeep2", _ => _ == "remove")); [Fact] - public void FilterLines_WindowsLineEndings() - { - var builder = new StringBuilder("line1\r\nline2\r\nline3"); - - builder.FilterLines(line => line == "line2"); + public void FilterLines_RemovesLastLine() => + Assert.Equal("keep1\nkeep2", Filter("keep1\nkeep2\nremove", _ => _ == "remove")); - // Note: StringReader normalizes \r\n to \n - Assert.Equal("line1\nline3", builder.ToString()); - } + [Fact] + public void FilterLines_RemovesLastLineWithTrailingNewline() => + Assert.Equal("keep1\nkeep2\n", Filter("keep1\nkeep2\nremove\n", _ => _ == "remove")); [Fact] - public void FilterLines_MixedLineEndings() - { - var builder = new StringBuilder("line1\nline2\r\nline3"); + public void FilterLines_ComplexPredicate() => + Assert.Equal("abc123\nghi789\njkl012", Filter("abc123\ndef456\nghi789\njkl012", _ => _.Any(char.IsDigit) && _.Contains('4'))); - builder.FilterLines(line => line == "line2"); + [Fact] + public void FilterLines_WindowsLineEndings() => + // \r\n is normalized to \n + Assert.Equal("line1\nline3", Filter("line1\r\nline2\r\nline3", _ => _ == "line2")); - Assert.Equal("line1\nline3", builder.ToString()); - } + [Fact] + public void FilterLines_MixedLineEndings() => + Assert.Equal("line1\nline3", Filter("line1\nline2\r\nline3", _ => _ == "line2")); [Fact] public void FilterLines_LargeContent() { - var lines = Enumerable.Range(1, 1000).Select(i => $"line{i}"); - var builder = new StringBuilder(string.Join('\n', lines)); + var lines = Enumerable.Range(1, 1000).Select(_ => $"line{_}"); + var input = string.Join('\n', lines); - builder.FilterLines(line => int.Parse(line[4..]) % 2 == 0); + var result = Filter(input, _ => int.Parse(_[4..]) % 2 == 0); - var remaining = builder.ToString().Split('\n'); + var remaining = result.Split('\n'); Assert.Equal(500, remaining.Length); - Assert.All(remaining, line => Assert.True(int.Parse(line[4..]) % 2 == 1)); + Assert.All(remaining, _ => Assert.True(int.Parse(_[4..]) % 2 == 1)); } [Fact] - public void FilterLines_PreservesWhitespace() - { - var builder = new StringBuilder(" line1 \n\t line2\t\nline3 "); - - builder.FilterLines(line => line.Trim() == "line2"); - - Assert.Equal(" line1 \nline3 ", builder.ToString()); - } + public void FilterLines_PreservesWhitespace() => + Assert.Equal(" line1 \nline3 ", Filter(" line1 \n\t line2\t\nline3 ", _ => _.Trim() == "line2")); [Fact] - public void FilterLines_SingleLineRemoved() - { - var builder = new StringBuilder("remove this"); - - builder.FilterLines(_ => true); - - Assert.Equal(string.Empty, builder.ToString()); - } + public void FilterLines_SingleLineRemoved() => + Assert.Equal(string.Empty, Filter("remove this", _ => true)); [Fact] - public void FilterLines_AllButLastRemoved() - { - var builder = new StringBuilder("remove1\nremove2\nkeep\n"); - - builder.FilterLines(line => line.StartsWith("remove")); - - Assert.Equal("keep\n", builder.ToString()); - } + public void FilterLines_AllButLastRemoved() => + Assert.Equal("keep\n", Filter("remove1\nremove2\nkeep\n", _ => _.StartsWith("remove"))); [Fact] - public void FilterLines_MultipleConsecutiveRemoved() - { - var builder = new StringBuilder("keep1\nremove1\nremove2\nremove3\nkeep2"); - - builder.FilterLines(line => line.StartsWith("remove")); - - Assert.Equal("keep1\nkeep2", builder.ToString()); - } + public void FilterLines_MultipleConsecutiveRemoved() => + Assert.Equal("keep1\nkeep2", Filter("keep1\nremove1\nremove2\nremove3\nkeep2", _ => _.StartsWith("remove"))); [Fact] - public void FilterLines_LineSpansMultipleChunks() + public void FilterLines_VeryLongLine() { - // StringBuilder default chunk size is ~8000 chars - // Create a line that's definitely longer than a chunk var longLine = new string('a', 10_000); - var builder = new StringBuilder(); - builder.Append("keep1\n"); - builder.Append(longLine); - builder.Append("\nkeep2"); - - builder.FilterLines(line => line == longLine); - Assert.Equal("keep1\nkeep2", builder.ToString()); - } + var result = Filter($"keep1\n{longLine}\nkeep2", _ => _ == longLine); - [Fact] - public void FilterLines_NewlineAtChunkBoundary() - { - // Fill first chunk, then add newline at boundary - var chunkFiller = new string('x', 8000); - var builder = new StringBuilder(); - builder.Append(chunkFiller); - builder.Append('\n'); - builder.Append("line2"); - - builder.FilterLines(line => line == chunkFiller); - - Assert.Equal("line2", builder.ToString()); + Assert.Equal("keep1\nkeep2", result); } [Fact] - public void FilterLines_CrLfSplitAcrossChunks() + public void FilterLines_CrLfInLongContent() { - // Create scenario where \r is at end of one chunk and \n at start of next var lineContent = new string('y', 7999); - var builder = new StringBuilder(); - builder.Append(lineContent); - builder.Append("\r\n"); - builder.Append("nextline"); - builder.FilterLines(line => line == lineContent); + var result = Filter($"{lineContent}\r\nnextline", _ => _ == lineContent); - Assert.Equal("nextline", builder.ToString()); + Assert.Equal("nextline", result); } [Fact] - public void FilterLines_MultipleLinesSpanningChunks() + public void FilterLines_ManyLongLines() { var builder = new StringBuilder(); var lines = new List(); - // Create lines of varying lengths that will span chunk boundaries for (var i = 0; i < 20; i++) { var line = $"line{i}_" + new string((char) ('a' + i % 26), 1000 + i * 100); @@ -401,28 +242,24 @@ public void FilterLines_MultipleLinesSpanningChunks() } // Remove even-indexed lines - builder.FilterLines(line => lines.IndexOf(line) % 2 == 0); + var result = Filter(builder.ToString(), _ => lines.IndexOf(_) % 2 == 0); - var expected = string.Join('\n', lines.Where((_, i) => i % 2 == 1)); - Assert.Equal(expected, builder.ToString()); + var expected = string.Join('\n', lines.Where((_, index) => index % 2 == 1)); + Assert.Equal(expected, result); } [Fact] public void FilterLines_VeryLongLineKept() { var longLine = new string('z', 20_000); - var builder = new StringBuilder(); - builder.Append("remove\n"); - builder.Append(longLine); - builder.Append("\nremove"); - builder.FilterLines(line => line == "remove"); + var result = Filter($"remove\n{longLine}\nremove", _ => _ == "remove"); - Assert.Equal(longLine, builder.ToString()); + Assert.Equal(longLine, result); } [Fact] - public void FilterLines_MultipleChunksAllRemoved() + public void FilterLines_ManyLinesAllRemoved() { var builder = new StringBuilder(); for (var i = 0; i < 10; i++) @@ -431,58 +268,43 @@ public void FilterLines_MultipleChunksAllRemoved() builder.Append('\n'); } - builder.FilterLines(_ => true); - - Assert.Equal(string.Empty, builder.ToString()); + Assert.Equal(string.Empty, Filter(builder.ToString(), _ => true)); } [Fact] - public void FilterLines_MultipleChunksNoneRemoved() + public void FilterLines_ManyLinesNoneRemoved() { var builder = new StringBuilder(); for (var i = 0; i < 10; i++) { - var line = new string((char) ('a' + i), 2000); - builder.Append(line); + builder.Append((char) ('a' + i), 2000); builder.Append('\n'); } var original = builder.ToString(); - builder.FilterLines(_ => false); - - Assert.Equal(original, builder.ToString()); + Assert.Equal(original, Filter(original, _ => false)); } [Fact] - public void FilterLines_ChunkBoundaryInMiddleOfLine() + public void FilterLines_LongLinePreserved() { - // Specifically target chunk boundary within line content var prefix = new string('p', 7990); var suffix = new string('s', 100); var fullLine = prefix + suffix; - var builder = new StringBuilder(); - builder.Append("before\n"); - builder.Append(fullLine); - builder.Append("\nafter"); - - builder.FilterLines(line => line == "before"); + var result = Filter($"before\n{fullLine}\nafter", _ => _ == "before"); - Assert.Equal(fullLine + "\nafter", builder.ToString()); + Assert.Equal(fullLine + "\nafter", result); } [Fact] - public void FilterLines_EmptyLinesAroundChunkBoundaries() + public void FilterLines_EmptyLinesAfterLongLine() { var longLine = new string('x', 8000); - var builder = new StringBuilder(); - builder.Append(longLine); - builder.Append("\n\n\n"); - builder.Append("keep"); - builder.FilterLines(string.IsNullOrEmpty); + var result = Filter($"{longLine}\n\n\nkeep", string.IsNullOrEmpty); - Assert.Equal(longLine + "\nkeep", builder.ToString()); + Assert.Equal(longLine + "\nkeep", result); } -} \ No newline at end of file +} diff --git a/src/Verify.Tests/Serialization/SerializationTests.ScrubEngineMethods.verified.txt b/src/Verify.Tests/Serialization/SerializationTests.ScrubEngineMethods.verified.txt new file mode 100644 index 000000000..e03f786bd --- /dev/null +++ b/src/Verify.Tests/Serialization/SerializationTests.ScrubEngineMethods.verified.txt @@ -0,0 +1 @@ +{Value} {Id} xyz \ No newline at end of file diff --git a/src/Verify.Tests/Serialization/SerializationTests.cs b/src/Verify.Tests/Serialization/SerializationTests.cs index 8f3ba88fe..8a624fd50 100644 --- a/src/Verify.Tests/Serialization/SerializationTests.cs +++ b/src/Verify.Tests/Serialization/SerializationTests.cs @@ -2023,6 +2023,39 @@ public Task NewLineNotEscapedInProperty() => Property = "a\r\nb\\nc" }); + [Fact] + public Task ScrubEngineMethods() => + Verify("value id-123 abc") + .ScrubReplace("abc", "xyz") + .ScrubWindow( + minLength: 4, + maxLength: 6, + matcher: (window, _, _) => + { + if (window.Length == 6 && + window.StartsWith("id-")) + { + return "{Id}"; + } + + return null; + }) + .ScrubMatch( + (ReadOnlySpan segment, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + index = segment.IndexOf("value".AsSpan()); + if (index == -1) + { + length = 0; + replacement = null; + return false; + } + + length = "value".Length; + replacement = "{Value}"; + return true; + }); + // ReSharper disable once UnusedMember.Local static void List() { @@ -2069,6 +2102,31 @@ static void List() verifySettings.AddScrubber(_ => _.Remove(0, 100)); #endregion + + #region ScrubEngine + + verifySettings.ScrubReplace("abc", "xyz"); + + verifySettings.ScrubWindow( + minLength: 3, + maxLength: 10, + matcher: (window, _, _) => + { + if (window.StartsWith("id-")) + { + return "{Id}"; + } + + return null; + }); + + #endregion + + #region ScrubEngineExtension + + verifySettings.ScrubReplace("json", "abc", "xyz"); + + #endregion } [Fact] diff --git a/src/Verify.Tests/UserMachineScrubberChunkTests.cs b/src/Verify.Tests/UserMachineScrubberChunkTests.cs deleted file mode 100644 index 05ae11699..000000000 --- a/src/Verify.Tests/UserMachineScrubberChunkTests.cs +++ /dev/null @@ -1,38 +0,0 @@ -public class UserMachineScrubberChunkTests -{ - [Fact] - public void CrossChunkMatchEndingExactlyAtChunkBoundary() - { - // "ABCD" fills the capacity-4 chunk; the next append lands in a fresh - // 16-char chunk holding the remaining 16 chars of the 20-char match, so - // the match ends exactly at that chunk's boundary with a chunk after it. - // The trailing-char check must not read chunkSpan[chunkSpan.Length]. - var builder = new StringBuilder(capacity: 4); - builder.Append("ABCD"); - builder.Append("EFGHIJKLMNOPQRST"); - builder.Append('.'); - - UserMachineScrubber.PerformReplacements(builder, "ABCDEFGHIJKLMNOPQRST", "TheUserName"); - - Assert.Equal("TheUserName.", builder.ToString()); - } - - [Fact] - public void TokenSpanningThreeChunks() - { - // Capacity 4 forces small chunks [4][4][…]. The 10-char match spans all - // three, and the 4-char middle chunk is shorter than the match. The - // carryover must accumulate across chunks; otherwise the first chunk's - // prefix is dropped when the short middle chunk overwrites it, and the - // match is never found (silent leak). - var find = "ABCDEFGHIJ"; // 10 chars - var builder = new StringBuilder(capacity: 4); - builder.Append("ABCD"); // chunk0 = find[0..4] - builder.Append("EFGH"); // chunk1 = find[4..8] (short middle chunk) - builder.Append("IJ."); // chunk2 = find[8..10] + wrapper - - UserMachineScrubber.PerformReplacements(builder, find, "TheUserName"); - - Assert.Equal("TheUserName.", builder.ToString()); - } -} diff --git a/src/Verify.slnx b/src/Verify.slnx index 3e903a52e..1dd945c0f 100644 --- a/src/Verify.slnx +++ b/src/Verify.slnx @@ -13,6 +13,7 @@ + diff --git a/src/Verify/Extensions.cs b/src/Verify/Extensions.cs index 359de280f..32c26494d 100644 --- a/src/Verify/Extensions.cs +++ b/src/Verify/Extensions.cs @@ -149,63 +149,6 @@ public static string NameWithParent(this Type type) return attribute?.Configuration; } - public static char? FirstChar(this StringBuilder builder) - { - if (builder.Length > 0) - { - return builder[0]; - } - - return null; - } - - public static char? LastChar(this StringBuilder builder) - { - if (builder.Length > 0) - { - return builder[^1]; - } - - return null; - } - - public static void FilterLines(this StringBuilder input, Func removeLine) - { - var theString = input.ToString(); - using var reader = new StringReader(theString); - input.Clear(); - - while (reader.ReadLine() is { } line) - { - if (removeLine(line)) - { - continue; - } - - input.AppendLineN(line); - } - - var endsWithNewLine = theString.EndsWith('\n'); - if (input.Length > 0 && !endsWithNewLine) - { - input.Length -= 1; - } - } - - public static void RemoveEmptyLines(this StringBuilder builder) - { - builder.FilterLines(string.IsNullOrWhiteSpace); - if (builder.FirstChar() is '\n') - { - builder.Remove(0, 1); - } - - if (builder.LastChar() is '\n') - { - builder.Length--; - } - } - public static string Remove(this string value, string toRemove) => value.Replace(toRemove, ""); diff --git a/src/Verify/Serialization/CustomContractResolver_Dictionary.cs b/src/Verify/Serialization/CustomContractResolver_Dictionary.cs index cde18436a..06f58eada 100644 --- a/src/Verify/Serialization/CustomContractResolver_Dictionary.cs +++ b/src/Verify/Serialization/CustomContractResolver_Dictionary.cs @@ -100,7 +100,7 @@ static bool TryConvertDictionaryKey(VerifyJsonWriter writer, object original, [N return true; } - result = ApplyScrubbers.ApplyForPropertyValue(stringValue.AsSpan(), writer.settings, counter); + result = ApplyScrubbers.ApplyForPropertyValue(stringValue, writer.settings, counter); return true; } diff --git a/src/Verify/Serialization/Scrubbers/ApplyScrubbers.cs b/src/Verify/Serialization/Scrubbers/ApplyScrubbers.cs index 4dba9a4e0..ccd64e58f 100644 --- a/src/Verify/Serialization/Scrubbers/ApplyScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/ApplyScrubbers.cs @@ -1,5 +1,3 @@ -// ReSharper disable RedundantSuppressNullableWarningExpression - static class ApplyScrubbers { public static void ApplyForExtension(string extension, StringBuilder target, VerifySettings settings, Counter counter) @@ -10,6 +8,28 @@ public static void ApplyForExtension(string extension, StringBuilder target, Ver return; } + var set = EngineScrubberSet.ForExtension(settings, extension); + var source = target.ToString(); + + if (!HasLegacyForExtension(settings, extension)) + { + ScrubEngine.RunToBuilder(source, set, counter, settings.Context, applyDirectoryReplacements: true, target); + return; + } + + // Span scrubbers run first, then the legacy pass over the intermediate result. + // The engine normalizes newlines up front, so a legacy scrubber sees text + // that already uses '\n' and one matching a literal "\r\n" will not fire. + // Newlines are normalized again at the end because a legacy scrubber can + // inject '\r'. Path replacements are the only part held back until after the + // legacy pass, so those scrubbers still see raw paths. + var intermediate = ScrubEngine.Run(source, set, counter, settings.Context, applyDirectoryReplacements: false); + if (!ReferenceEquals(intermediate, source)) + { + target.Clear(); + target.Append(intermediate); + } + if (settings.InstanceScrubbers != null) { foreach (var scrubber in settings.InstanceScrubbers) @@ -45,40 +65,67 @@ public static void ApplyForExtension(string extension, StringBuilder target, Ver target.FixNewlines(); } - public static string ApplyForPropertyValue(CharSpan value, VerifySettings settings, Counter counter) + // Fast path: when nothing can change the value, skip all scrubbing work + // (this is the hottest loop in the library — one call per serialized string + // value). Returns the input span unchanged (zero alloc) when possible. + public static CharSpan ApplyForPropertyValue(CharSpan value, VerifySettings settings, Counter counter) { - // Fast path: when nothing can change the value, skip the StringBuilder - // round-trip and the directory-replacement scan (this is the hottest loop - // in the library — one call per serialized string value). - if (!value.Contains('\r')) + if (!settings.ScrubbersEnabled) { - if (!settings.ScrubbersEnabled) + if (!value.Contains('\r')) { - return value.ToString(); + return value; } - if (settings.InstanceScrubbers == null && - VerifierSettings.GlobalScrubbers.Count == 0 && - value.Length < DirectoryReplacements.ShortestFindLength) + return Scrubber.NormalizeNewlines(value.ToString()).AsSpan(); + } + + if (!HasLegacyForPropertyValue(settings)) + { + var set = EngineScrubberSet.ForPropertyValue(settings); + var minTrigger = Math.Min(set.MinTrigger, DirectoryReplacements.ShortestFindLength); + if (value.Length < minTrigger && + !value.Contains('\r')) { - return value.ToString(); + return value; } + + return ScrubEngine.Run(value.ToString(), set, counter, settings.Context, applyDirectoryReplacements: true).AsSpan(); } - var builder = new StringBuilder(value.Length); - builder.Append(value); - ApplyForPropertyValue(settings, counter, builder); - return builder.ToString(); + return ApplyWithLegacy(value.ToString(), settings, counter).AsSpan(); } - public static void ApplyForPropertyValue(VerifySettings settings, Counter counter, StringBuilder builder) + // String variant: returns the same string instance when nothing changed + public static string ApplyForPropertyValue(string value, VerifySettings settings, Counter counter) { if (!settings.ScrubbersEnabled) { - builder.FixNewlines(); - return; + return Scrubber.NormalizeNewlines(value); } + if (!HasLegacyForPropertyValue(settings)) + { + var set = EngineScrubberSet.ForPropertyValue(settings); + var minTrigger = Math.Min(set.MinTrigger, DirectoryReplacements.ShortestFindLength); + if (value.Length < minTrigger && + !value.Contains('\r')) + { + return value; + } + + return ScrubEngine.Run(value, set, counter, settings.Context, applyDirectoryReplacements: true); + } + + return ApplyWithLegacy(value, settings, counter); + } + + static string ApplyWithLegacy(string value, VerifySettings settings, Counter counter) + { + var set = EngineScrubberSet.ForPropertyValue(settings); + var intermediate = ScrubEngine.Run(value, set, counter, settings.Context, applyDirectoryReplacements: false); + + var builder = new StringBuilder(intermediate); if (settings.InstanceScrubbers != null) { foreach (var scrubber in settings.InstanceScrubbers) @@ -92,11 +139,26 @@ public static void ApplyForPropertyValue(VerifySettings settings, Counter counte scrubber(builder, counter, settings.Context); } - DirectoryReplacements.Replace(builder); - - builder.FixNewlines(); + // Path replacements and the newline fix both run on the engine, so the + // builder is materialized once. Doing them on the builder materialized it + // again inside the path scan, copying the whole value twice. + // The engine normalizes newlines before replacing paths rather than after, + // which cannot change a match: a path never contains '\r' or '\n', and both + // are non alphanumeric so the boundary and trailing separator rules see them + // the same either way. + return ScrubEngine.Run(builder.ToString(), EngineScrubberSet.Empty, counter, settings.Context, applyDirectoryReplacements: true); } - static string CleanPath(string directory) => - directory.TrimEnd('/', '\\'); -} \ No newline at end of file + static bool HasLegacyForExtension(VerifySettings settings, string extension) => + settings.InstanceScrubbers is { Count: > 0 } || + (settings.ExtensionMappedInstanceScrubbers != null && + settings.ExtensionMappedInstanceScrubbers.TryGetValue(extension, out var extensionInstance) && + extensionInstance.Count > 0) || + (VerifierSettings.ExtensionMappedGlobalScrubbers.TryGetValue(extension, out var extensionGlobal) && + extensionGlobal.Count > 0) || + VerifierSettings.GlobalScrubbers.Count > 0; + + static bool HasLegacyForPropertyValue(VerifySettings settings) => + settings.InstanceScrubbers is { Count: > 0 } || + VerifierSettings.GlobalScrubbers.Count > 0; +} diff --git a/src/Verify/Serialization/Scrubbers/DateMatchers.cs b/src/Verify/Serialization/Scrubbers/DateMatchers.cs new file mode 100644 index 000000000..6e694d86d --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/DateMatchers.cs @@ -0,0 +1,297 @@ +// The inline date scrubbers: window scrubbers whose bounds come from +// DateFormatLengthCalculator and whose matcher is a TryParseExact for the format. +// Formats ending in upper case fraction specifiers (.F to .FFFF) produce a second +// scrubber for the trimmed format, since those fractions render as empty when zero; +// the longer max of the untrimmed scrubber naturally orders it first. +// An explicit culture is resolved when the scrubber is created. When none is given +// the culture in effect at scrub time is used, resolved (and cached) per culture on +// each scrub, since the parse, the window bounds, and the anchor all depend on it. +static class DateMatchers +{ + // A probe date within the supported range of every calendar + // (DateTime.MaxValue is out of range for e.g. the UmAlQura calendar) + static DateTime probeDateTime = new(2000, 1, 1, 1, 1, 1, DateTimeKind.Utc); + + public static Scrubber[] DateTimes( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, + Culture? culture) + { + var resolvedCulture = culture ?? Culture.CurrentCulture; + try + { + probeDateTime.ToString(format, resolvedCulture); + } + catch (FormatException exception) + { + throw new($"Format '{format}' is not valid for DateTime.ToString(format, culture).", exception); + } + + return BuildForFormats( + format, + culture, + resolvedCulture, + static (format, culture) => (window, counter) => + { + if (DateTime.TryParseExact(window, format, culture, DateTimeStyles.None, out var date)) + { + return counter.Convert(date); + } + + return null; + }); + } + + public static Scrubber[] DateTimeOffsets( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, + Culture? culture) + { + var resolvedCulture = culture ?? Culture.CurrentCulture; + try + { + new DateTimeOffset(probeDateTime).ToString(format, resolvedCulture); + } + catch (FormatException exception) + { + throw new($"Format '{format}' is not valid for DateTimeOffset.ToString(format, culture).", exception); + } + + return BuildForFormats( + format, + culture, + resolvedCulture, + static (format, culture) => (window, counter) => + { + if (DateTimeOffset.TryParseExact(window, format, culture, DateTimeStyles.None, out var date)) + { + return counter.Convert(date); + } + + return null; + }); + } + +#if NET6_0_OR_GREATER + + public static Scrubber[] Dates( + [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, + Culture? culture) + { + var resolvedCulture = culture ?? Culture.CurrentCulture; + try + { + Date.FromDateTime(probeDateTime).ToString(format, resolvedCulture); + } + catch (FormatException exception) + { + throw new($"Format '{format}' is not valid for DateOnly.ToString(format, culture).", exception); + } + + return BuildForFormats( + format, + culture, + resolvedCulture, + static (format, culture) => (window, counter) => + { + if (Date.TryParseExact(window, format, culture, DateTimeStyles.None, out var date)) + { + return counter.Convert(date); + } + + return null; + }); + } + +#endif + + delegate string? ParseWindow(CharSpan window, Counter counter); + + // Builds the parse for one format and culture pair + delegate ParseWindow ParseFactory(string format, Culture culture); + + static Scrubber[] BuildForFormats( + string format, + Culture? culture, + Culture registrationCulture, + ParseFactory parseFactory) + { + if (TryGetFormatWithUpperMillisecondsTrimmed(format, out var trimmedFormat)) + { + return + [ + ForCulture(format, culture, registrationCulture, parseFactory), + ForCulture(trimmedFormat, culture, registrationCulture, parseFactory) + ]; + } + + return [ForCulture(format, culture, registrationCulture, parseFactory)]; + } + + static Scrubber ForCulture( + string format, + Culture? culture, + Culture registrationCulture, + ParseFactory parseFactory) + { + if (culture != null) + { + return Single(format, culture, parseFactory(format, culture)); + } + + // No culture was supplied, so each scrub uses the culture in effect at that + // point. Building a scrubber reads the format lengths and expands the + // pattern, so the result is cached per culture. + ConcurrentDictionary cache = new(); + + Scrubber ForCurrentCulture() + { + var current = Culture.CurrentCulture; + if (cache.TryGetValue(current, out var existing)) + { + return existing; + } + + return cache.GetOrAdd(current, Single(format, current, parseFactory(format, current))); + } + + // The registration culture instance supplies the bounds used for ordering + // and the length fast path; the resolver supplies the one actually run + return Single( + format, + registrationCulture, + parseFactory(format, registrationCulture), + ForCurrentCulture); + } + + static Scrubber Single(string format, Culture culture, ParseWindow parse, Func? resolver = null) + { + var (max, min) = DateFormatLengthCalculator.GetLength(format, culture); + var digitPrefilter = HasDigitPrefilter(format, culture); + + string? Match(CharSpan window, Counter counter, IReadOnlyDictionary context) => + parse(window, counter); + + if (digitPrefilter) + { + // The engine anchor jumps between digits, so no-match text is scanned + // vectorized instead of per position + return Scrubber.GatedWindow( + Math.Max(1, min), + max, + Match, + static counter => counter.ScrubDateTimes, + anchor: WindowAnchor.Digit, + resolver: resolver); + } + + return Scrubber.GatedWindow( + Math.Max(1, min), + max, + Match, + static counter => counter.ScrubDateTimes, + resolver: resolver); + } + + // Probe dates for the digit prefilter: a zero fraction and a non zero one, + // single and double digit components, AM and PM. All sit within the supported + // range of every calendar. + static DateTime[] prefilterProbes = + [ + new(2000, 1, 1, 1, 1, 1, DateTimeKind.Utc), + new(2000, 12, 25, 23, 59, 59, 999, DateTimeKind.Utc), + new(2001, 6, 15, 11, 30, 5, DateTimeKind.Utc) + ]; + + // True when the format renders an ASCII digit first, letting the engine jump + // between digit positions instead of probing every one. The format grammar is + // only a necessary condition, so it is confirmed by rendering: that excludes + // tokens which can render empty (upper case F with a zero fraction) and + // calendars that render numbers as letters (for example Hebrew). + static bool HasDigitPrefilter(string format, Culture culture) + { + // Single char standard formats expand to the culture's pattern, so the + // grammar check runs on what will actually render + if (!StartsWithNumericToken(culture.DateTimeFormat.ExpandFormat(format))) + { + return false; + } + + foreach (var probe in prefilterProbes) + { + string rendered; + try + { + rendered = probe.ToString(format, culture); + } + catch (FormatException) + { + return false; + } + + if (rendered.Length == 0 || + !IsAsciiDigit(rendered[0])) + { + return false; + } + } + + return true; + } + + static bool IsAsciiDigit(char value) => + value is >= '0' and <= '9'; + + // True when the (expanded) format starts with a token that renders a number. + // Formats starting with a name, era, or literal token are not prefiltered. + // Upper case F is excluded since it renders nothing when the fraction is zero. + static bool StartsWithNumericToken(string format) + { + if (format.Length < 2) + { + return false; + } + + var first = format[0]; + switch (first) + { + case 'y' or 'H' or 'h' or 'm' or 's' or 'f': + return true; + case 'd' or 'M': + // 1 or 2 repeats render digits; 3+ render names + return format.Length < 3 || + format[1] != first || + format[2] != first; + default: + return false; + } + } + + static bool TryGetFormatWithUpperMillisecondsTrimmed(string format, [NotNullWhen(true)] out string? trimmedFormat) + { + if (format.EndsWith(".FFFF", StringComparison.Ordinal)) + { + trimmedFormat = format[..^5]; + return true; + } + + if (format.EndsWith(".FFF", StringComparison.Ordinal)) + { + trimmedFormat = format[..^4]; + return true; + } + + if (format.EndsWith(".FF", StringComparison.Ordinal)) + { + trimmedFormat = format[..^3]; + return true; + } + + if (format.EndsWith(".F", StringComparison.Ordinal)) + { + trimmedFormat = format[..^2]; + return true; + } + + trimmedFormat = null; + return false; + } +} diff --git a/src/Verify/Serialization/Scrubbers/DirectoryReplacements.cs b/src/Verify/Serialization/Scrubbers/DirectoryReplacements.cs index ca22140ae..7c5a19dcc 100644 --- a/src/Verify/Serialization/Scrubbers/DirectoryReplacements.cs +++ b/src/Verify/Serialization/Scrubbers/DirectoryReplacements.cs @@ -2,15 +2,144 @@ static partial class DirectoryReplacements { - static List items = []; - static int shortestFindLength = int.MaxValue; + public readonly struct Pair + { + public Pair(string find, string replace) + { +#if DEBUG + if (find.Contains('\\')) + { + throw new("Slashes should be sanitized"); + } +#endif + Find = find; + Replace = replace; + } + + public string Find { get; } + public string Replace { get; } + } + + // The anchors and the shortest find are derived from the pairs, so the three + // travel together rather than as separate statics a scan has to re-read. + // Assigned once, from AssignTargetAssembly. + internal sealed class Snapshot(List pairs, int shortestFindLength, string anchors) + { + public readonly List Pairs = pairs; + + public readonly int ShortestFindLength = shortestFindLength; + + // The distinct first chars of the Finds, so scans can skip to candidate + // positions with a vectorized IndexOfAny instead of probing every position + public readonly string Anchors = anchors; + } + + static Snapshot current = new([], int.MaxValue, ""); + + internal static Snapshot Current => current; // Length of the shortest Find, so callers can cheaply skip values that are // too short to contain any replacement. int.MaxValue when there are none. - public static int ShortestFindLength => shortestFindLength; + public static int ShortestFindLength => current.ShortestFindLength; + + internal static string BuildAnchors(List pairs) + { + var anchors = new List(); + foreach (var pair in pairs) + { + var first = pair.Find[0]; + if (!anchors.Contains(first)) + { + anchors.Add(first); + } + + // '/' and '\' are equivalent during matching + if (first == '/' && + !anchors.Contains('\\')) + { + anchors.Add('\\'); + } + } + + return new([.. anchors]); + } + + public static void Replace(StringBuilder builder) + { + var snapshot = current; + Replace(builder, snapshot.Pairs, snapshot.Anchors); + } + + public static void Replace(StringBuilder builder, List pairs) => + Replace(builder, pairs, BuildAnchors(pairs)); + + // Legacy pass entry point: materialize once, scan with the span matcher, apply + // matches position-descending so earlier indexes stay valid. + static void Replace(StringBuilder builder, List pairs, string anchors) + { +#if DEBUG + var finds = pairs.Select(_ => _.Find).ToList(); + if (!finds.OrderByDescending(_ => _.Length).SequenceEqual(finds)) + { + throw new("Pairs should be ordered"); + } + + if (finds.Count != finds.Distinct().Count()) + { + throw new("Find should be distinct"); + } +#endif + if (pairs.Count == 0 || + builder.Length == 0) + { + return; + } + + // pairs are ordered by length desc, so the last is the shortest. If the + // builder is shorter than that, no pair can match. + var shortest = pairs[^1].Find.Length; + if (builder.Length < shortest) + { + return; + } + + var span = builder.ToString().AsSpan(); + List<(int Index, int Length, string Value)>? matches = null; + for (var position = 0; position + shortest <= span.Length; position++) + { + var skip = span[position..].IndexOfAny(anchors.AsSpan()); + if (skip < 0) + { + break; + } + + position += skip; + if (position + shortest > span.Length) + { + break; + } + + if (!TryMatchAt(span, position, null, null, pairs, out var matchLength, out var replacement)) + { + continue; + } + + matches ??= []; + matches.Add((position, matchLength, replacement)); + position += matchLength - 1; + } + + if (matches == null) + { + return; + } - public static void Replace(StringBuilder builder) => - Replace(builder, items); + for (var index = matches.Count - 1; index >= 0; index--) + { + var match = matches[index]; + builder.Overwrite(match.Value, match.Index, match.Length); + } + } public static void UseAssembly(string? solutionDir, string projectDir) { @@ -48,10 +177,13 @@ public static void UseAssembly(string? solutionDir, string projectDir) } AddProjectAndSolutionReplacements(solutionDir, projectDir, values); - items = values + var ordered = values .OrderByDescending(_ => _.Find.Length) .ToList(); - shortestFindLength = items.Count == 0 ? int.MaxValue : items[^1].Find.Length; + current = new( + ordered, + ordered.Count == 0 ? int.MaxValue : ordered[^1].Find.Length, + BuildAnchors(ordered)); } static void AddProjectAndSolutionReplacements(string? solutionDir, string projectDir, List replacements) diff --git a/src/Verify/Serialization/Scrubbers/DirectoryReplacements_Span.cs b/src/Verify/Serialization/Scrubbers/DirectoryReplacements_Span.cs new file mode 100644 index 000000000..dcaf94a59 --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/DirectoryReplacements_Span.cs @@ -0,0 +1,105 @@ +// The span based path matcher shared by the engine pass and the legacy StringBuilder pass. +// Matching rules: '/' and '\' are equivalent (Find is pre-sanitized to '/'); the char before a match +// must not be a letter or digit; a trailing letter or digit invalidates the match; a trailing '/' or +// '\' within the current segment is greedily absorbed into the match. Pairs are ordered longest +// first, so at a given position the most specific path wins. +static partial class DirectoryReplacements +{ + // Attempts to match any pair at the given position within the span. + // beforeSegment/afterSegment are the neighbor chars when the position touches the + // segment edge (null at the document edge). Greedy trailing separator absorption + // never crosses the segment boundary. + internal static bool TryMatchAt( + CharSpan span, + int position, + char? beforeSegment, + char? afterSegment, + List pairs, + out int matchLength, + out string replacement) + { + matchLength = 0; + replacement = string.Empty; + + var preceding = position > 0 ? span[position - 1] : beforeSegment; + if (preceding is { } precedingChar && + char.IsLetterOrDigit(precedingChar)) + { + return false; + } + + foreach (var pair in pairs) + { + var find = pair.Find; + var end = position + find.Length; + if (end > span.Length) + { + continue; + } + + if (!IsPathMatch(span.Slice(position, find.Length), find)) + { + continue; + } + + if (end < span.Length) + { + var trailing = span[end]; + if (char.IsLetterOrDigit(trailing)) + { + continue; + } + + matchLength = find.Length; + + // Greedy: include a trailing separator + if (trailing is '/' or '\\') + { + matchLength++; + } + } + else + { + if (afterSegment is { } after && + char.IsLetterOrDigit(after)) + { + continue; + } + + matchLength = find.Length; + } + + replacement = pair.Replace; + return true; + } + + return false; + } + + // Treat '/' and '\' as equivalent. Find never contains '\'. + static bool IsPathMatch(CharSpan candidate, string find) + { + for (var index = 0; index < find.Length; index++) + { + var candidateChar = candidate[index]; + var findChar = find[index]; + + if (candidateChar is '/' or '\\') + { + if (findChar != '/') + { + return false; + } + + continue; + } + + if (candidateChar != findChar) + { + return false; + } + } + + return true; + } +} diff --git a/src/Verify/Serialization/Scrubbers/DirectoryReplacements_StringBuilder.cs b/src/Verify/Serialization/Scrubbers/DirectoryReplacements_StringBuilder.cs deleted file mode 100644 index 653eca376..000000000 --- a/src/Verify/Serialization/Scrubbers/DirectoryReplacements_StringBuilder.cs +++ /dev/null @@ -1,354 +0,0 @@ -static partial class DirectoryReplacements -{ - public readonly struct Pair - { - public Pair(string find, string replace) - { -#if DEBUG - if (find.Contains('\\')) - { - throw new("Slashes should be sanitized"); - } -#endif - Find = find; - Replace = replace; - } - - public string Find { get; } - public string Replace { get; } - } - - public static void Replace(StringBuilder builder, List paths) - { -#if DEBUG - var finds = paths.Select(_ => _.Find).ToList(); - if (!finds.OrderByDescending(_ => _.Length).SequenceEqual(finds)) - { - throw new("Pairs should be ordered"); - } - - if (finds.Count != finds.Distinct().Count()) - { - throw new("Find should be distinct"); - } -#endif - if (builder.Length == 0) - { - return; - } - - var matches = FindMatches(builder, paths); - - // Sort by position descending. In-place to avoid LINQ allocation - matches.Sort((a, b) => b.Index.CompareTo(a.Index)); - - // Apply matches - foreach (var match in matches) - { - builder.Overwrite(match.Value, match.Index, match.Length); - } - } - - static List FindMatches(StringBuilder builder, List pairs) - { - if (pairs.Count == 0) - { - return []; - } - - // pairs are ordered by length desc, so the last is the shortest. If the - // builder is shorter than that, no pair can match. - if (builder.Length < pairs[^1].Find.Length) - { - return []; - } - - var matches = new List(); - // Track matched positions - var matchedRanges = new List<(int Start, int End)>(); - var absolutePosition = 0; - - // pairs are ordered by length. so max length is the first one - var maxLength = pairs[0].Find.Length; - var carryoverSize = maxLength - 1; - - Span carryoverBuffer = stackalloc char[carryoverSize]; - Span combinedBuffer = stackalloc char[maxLength * 2]; - var carryoverLength = 0; - var previousChunkAbsoluteEnd = 0; - - foreach (var chunk in builder.GetChunks()) - { - var chunkSpan = chunk.Span; - - // Check for matches spanning from previous chunk to current chunk - if (carryoverLength > 0) - { - for (var carryoverIndex = 0; carryoverIndex < carryoverLength; carryoverIndex++) - { - foreach (var pair in pairs) - { - var remainingInCarryover = carryoverLength - carryoverIndex; - var neededFromCurrent = pair.Find.Length - remainingInCarryover; - - if (neededFromCurrent <= 0 || - neededFromCurrent > chunkSpan.Length) - { - continue; - } - - var combinedLength = remainingInCarryover + neededFromCurrent; - carryoverBuffer.Slice(carryoverIndex, remainingInCarryover).CopyTo(combinedBuffer); - chunkSpan[..neededFromCurrent].CopyTo(combinedBuffer[remainingInCarryover..]); - - var startPosition = previousChunkAbsoluteEnd - carryoverLength + carryoverIndex; - - // Check if this position overlaps with existing match - if (OverlapsExistingMatch(startPosition, pair.Find.Length, matchedRanges)) - { - continue; - } - - if (!TryMatchAtCrossChunk( - builder, - combinedBuffer[..combinedLength], - chunkSpan, - startPosition, - neededFromCurrent, - pair.Find, - out var matchLength)) - { - continue; - } - - matches.Add(new(startPosition, matchLength, pair.Replace)); - matchedRanges.Add((startPosition, startPosition + matchLength)); - // Found a match at this position, skip other pairs - break; - } - } - } - - // Process matches entirely within this chunk - for (var chunkIndex = 0; chunkIndex < chunk.Length; chunkIndex++) - { - var absoluteIndex = absolutePosition + chunkIndex; - - // Skip if already matched - if (IsPositionMatched(absoluteIndex, matchedRanges)) - { - continue; - } - - foreach (var pair in pairs) - { - // Check if we have enough characters left in this chunk - if (chunkIndex + pair.Find.Length > chunk.Length) - { - continue; - } - - // Check if this would overlap with existing match - if (OverlapsExistingMatch(absoluteIndex, pair.Find.Length, matchedRanges)) - { - continue; - } - - // Try to match at this position - if (!TryMatchAt(chunk, chunkIndex, pair.Find, out var matchLength)) - { - continue; - } - - matches.Add(new(absoluteIndex, matchLength, pair.Replace)); - matchedRanges.Add((absoluteIndex, absoluteIndex + matchLength)); - // Skip past this match - chunkIndex += matchLength - 1; - // Found a match, skip other pairs at this position - break; - } - } - - // Roll the carryover forward: keep the last carryoverSize chars of - // everything seen so far. Rebuilding it from the current chunk alone - // drops the prefix when a chunk is shorter than carryoverSize, so a - // path spanning three or more chunks would never be found. - if (chunk.Length >= carryoverSize) - { - chunkSpan.Slice(chunk.Length - carryoverSize, carryoverSize).CopyTo(carryoverBuffer); - carryoverLength = carryoverSize; - } - else - { - var keep = Math.Min(carryoverLength, carryoverSize - chunk.Length); - carryoverBuffer.Slice(carryoverLength - keep, keep).CopyTo(carryoverBuffer); - chunkSpan.CopyTo(carryoverBuffer[keep..]); - carryoverLength = keep + chunk.Length; - } - - previousChunkAbsoluteEnd = absolutePosition + chunk.Length; - absolutePosition += chunk.Length; - } - - return matches; - } - - static bool IsPositionMatched(int position, List<(int Start, int End)> matchedRanges) - { - foreach (var (start, end) in matchedRanges) - { - if (position >= start && position < end) - { - return true; - } - } - - return false; - } - - static bool OverlapsExistingMatch(int start, int length, List<(int Start, int End)> matchedRanges) - { - var end = start + length; - foreach (var range in matchedRanges) - { - // Check if ranges overlap - if (start < range.End && end > range.Start) - { - return true; - } - } - - return false; - } - - static bool TryMatchAtCrossChunk( - StringBuilder builder, - CharSpan combinedSpan, - CharSpan currentChunkSpan, - int absoluteStartPosition, - int neededFromCurrent, - string find, - out int matchLength) - { - matchLength = 0; - - // Check preceding character - if (absoluteStartPosition > 0) - { - var preceding = builder[absoluteStartPosition - 1]; - if (char.IsLetterOrDigit(preceding)) - { - return false; - } - } - - // Check if the path matches - if (!IsPathMatchAt(combinedSpan, 0, find)) - { - return false; - } - - matchLength = find.Length; - - // Check trailing character (it's in the current chunk) - if (neededFromCurrent < currentChunkSpan.Length) - { - var trailing = currentChunkSpan[neededFromCurrent]; - - // Invalid if trailing is letter or digit - if (char.IsLetterOrDigit(trailing)) - { - return false; - } - - // Greedy: include trailing separator - if (trailing is '/' or '\\') - { - matchLength++; - } - } - - return true; - } - - static bool TryMatchAt(ReadOnlyMemory chunk, int chunkPos, string find, out int matchLength) - { - var span = chunk.Span; - matchLength = 0; - - // Check preceding character - if (chunkPos > 0) - { - var preceding = span[chunkPos - 1]; - if (char.IsLetterOrDigit(preceding)) - { - return false; - } - } - - // Check if the path matches - if (!IsPathMatchAt(span, chunkPos, find)) - { - return false; - } - - // Check trailing character - matchLength = find.Length; - var trailingPos = chunkPos + find.Length; - - if (trailingPos >= span.Length) - { - return true; - } - - var trailing = span[trailingPos]; - - // Invalid if trailing is letter or digit - if (char.IsLetterOrDigit(trailing)) - { - return false; - } - - // Greedy: include trailing separator - if (trailing is '/' or '\\') - { - matchLength++; - } - - return true; - } - - static bool IsPathMatchAt(CharSpan chunk, int chunkPos, string find) - { - for (var i = 0; i < find.Length; i++) - { - var chunkChar = chunk[chunkPos + i]; - var findChar = find[i]; - - // Treat / and \ as equivalent - if (chunkChar is '/' or '\\') - { - if (findChar != '/') - { - return false; - } - - continue; - } - - if (chunkChar != findChar) - { - return false; - } - } - - return true; - } - - readonly struct Match(int index, int length, string value) - { - public readonly int Index = index; - public readonly int Length = length; - public readonly string Value = value; - } -} \ No newline at end of file diff --git a/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs b/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs new file mode 100644 index 000000000..041de7b4d --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/EngineScrubberSet.cs @@ -0,0 +1,212 @@ +// The merged, ordered view of all registered span scrubbers that apply to a single scrub operation. +// Levels merge in priority order: instance, extension-mapped instance, extension-mapped global, global. +// Inline scrubbers are ordered: unknown max length first, then descending max length, ties broken by +// level then registration order (stable via the collection index). +sealed class EngineScrubberSet +{ + public Scrubber[] LineDrops { get; } + public Scrubber[] LineTransforms { get; } + public Scrubber[] Inline { get; } + + // Values shorter than this cannot be changed by any scrubber in the set. + // 0 disables the fast path. int.MaxValue when the set is empty. + public int MinTrigger { get; } + + // Replicates RemoveEmptyLines trimming one leading and one trailing newline + public bool TrimOuterEmptyLines { get; } + + public bool HasLinePhase => LineDrops.Length > 0 || LineTransforms.Length > 0; + + static EngineScrubberSet empty = new([], [], []); + + // A set with no scrubbers, for a pass that only needs the engine's pinned + // trailing work (path replacements and newline normalization) + public static EngineScrubberSet Empty => empty; + + EngineScrubberSet(Scrubber[] lineDrops, Scrubber[] lineTransforms, Scrubber[] inline) + { + LineDrops = lineDrops; + LineTransforms = lineTransforms; + Inline = inline; + + var minTrigger = int.MaxValue; + foreach (var scrubber in lineDrops) + { + if (scrubber.Kind == ScrubberKind.LineDropEmpty) + { + TrimOuterEmptyLines = true; + } + + minTrigger = Math.Min(minTrigger, MinLengthFor(scrubber)); + } + + foreach (var scrubber in lineTransforms) + { + minTrigger = Math.Min(minTrigger, MinLengthFor(scrubber)); + } + + foreach (var scrubber in inline) + { + minTrigger = Math.Min(minTrigger, MinLengthFor(scrubber)); + } + + MinTrigger = minTrigger; + } + + static int MinLengthFor(Scrubber scrubber) => + scrubber.Kind switch + { + // A whitespace-only line needs at least one char to be droppable. + // Removing empty lines from an empty string is the identity. + ScrubberKind.LineDropEmpty => 1, + // Predicates and transforms can act on empty lines + ScrubberKind.LineDropSpan or + ScrubberKind.LineDropString or + ScrubberKind.LineTransformSpan or + ScrubberKind.LineTransformString => 0, + // Replace, Window, LineDropNeedles are at least 1. Match may be 0 (unknown) + _ => scrubber.MinLength + }; + + static ConcurrentDictionary globalExtensionCache = new(StringComparer.Ordinal); + static EngineScrubberSet? globalOnlyCache; + + public static void InvalidateGlobalCache() + { + globalExtensionCache.Clear(); + globalOnlyCache = null; + } + + // Whole-file pass: all four levels + public static EngineScrubberSet ForExtension(VerifySettings settings, string extension) + { + var instance = settings.InstanceSpanScrubbers; + List? extensionInstance = null; + settings.ExtensionMappedInstanceSpanScrubbers?.TryGetValue(extension, out extensionInstance); + + if (IsNullOrEmpty(instance) && + IsNullOrEmpty(extensionInstance)) + { + return globalExtensionCache.GetOrAdd( + extension, + static ext => + { + VerifierSettings.ExtensionMappedGlobalSpanScrubbers.TryGetValue(ext, out var extensionGlobal); + return Build(null, null, extensionGlobal, VerifierSettings.GlobalSpanScrubbers); + }); + } + + VerifierSettings.ExtensionMappedGlobalSpanScrubbers.TryGetValue(extension, out var extensionGlobalScrubbers); + return Build(instance, extensionInstance, extensionGlobalScrubbers, VerifierSettings.GlobalSpanScrubbers); + } + + // Property-value pass: instance and global only. Extension-mapped scrubbers + // are excluded, matching the legacy pipeline. The merged set is cached on the + // settings instance since this runs once per serialized string value. + public static EngineScrubberSet ForPropertyValue(VerifySettings settings) + { + var instance = settings.InstanceSpanScrubbers; + if (IsNullOrEmpty(instance)) + { + return globalOnlyCache ??= Build(null, null, null, VerifierSettings.GlobalSpanScrubbers); + } + + return settings.PropertyValueSetCache ??= Build(instance, null, null, VerifierSettings.GlobalSpanScrubbers); + } + + static bool IsNullOrEmpty(List? list) => + list == null || + list.Count == 0; + + // Test hook: a set over explicit scrubbers, bypassing the registration levels + internal static EngineScrubberSet ForScrubbers(List scrubbers) => + Build(scrubbers, null, null, []); + + static EngineScrubberSet Build( + List? instance, + List? extensionInstance, + List? extensionGlobal, + List global) + { + var count = (instance?.Count ?? 0) + + (extensionInstance?.Count ?? 0) + + (extensionGlobal?.Count ?? 0) + + global.Count; + if (count == 0) + { + return empty; + } + + var lineDrops = new List(); + var lineTransforms = new List(); + var inline = new List<(Scrubber Scrubber, int Order)>(); + var order = 0; + + void Collect(List? scrubbers) + { + if (scrubbers == null) + { + return; + } + + foreach (var scrubber in scrubbers) + { + if (scrubber.IsLineDrop) + { + lineDrops.Add(scrubber); + } + else if (scrubber.IsLineTransform) + { + lineTransforms.Add(scrubber); + } + else + { + inline.Add((scrubber, order)); + } + + order++; + } + } + + Collect(instance); + Collect(extensionInstance); + Collect(extensionGlobal); + Collect(global); + + inline.Sort((left, right) => + { + var leftMax = left.Scrubber.MaxLength; + var rightMax = right.Scrubber.MaxLength; + + // Unknown max length runs first + if (leftMax.HasValue != rightMax.HasValue) + { + return leftMax.HasValue ? 1 : -1; + } + + if (leftMax.HasValue) + { + // Longest max first + var compare = rightMax!.Value.CompareTo(leftMax.Value); + if (compare != 0) + { + return compare; + } + } + + // Level then registration order + return left.Order.CompareTo(right.Order); + }); + + var inlineArray = new Scrubber[inline.Count]; + for (var index = 0; index < inline.Count; index++) + { + inlineArray[index] = inline[index].Scrubber; + } + + return new( + [.. lineDrops], + [.. lineTransforms], + inlineArray); + } +} diff --git a/src/Verify/Serialization/Scrubbers/GuidMatcher.cs b/src/Verify/Serialization/Scrubbers/GuidMatcher.cs new file mode 100644 index 000000000..a5f9b2c9a --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/GuidMatcher.cs @@ -0,0 +1,34 @@ +// The inline guid scrubber: a fixed 36 char window over the canonical "D" format. +// The engine anchor jumps between '-' chars at offset 8 (the first dash of the "D" +// format), so no-match text is scanned vectorized instead of per position. +static class GuidMatcher +{ + public static readonly Scrubber Instance = Scrubber.GatedWindow( + 36, + 36, + Match, + static counter => counter.ScrubGuids, + requireWordBoundary: true, + anchor: WindowAnchor.Char, + anchorChar: '-', + anchorOffset: 8); + + static string? Match(CharSpan window, Counter counter, IReadOnlyDictionary context) + { + // Cheap prefilter: the "D" format has dashes at fixed offsets + if (window[8] != '-' || + window[13] != '-' || + window[18] != '-' || + window[23] != '-') + { + return null; + } + + if (!Guid.TryParseExact(window, "D", out var guid)) + { + return null; + } + + return counter.Convert(guid); + } +} diff --git a/src/Verify/Serialization/Scrubbers/LineResult.cs b/src/Verify/Serialization/Scrubbers/LineResult.cs new file mode 100644 index 000000000..ec6d8d95b --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/LineResult.cs @@ -0,0 +1,39 @@ +namespace VerifyTests; + +/// +/// The result of processing a line via . +/// +public readonly struct LineResult +{ + internal const byte KeepKind = 0; + internal const byte RemoveKind = 1; + internal const byte ReplaceKind = 2; + + internal byte Kind { get; } + internal string? Text { get; } + + LineResult(byte kind, string? text) + { + Kind = kind; + Text = text; + } + + /// + /// Keep the line unchanged. + /// + public static LineResult Keep { get; } = new(KeepKind, null); + + /// + /// Remove the line. + /// + public static LineResult Remove { get; } = new(RemoveKind, null); + + /// + /// Replace the line with . + /// + public static LineResult Replace(string line) + { + Ensure.NotNull(line); + return new(ReplaceKind, Scrubber.NormalizeNewlines(line)); + } +} diff --git a/src/Verify/Serialization/Scrubbers/ScrubEngine.cs b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs new file mode 100644 index 000000000..5bc31e2d9 --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/ScrubEngine.cs @@ -0,0 +1,729 @@ +// The span based scrub engine. Operates on a contiguous source string, tracking the document as a +// list of chunks: scannable source slices and quarantined replacement strings. Each scrubber scans +// the remaining scannable chunks; a match splits its chunk and the replacement is never re-examined +// by other scrubbers. Assembly produces a single string (zero-copy when nothing changed) or fills a +// StringBuilder when a legacy scrubber pass follows. +static partial class ScrubEngine +{ + internal readonly struct Chunk(string text, int start, int length, bool scannable) + { + public readonly string Text = text; + public readonly int Start = start; + public readonly int Length = length; + public readonly bool Scannable = scannable; + + public CharSpan Span => Text.AsSpan(Start, Length); + + public char First => Text[Start]; + + public char Last => Text[Start + Length - 1]; + } + + public static string Run( + string source, + EngineScrubberSet set, + Counter counter, + IReadOnlyDictionary context, + bool applyDirectoryReplacements) + { + var working = Scrubber.NormalizeNewlines(source); + var chunks = ScrubCore(working, set, counter, context, applyDirectoryReplacements); + if (chunks == null) + { + return working; + } + + return AssembleString(chunks); + } + + // target must already contain source, so an unchanged result leaves it untouched + public static void RunToBuilder( + string source, + EngineScrubberSet set, + Counter counter, + IReadOnlyDictionary context, + bool applyDirectoryReplacements, + StringBuilder target) + { + var working = Scrubber.NormalizeNewlines(source); + var chunks = ScrubCore(working, set, counter, context, applyDirectoryReplacements); + if (chunks == null) + { + if (ReferenceEquals(working, source)) + { + return; + } + + target.Clear(); + target.Append(working); + return; + } + + target.Clear(); + + var total = 0; + foreach (var chunk in chunks) + { + total += chunk.Length; + } + + target.EnsureCapacity(total); + foreach (var chunk in chunks) + { + target.Append(chunk.Text, chunk.Start, chunk.Length); + } + } + + // Returns null when nothing changed (the working source is already the result) + static List? ScrubCore( + string source, + EngineScrubberSet set, + Counter counter, + IReadOnlyDictionary context, + bool applyDirectoryReplacements) + { + if (source.Length == 0) + { + return null; + } + + List? chunks = null; + var changed = false; + + if (set.HasLinePhase) + { + chunks = LinePhase(source, set); + changed = chunks != null; + } + + foreach (var registered in set.Inline) + { + // A gated off built-in (inline dates or guids when the corresponding + // scrubbing is disabled) is skipped for the whole scrub rather than + // being probed at every candidate window + if (registered.Gate is { } gate && + !gate(counter)) + { + continue; + } + + // A scrubber registered without an explicit culture resolves here to + // the instance built for the culture in effect for this scrub + var scrubber = registered.Resolve(); + + // chunks stays null until something matches, so a pass that changes + // nothing allocates nothing + if (ApplyInline(chunks, source, scrubber, counter, context) is { } applied) + { + chunks = applied; + changed = true; + } + } + + // Path replacements are pinned last so user scrubbers always see raw paths + if (applyDirectoryReplacements) + { + var replacements = DirectoryReplacements.Current; + // When nothing has run the source is still the whole document, so its + // length gates the scan. Once a scrubber has run the document may have + // grown past the shortest find, and ApplyDirectoryReplacements skips any + // chunk that is too short on its own. + if (replacements.Pairs.Count > 0 && + (chunks != null || + source.Length >= replacements.ShortestFindLength)) + { + if (ApplyDirectoryReplacements(chunks, source, replacements) is { } replaced) + { + chunks = replaced; + changed = true; + } + } + } + + if (changed) + { + return chunks; + } + + return null; + } + + // Rebuilds the list, as ApplyInline does. Path replacements are never empty, so + // no join can appear and the result needs no coalescing. + static List? ApplyDirectoryReplacements(List? chunks, string source, DirectoryReplacements.Snapshot replacements) + { + var shortest = replacements.ShortestFindLength; + List? output = null; + var count = chunks?.Count ?? 1; + + for (var index = 0; index < count; index++) + { + var chunk = chunks?[index] ?? new(source, 0, source.Length, scannable: true); + if (!chunk.Scannable || + chunk.Length < shortest) + { + output?.Add(chunk); + continue; + } + + var after = chunks != null && index + 1 < count ? chunks[index + 1].First : (char?) null; + var offset = 0; + while (chunk.Length - offset >= shortest) + { + var span = chunk.Text.AsSpan(chunk.Start + offset, chunk.Length - offset); + var before = PrecedingChar(output, chunks, index); + if (!TryFindDirectoryMatch(span, before, after, replacements, out var matchStart, out var matchLength, out var replacement)) + { + break; + } + + output ??= CopyUpTo(chunks, index); + if (matchStart > 0) + { + output.Add(new(chunk.Text, chunk.Start + offset, matchStart, scannable: true)); + } + + output.Add(new(replacement, 0, replacement.Length, scannable: false)); + offset += matchStart + matchLength; + } + + if (output != null) + { + var remaining = chunk.Length - offset; + if (remaining > 0) + { + output.Add(new(chunk.Text, chunk.Start + offset, remaining, scannable: true)); + } + } + } + + return output; + } + + static bool TryFindDirectoryMatch( + CharSpan span, + char? beforeSegment, + char? afterSegment, + DirectoryReplacements.Snapshot replacements, + out int matchStart, + out int matchLength, + out string replacement) + { + var pairs = replacements.Pairs; + var shortest = replacements.ShortestFindLength; + var anchors = replacements.Anchors.AsSpan(); + for (var position = 0; position + shortest <= span.Length; position++) + { + // Skip to the next candidate first char + var skip = span[position..].IndexOfAny(anchors); + if (skip < 0) + { + break; + } + + position += skip; + if (position + shortest > span.Length) + { + break; + } + + if (DirectoryReplacements.TryMatchAt(span, position, beforeSegment, afterSegment, pairs, out matchLength, out replacement)) + { + matchStart = position; + return true; + } + } + + matchStart = 0; + matchLength = 0; + replacement = string.Empty; + return false; + } + + static string AssembleString(List chunks) + { + var total = 0; + foreach (var chunk in chunks) + { + total += chunk.Length; + } + + if (total == 0) + { + return string.Empty; + } + +#if NET6_0_OR_GREATER + return string.Create( + total, + chunks, + static (span, chunks) => + { + foreach (var chunk in chunks) + { + chunk.Span.CopyTo(span); + span = span[chunk.Length..]; + } + }); +#else + var buffer = new char[total]; + var position = 0; + foreach (var chunk in chunks) + { + chunk.Text.CopyTo(chunk.Start, buffer, position, chunk.Length); + position += chunk.Length; + } + + return new(buffer); +#endif + } + + // Rebuilds the list instead of splicing in place. An in place remove and insert + // shifts every element after the match, so a pass over a list that an earlier + // scrubber already fragmented would cost O(matches x chunks). + // Returns null when nothing matched, leaving the caller's list untouched and + // unallocated. + static List? ApplyInline( + List? chunks, + string source, + Scrubber scrubber, + Counter counter, + IReadOnlyDictionary context) + { + var effectiveMin = Math.Max(1, scrubber.MinLength); + List? output = null; + var deleted = false; + var count = chunks?.Count ?? 1; + + for (var index = 0; index < count; index++) + { + // A null list means nothing has changed yet, so the document is still + // the whole source + var chunk = chunks?[index] ?? new(source, 0, source.Length, scannable: true); + if (!chunk.Scannable || + chunk.Length < effectiveMin) + { + output?.Add(chunk); + continue; + } + + // Chunks past this one are untouched, so the following neighbor is fixed + var after = chunks != null && index + 1 < count ? chunks[index + 1].First : (char?) null; + var offset = 0; + while (chunk.Length - offset >= effectiveMin) + { + var span = chunk.Text.AsSpan(chunk.Start + offset, chunk.Length - offset); + var before = PrecedingChar(output, chunks, index); + if (!TryFindMatch(span, before, after, scrubber, counter, context, out var matchStart, out var matchLength, out var replacement)) + { + break; + } + + output ??= CopyUpTo(chunks, index); + if (matchStart > 0) + { + output.Add(new(chunk.Text, chunk.Start + offset, matchStart, scannable: true)); + } + + if (replacement.Length > 0) + { + output.Add(new(replacement, 0, replacement.Length, scannable: false)); + } + else + { + deleted = true; + } + + offset += matchStart + matchLength; + } + + if (output != null) + { + var remaining = chunk.Length - offset; + if (remaining > 0) + { + output.Add(new(chunk.Text, chunk.Start + offset, remaining, scannable: true)); + } + } + } + + if (output == null) + { + return null; + } + + if (deleted) + { + CoalesceScannable(output); + } + + return output; + } + + // The chunks before index are emitted unchanged, so they seed the rebuilt list + static List CopyUpTo(List? chunks, int index) + { + var output = new List((chunks?.Count ?? 1) + 4); + for (var copied = 0; copied < index; copied++) + { + output.Add(chunks![copied]); + } + + return output; + } + + // The char before the position being scanned: the last one emitted so far, or + // the end of the preceding chunk while nothing has been emitted + static char? PrecedingChar(List? output, List? chunks, int index) + { + if (output != null) + { + return output.Count > 0 ? output[^1].Last : null; + } + + return chunks != null && index > 0 ? chunks[index - 1].Last : null; + } + + // An empty replacement quarantines nothing, so the document text on either + // side of it is now adjacent. Merging those chunks keeps later scrubbers, and + // the path replacement pass, able to match across the join. + // Only called after a pass that deleted, since the line phase legitimately + // leaves the whole document as adjacent scannable chunks and merging those + // would materialize it for no gain. + static void CoalesceScannable(List chunks) + { + var index = 0; + while (index < chunks.Count - 1) + { + if (!chunks[index].Scannable || + !chunks[index + 1].Scannable) + { + index++; + continue; + } + + var end = index + 1; + while (end + 1 < chunks.Count && + chunks[end + 1].Scannable) + { + end++; + } + + var merged = Merge(chunks, index, end); + chunks.RemoveRange(index, end - index + 1); + chunks.Insert(index, merged); + index++; + } + } + + static Chunk Merge(List chunks, int start, int end) + { + var first = chunks[start]; + var total = first.Length; + var contiguous = true; + for (var index = start + 1; index <= end; index++) + { + var chunk = chunks[index]; + var previous = chunks[index - 1]; + if (!ReferenceEquals(chunk.Text, previous.Text) || + previous.Start + previous.Length != chunk.Start) + { + contiguous = false; + } + + total += chunk.Length; + } + + // Slices of the same string that are already adjacent need no allocation + if (contiguous) + { + return new(first.Text, first.Start, total, scannable: true); + } + + var builder = new StringBuilder(total); + for (var index = start; index <= end; index++) + { + var chunk = chunks[index]; + builder.Append(chunk.Text, chunk.Start, chunk.Length); + } + + return new(builder.ToString(), 0, total, scannable: true); + } + + static bool TryFindMatch( + CharSpan span, + char? before, + char? after, + Scrubber scrubber, + Counter counter, + IReadOnlyDictionary context, + out int matchStart, + out int matchLength, + out string replacement) + { + switch (scrubber.Kind) + { + case ScrubberKind.Replace: + return TryFindReplaceMatch(span, before, after, scrubber, out matchStart, out matchLength, out replacement); + case ScrubberKind.Window: + return TryFindWindowMatch(span, before, after, scrubber, counter, context, out matchStart, out matchLength, out replacement); + case ScrubberKind.Match: + return TryFindCustomMatch(span, scrubber, counter, context, out matchStart, out matchLength, out replacement); + default: + throw new($"Unexpected inline scrubber kind: {scrubber.Kind}"); + } + } + + static bool TryFindReplaceMatch( + CharSpan span, + char? before, + char? after, + Scrubber scrubber, + out int matchStart, + out int matchLength, + out string replacement) + { + var pairs = scrubber.Pairs!; + matchStart = -1; + matchLength = 0; + replacement = string.Empty; + + // Pairs are ordered longest first, so at equal positions the longest Find wins + foreach (var (find, pairReplacement) in pairs) + { + if (find.Length > span.Length) + { + continue; + } + + var searchFrom = 0; + while (searchFrom + find.Length <= span.Length) + { + var found = span[searchFrom..].IndexOf(find.AsSpan(), scrubber.Comparison); + if (found < 0) + { + break; + } + + var index = searchFrom + found; + if (matchStart >= 0 && + index >= matchStart) + { + // Cannot beat the current best position + break; + } + + if (scrubber.RequireWordBoundary && + !BoundaryOk(span, before, after, index, find.Length)) + { + searchFrom = index + 1; + continue; + } + + matchStart = index; + matchLength = find.Length; + replacement = pairReplacement; + break; + } + } + + return matchStart >= 0; + } + + static bool TryFindWindowMatch( + CharSpan span, + char? before, + char? after, + Scrubber scrubber, + Counter counter, + IReadOnlyDictionary context, + out int matchStart, + out int matchLength, + out string replacement) + { + var min = scrubber.MinLength; + var max = scrubber.MaxLength!.Value; + var matcher = scrubber.WindowMatcher!; + + // Windows never contain a line break, so scan the newline-free regions + var regionStart = 0; + while (regionStart < span.Length) + { + var newlineOffset = span[regionStart..].IndexOf('\n'); + var regionEnd = newlineOffset < 0 ? span.Length : regionStart + newlineOffset; + + for (var position = regionStart; position <= regionEnd - min; position++) + { + if (scrubber.Anchor != WindowAnchor.None) + { + // A match can only start where the anchor appears at the fixed + // offset, so jump to the next candidate + position = NextAnchoredPosition(span, position, regionEnd, scrubber); + if (position > regionEnd - min) + { + break; + } + } + + if (scrubber.RequireWordBoundary && + PreviousChar(span, before, position) is { } precedingChar && + char.IsLetterOrDigit(precedingChar)) + { + continue; + } + + var upper = Math.Min(max, regionEnd - position); + for (var length = upper; length >= min; length--) + { + if (scrubber.RequireWordBoundary && + NextChar(span, after, position + length) is { } followingChar && + char.IsLetterOrDigit(followingChar)) + { + continue; + } + + var result = matcher(span.Slice(position, length), counter, context); + if (result == null) + { + continue; + } + + matchStart = position; + matchLength = length; + replacement = Scrubber.NormalizeNewlines(result); + return true; + } + } + + if (newlineOffset < 0) + { + break; + } + + regionStart = regionEnd + 1; + } + + matchStart = 0; + matchLength = 0; + replacement = string.Empty; + return false; + } + +#if !NET8_0_OR_GREATER + static string asciiDigits = "0123456789"; +#endif + + // The next position at or after the given one where the scrubber's anchor + // appears at its offset from the window start. Returns past regionEnd when no + // candidate remains. + static int NextAnchoredPosition(CharSpan span, int position, int regionEnd, Scrubber scrubber) + { + var searchFrom = position + scrubber.AnchorOffset; + if (searchFrom >= regionEnd) + { + return regionEnd; + } + + var region = span[searchFrom..regionEnd]; + int found; + if (scrubber.Anchor == WindowAnchor.Char) + { + found = region.IndexOf(scrubber.AnchorChar); + } + else + { + // ASCII only on every target: char.IsDigit would also accept other + // Unicode digits, so the same input would anchor differently per + // target framework while sharing one verified file +#if NET8_0_OR_GREATER + found = region.IndexOfAnyInRange('0', '9'); +#else + found = region.IndexOfAny(asciiDigits.AsSpan()); +#endif + } + + if (found < 0) + { + return regionEnd; + } + + return searchFrom + found - scrubber.AnchorOffset; + } + + static bool TryFindCustomMatch( + CharSpan span, + Scrubber scrubber, + Counter counter, + IReadOnlyDictionary context, + out int matchStart, + out int matchLength, + out string replacement) + { + if (!scrubber.SegmentMatcher!(span, counter, context, out matchStart, out matchLength, out var result)) + { + replacement = string.Empty; + return false; + } + + if (result == null) + { + throw new("SegmentMatch returned true but no replacement."); + } + + if (matchStart < 0 || + matchLength <= 0 || + matchStart + matchLength > span.Length) + { + throw new($"SegmentMatch returned an invalid range. index: {matchStart}, length: {matchLength}, segment length: {span.Length}."); + } + + if (span.Slice(matchStart, matchLength).Contains('\n')) + { + throw new("SegmentMatch returned a range containing a line break. Matches must not span lines."); + } + + replacement = Scrubber.NormalizeNewlines(result); + return true; + } + + static bool BoundaryOk(CharSpan span, char? before, char? after, int index, int length) + { + if (PreviousChar(span, before, index) is { } precedingChar && + char.IsLetterOrDigit(precedingChar)) + { + return false; + } + + if (NextChar(span, after, index + length) is { } followingChar && + char.IsLetterOrDigit(followingChar)) + { + return false; + } + + return true; + } + + // The char before the given position within the span. At the span start the + // neighbor is the one preceding it in the document (replacement text counts). + static char? PreviousChar(CharSpan span, char? before, int position) + { + if (position > 0) + { + return span[position - 1]; + } + + return before; + } + + // The char at the given position within the span. At the span end the neighbor + // is the one following it in the document (replacement text counts). + static char? NextChar(CharSpan span, char? after, int position) + { + if (position < span.Length) + { + return span[position]; + } + + return after; + } +} diff --git a/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs b/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs new file mode 100644 index 000000000..90fdcaee1 --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/ScrubEngine_Lines.cs @@ -0,0 +1,334 @@ +// The line phase: walks the (already newline-normalized) source once, applying line drops first +// (needle, whitespace, and predicate based) then line transforms in registration order. Drops always +// evaluate the raw line; transform output becomes fresh scannable source for the inline phase. +// A transform that returns line breaks produces several lines, and the transforms after it see each +// of those on its own, since a line transform is defined over a single line. +// Join semantics replicate the legacy StringReader based line scrubbers: lines are joined with \n, +// the trailing newline is preserved only when the original ended with one, and RemoveEmptyLines +// additionally trims the trailing newline. +static partial class ScrubEngine +{ + static string newlineText = "\n"; + + readonly struct LineItem(int start, int end, string? fresh) + { + // Source slice [Start, End) when Fresh is null, otherwise Fresh is replacement line text + public readonly int Start = start; + public readonly int End = end; + public readonly string? Fresh = fresh; + } + + // Returns the chunks representing the document after line drops and transforms, + // or null when no line changed. + static List? LinePhase(string source, EngineScrubberSet set) + { + var endsWithNewline = source[^1] == '\n'; + // Created on the first change, so a document no line scrubber touches + // allocates nothing + List? items = null; + var changed = false; + + // Coalesce runs of adjacent untouched kept lines (including their inner + // separators) into single items + int? runStart = null; + var runEnd = 0; + + void FlushRun() + { + if (runStart is { } start) + { + (items ??= []).Add(new(start, runEnd, null)); + runStart = null; + } + } + + var position = 0; + while (position < source.Length) + { + var newlineIndex = source.IndexOf('\n', position); + var lineEnd = newlineIndex < 0 ? source.Length : newlineIndex; + var lineSpan = source.AsSpan(position, lineEnd - position); + + // Materialized at most once per line, shared by all string based scrubbers + string? lineString = null; + + if (IsDropped(lineSpan, ref lineString, set.LineDrops)) + { + changed = true; + FlushRun(); + } + else + { + var (removed, current) = ApplyTransforms(lineSpan, ref lineString, set.LineTransforms); + if (removed) + { + changed = true; + FlushRun(); + } + else if (current != null && + !lineSpan.SequenceEqual(current.AsSpan())) + { + changed = true; + FlushRun(); + (items ??= []).Add(new(0, 0, current)); + } + else + { + runStart ??= position; + runEnd = lineEnd; + } + } + + if (newlineIndex < 0) + { + break; + } + + position = newlineIndex + 1; + } + + // RemoveEmptyLines trims the trailing newline even when no line was dropped. + // Checked before the final flush so an untouched document stays unallocated: + // flushing would produce an item exactly when one exists or a run is pending. + if (set.TrimOuterEmptyLines && + endsWithNewline && + (items != null || runStart != null)) + { + changed = true; + } + + if (!changed) + { + return null; + } + + FlushRun(); + + var itemCount = items?.Count ?? 0; + var chunks = new List(itemCount + 1); + for (var index = 0; index < itemCount; index++) + { + var item = items![index]; + var needsSeparator = index < itemCount - 1 || + (endsWithNewline && !set.TrimOuterEmptyLines); + + if (item.Fresh is { } fresh) + { + // Transformed line text is fresh source, scannable by inline scrubbers + if (fresh.Length > 0) + { + chunks.Add(new(fresh, 0, fresh.Length, scannable: true)); + } + + if (needsSeparator) + { + chunks.Add(new(newlineText, 0, 1, scannable: true)); + } + + continue; + } + + // A separator is only needed when another item follows (the run ended at + // a dropped or transformed line, so its last line was terminated) or when + // the document ends with a newline. Either way the char after the run in + // the source is '\n', so the separator folds into the slice. + var length = item.End - item.Start + (needsSeparator ? 1 : 0); + if (length > 0) + { + chunks.Add(new(source, item.Start, length, scannable: true)); + } + } + + return chunks; + } + + static bool IsDropped(CharSpan lineSpan, ref string? lineString, Scrubber[] lineDrops) + { + foreach (var drop in lineDrops) + { + switch (drop.Kind) + { + case ScrubberKind.LineDropNeedles: + if (lineSpan.Length < drop.MinLength) + { + continue; + } + + foreach (var needle in drop.Needles!) + { + if (lineSpan.Contains(needle.AsSpan(), drop.Comparison)) + { + return true; + } + } + + continue; + case ScrubberKind.LineDropEmpty: + if (lineSpan.IsWhiteSpace()) + { + return true; + } + + continue; + case ScrubberKind.LineDropSpan: + if (drop.LineMatcher!(lineSpan)) + { + return true; + } + + continue; + case ScrubberKind.LineDropString: + lineString ??= lineSpan.ToString(); + if (drop.LineStringMatcher!(lineString)) + { + return true; + } + + continue; + default: + throw new($"Unexpected line drop kind: {drop.Kind}"); + } + } + + return false; + } + + static (bool removed, string? current) ApplyTransforms(CharSpan lineSpan, ref string? lineString, Scrubber[] lineTransforms) + { + // The most recent replacement text; null while the line is unchanged + string? current = null; + for (var index = 0; index < lineTransforms.Length; index++) + { + var transform = lineTransforms[index]; + string output; + switch (transform.Kind) + { + case ScrubberKind.LineTransformSpan: + { + var result = transform.LineReplacer!(current is null ? lineSpan : current.AsSpan()); + if (result.Kind == LineResult.RemoveKind) + { + return (true, null); + } + + if (result.Kind != LineResult.ReplaceKind) + { + continue; + } + + output = result.Text!; + break; + } + case ScrubberKind.LineTransformString: + { + var input = current ?? (lineString ??= lineSpan.ToString()); + var replaced = transform.LineStringReplacer!(input); + if (replaced == null) + { + return (true, null); + } + + output = Scrubber.NormalizeNewlines(replaced); + break; + } + default: + throw new($"Unexpected line transform kind: {transform.Kind}"); + } + + current = output; + + // A replacement holding line breaks is several lines, and a line + // transform is defined over one line, so the remaining transforms see + // each produced line on its own. The pre engine pipeline did this by + // re-reading the document between passes. + if (output.Contains('\n')) + { + return ApplyToProducedLines(output, lineTransforms, index + 1); + } + } + + return (false, current); + } + + // The remaining transforms over each line of a multi line replacement. + // Splitting and joining on '\n' are exact inverses, so transforms that change + // nothing leave the text as it was. + static (bool removed, string? current) ApplyToProducedLines(string text, Scrubber[] lineTransforms, int startIndex) + { + if (startIndex == lineTransforms.Length) + { + return (false, text); + } + + var lines = new List(text.Split('\n')); + for (var index = startIndex; index < lineTransforms.Length; index++) + { + var transform = lineTransforms[index]; + var next = new List(lines.Count); + foreach (var line in lines) + { + if (!TryTransformLine(transform, line, out var replacement)) + { + continue; + } + + if (replacement == null) + { + next.Add(line); + continue; + } + + if (replacement.Contains('\n')) + { + next.AddRange(replacement.Split('\n')); + continue; + } + + next.Add(replacement); + } + + lines = next; + } + + if (lines.Count == 0) + { + return (true, null); + } + + return (false, string.Join("\n", lines)); + } + + // False when the line is removed. replacement is null when it is unchanged. + static bool TryTransformLine(Scrubber transform, string line, out string? replacement) + { + switch (transform.Kind) + { + case ScrubberKind.LineTransformSpan: + { + var result = transform.LineReplacer!(line.AsSpan()); + if (result.Kind == LineResult.RemoveKind) + { + replacement = null; + return false; + } + + replacement = result.Kind == LineResult.ReplaceKind ? result.Text! : null; + return true; + } + case ScrubberKind.LineTransformString: + { + var output = transform.LineStringReplacer!(line); + if (output == null) + { + replacement = null; + return false; + } + + replacement = Scrubber.NormalizeNewlines(output); + return true; + } + default: + throw new($"Unexpected line transform kind: {transform.Kind}"); + } + } +} diff --git a/src/Verify/Serialization/Scrubbers/Scrubber.cs b/src/Verify/Serialization/Scrubbers/Scrubber.cs new file mode 100644 index 000000000..9428def29 --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/Scrubber.cs @@ -0,0 +1,374 @@ +/// +/// Defines a scrubbing operation executed by the span based scrub engine. +/// +/// +/// +/// Semantics shared by all scrubbers created from this type: +/// +/// +/// Quarantine: text produced by a replacement is never re-examined by other s. +/// Legacy AddScrubber(Action<StringBuilder>) scrubbers run afterwards and can still modify it. +/// Ordering: line removals run first, then line transforms (registration order), then inline +/// scrubbers: unknown max length first, then descending max length, ties broken by registration level +/// (instance, extension mapped instance, extension mapped global, global) then registration order. +/// Path replacements ({ProjectDirectory} etc) always run last. +/// Length rules: text shorter than a scrubber's minimum length is skipped without invoking the scrubber. +/// Single line rule: a match may never contain a line break. Finds may not contain \n or \r. +/// Word boundary: when required, a match is rejected if the character on either side is a letter or digit. +/// +/// +sealed class Scrubber +{ + internal ScrubberKind Kind { get; } + internal int MinLength { get; } + internal int? MaxLength { get; } + internal StringComparison Comparison { get; } + internal bool RequireWordBoundary { get; } + internal WindowAnchor Anchor { get; } + internal char AnchorChar { get; } + internal int AnchorOffset { get; } + internal (string Find, string Replacement)[]? Pairs { get; } + internal string[]? Needles { get; } + internal WindowMatch? WindowMatcher { get; } + internal SegmentMatch? SegmentMatcher { get; } + internal LineMatch? LineMatcher { get; } + internal Func? LineStringMatcher { get; } + internal LineReplace? LineReplacer { get; } + internal Func? LineStringReplacer { get; } + + // When set and it returns false, the engine skips this scrubber for the whole + // scrub rather than invoking it per candidate + internal Func? Gate { get; } + + // When set, the instance actually run is resolved at scrub time. Used by the + // built-in date scrubbers, whose parse culture, window bounds, and anchor all + // depend on the culture in effect when the scrub runs. + Func? resolver; + + internal Scrubber Resolve() => + resolver?.Invoke() ?? this; + + Scrubber( + ScrubberKind kind, + int minLength = 0, + int? maxLength = null, + StringComparison comparison = StringComparison.Ordinal, + bool requireWordBoundary = false, + (string Find, string Replacement)[]? pairs = null, + string[]? needles = null, + WindowMatch? windowMatcher = null, + SegmentMatch? segmentMatcher = null, + LineMatch? lineMatcher = null, + Func? lineStringMatcher = null, + LineReplace? lineReplacer = null, + Func? lineStringReplacer = null, + WindowAnchor anchor = WindowAnchor.None, + char anchorChar = '\0', + int anchorOffset = 0, + Func? gate = null, + Func? resolver = null) + { + Kind = kind; + MinLength = minLength; + MaxLength = maxLength; + Comparison = comparison; + RequireWordBoundary = requireWordBoundary; + Pairs = pairs; + Needles = needles; + WindowMatcher = windowMatcher; + SegmentMatcher = segmentMatcher; + LineMatcher = lineMatcher; + LineStringMatcher = lineStringMatcher; + LineReplacer = lineReplacer; + LineStringReplacer = lineStringReplacer; + Anchor = anchor; + AnchorChar = anchorChar; + AnchorOffset = anchorOffset; + Gate = gate; + this.resolver = resolver; + } + + internal bool IsLineDrop => + Kind is ScrubberKind.LineDropNeedles + or ScrubberKind.LineDropSpan + or ScrubberKind.LineDropString + or ScrubberKind.LineDropEmpty; + + internal bool IsLineTransform => + Kind is ScrubberKind.LineTransformSpan + or ScrubberKind.LineTransformString; + + /// + /// Replace every occurrence of with . + /// must be or . + /// + public static Scrubber Replace( + string find, + string replacement, + StringComparison comparison = StringComparison.Ordinal, + bool requireWordBoundary = false) + { + ValidateFind(find); + Ensure.NotNull(replacement); + ValidateOrdinal(comparison); + return new( + ScrubberKind.Replace, + minLength: find.Length, + maxLength: find.Length, + comparison: comparison, + requireWordBoundary: requireWordBoundary, + pairs: [(find, NormalizeNewlines(replacement))]); + } + + /// + /// Replace every occurrence of each Find with its Replacement. + /// At a given position the longest matching Find wins. + /// must be or . + /// + public static Scrubber Replace( + StringComparison comparison, + bool requireWordBoundary, + params (string Find, string Replacement)[] pairs) + { + Ensure.NotNullOrEmpty(pairs); + ValidateOrdinal(comparison); + var ordered = new (string Find, string Replacement)[pairs.Length]; + for (var index = 0; index < pairs.Length; index++) + { + var (find, replacement) = pairs[index]; + ValidateFind(find); + Ensure.NotNull(replacement); + ordered[index] = (find, NormalizeNewlines(replacement)); + } + + // Longest first so the most specific Find wins at any given position + Array.Sort(ordered, (left, right) => right.Find.Length.CompareTo(left.Find.Length)); + return new( + ScrubberKind.Replace, + minLength: ordered[^1].Find.Length, + maxLength: ordered[0].Find.Length, + comparison: comparison, + requireWordBoundary: requireWordBoundary, + pairs: ordered); + } + + /// + /// Match candidate windows of text between and characters. + /// At each position the engine tries the longest window first. + /// + public static Scrubber Window( + int minLength, + int maxLength, + WindowMatch matcher, + bool requireWordBoundary = false) + { + Ensure.NotNull(matcher); + ValidateWindowLengths(minLength, maxLength); + + return new( + ScrubberKind.Window, + minLength: minLength, + maxLength: maxLength, + requireWordBoundary: requireWordBoundary, + windowMatcher: matcher); + } + + static void ValidateWindowLengths(int minLength, int maxLength) + { + if (minLength < 1) + { + throw new ArgumentOutOfRangeException(nameof(minLength), minLength, "minLength must be at least 1."); + } + + if (maxLength < minLength) + { + throw new ArgumentOutOfRangeException(nameof(maxLength), maxLength, "maxLength must be greater than or equal to minLength."); + } + } + + // A Window scrubber used by the built-in guid and date scrubbers. + // The gate is evaluated once per scrub, so a disabled built-in costs a single + // check instead of a full scan. + // When an anchor is supplied, matches can only start where it appears at + // anchorOffset from the window start, so no-match scans skip between candidate + // positions. + internal static Scrubber GatedWindow( + int minLength, + int maxLength, + WindowMatch matcher, + Func gate, + bool requireWordBoundary = false, + WindowAnchor anchor = WindowAnchor.None, + char anchorChar = '\0', + int anchorOffset = 0, + Func? resolver = null) + { + ValidateWindowLengths(minLength, maxLength); + + return new( + ScrubberKind.Window, + minLength: minLength, + maxLength: maxLength, + requireWordBoundary: requireWordBoundary, + windowMatcher: matcher, + anchor: anchor, + anchorChar: anchorChar, + anchorOffset: anchorOffset, + gate: gate, + resolver: resolver); + } + + /// + /// Find matches using custom search logic. + /// : segments shorter than this are skipped (null scans everything). + /// : used for ordering only; null (unknown) runs before all known length scrubbers. + /// + public static Scrubber Match( + SegmentMatch matcher, + int? minLength = null, + int? maxLength = null) + { + Ensure.NotNull(matcher); + if (minLength is < 1) + { + throw new ArgumentOutOfRangeException(nameof(minLength), minLength, "minLength must be at least 1."); + } + + if (maxLength < minLength) + { + throw new ArgumentOutOfRangeException(nameof(maxLength), maxLength, "maxLength must be greater than or equal to minLength."); + } + + return new( + ScrubberKind.Match, + minLength: minLength ?? 0, + maxLength: maxLength, + segmentMatcher: matcher); + } + + /// + /// Remove any lines containing any of . + /// + public static Scrubber RemoveLinesContaining(StringComparison comparison, params string[] needles) + { + Ensure.NotNullOrEmpty(needles); + var copy = new string[needles.Length]; + var minLength = int.MaxValue; + for (var index = 0; index < needles.Length; index++) + { + var needle = needles[index]; + ValidateFind(needle); + copy[index] = needle; + minLength = Math.Min(minLength, needle.Length); + } + + return new( + ScrubberKind.LineDropNeedles, + // A linguistic comparison can match a run shorter than the needle, so + // a line shorter than the needle cannot be skipped unread + minLength: IsOrdinal(comparison) ? minLength : 0, + comparison: comparison, + needles: copy); + } + + /// + /// Remove any lines containing any of , using . + /// + public static Scrubber RemoveLinesContaining(params string[] needles) => + RemoveLinesContaining(StringComparison.OrdinalIgnoreCase, needles); + + /// + /// Remove any lines matching . + /// No per line string is allocated for span predicates. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. RemoveLines((ReadOnlySpan<char> line) => ...). + /// + [OverloadResolutionPriority(-1)] + public static Scrubber RemoveLines(LineMatch shouldRemove) + { + Ensure.NotNull(shouldRemove); + return new(ScrubberKind.LineDropSpan, lineMatcher: shouldRemove); + } + + /// + /// Remove any lines matching . + /// + public static Scrubber RemoveLines(Func shouldRemove) + { + Ensure.NotNull(shouldRemove); + return new(ScrubberKind.LineDropString, lineStringMatcher: shouldRemove); + } + + /// + /// Process each line via . + /// No per line string is allocated for span based replacers. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ReplaceLines((ReadOnlySpan<char> line) => ...). + /// + [OverloadResolutionPriority(-1)] + public static Scrubber ReplaceLines(LineReplace replace) + { + Ensure.NotNull(replace); + return new(ScrubberKind.LineTransformSpan, lineReplacer: replace); + } + + /// + /// Process each line via . + /// can return the input to keep the line, a different string to replace it, or null to remove it. + /// + public static Scrubber ReplaceLines(Func replace) + { + Ensure.NotNull(replace); + return new(ScrubberKind.LineTransformString, lineStringReplacer: replace); + } + + /// + /// Remove any lines containing only whitespace. + /// + public static Scrubber RemoveEmptyLines() => + new(ScrubberKind.LineDropEmpty); + + internal static string NormalizeNewlines(string value) + { + if (!value.Contains('\r')) + { + return value; + } + + return value + .Replace("\r\n", "\n") + .Replace('\r', '\n'); + } + + // A replacement splices out exactly Find.Length chars, and the engine skips + // text shorter than Find. Both only hold for ordinal comparisons: a linguistic + // comparison can match a run of a different length than Find (for example an + // ignorable soft hyphen, or 'ss' matching 'ß'), which would splice the wrong + // range. + static void ValidateOrdinal(StringComparison comparison) + { + if (comparison is StringComparison.Ordinal or StringComparison.OrdinalIgnoreCase) + { + return; + } + + throw new ArgumentException( + $"Only Ordinal and OrdinalIgnoreCase are supported, but was {comparison}. A linguistic comparison can match a different number of characters than the find, so the match cannot be replaced reliably.", + nameof(comparison)); + } + + // True when a match is guaranteed to be the same length as the needle, so + // text shorter than the needle can be skipped without comparing + static bool IsOrdinal(StringComparison comparison) => + comparison is StringComparison.Ordinal or StringComparison.OrdinalIgnoreCase; + + static void ValidateFind(string find) + { + Ensure.NotNullOrEmpty(find); + if (find.Contains('\n') || + find.Contains('\r')) + { + throw new ArgumentException($"Find must not contain line breaks: {find}", nameof(find)); + } + } +} diff --git a/src/Verify/Serialization/Scrubbers/ScrubberKind.cs b/src/Verify/Serialization/Scrubbers/ScrubberKind.cs new file mode 100644 index 000000000..41634e269 --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/ScrubberKind.cs @@ -0,0 +1,22 @@ +enum ScrubberKind +{ + Replace, + Window, + Match, + LineDropNeedles, + LineDropSpan, + LineDropString, + LineDropEmpty, + LineTransformSpan, + LineTransformString, +} + +// A cheap vectorized scan target for Window scrubbers: a match can only start +// where the anchor appears at the given offset from the window start, so the +// engine can jump between candidate positions instead of probing every one. +enum WindowAnchor +{ + None, + Char, + Digit, +} diff --git a/src/Verify/Serialization/Scrubbers/Scrubber_Delegates.cs b/src/Verify/Serialization/Scrubbers/Scrubber_Delegates.cs new file mode 100644 index 000000000..b65ac4ecf --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/Scrubber_Delegates.cs @@ -0,0 +1,34 @@ +namespace VerifyTests; + +/// +/// Attempts to match a candidate window of text. +/// The window length is between the minLength and maxLength given at registration and never contains a line break. +/// Return the replacement text, or null when the window is not a match. +/// +public delegate string? WindowMatch( + CharSpan window, + Counter counter, + IReadOnlyDictionary context); + +/// +/// Attempts to find the next match within a segment of text. +/// The segment may contain line breaks, but the matched range must not. +/// Return true and set , , and when a match is found. +/// +public delegate bool SegmentMatch( + CharSpan segment, + Counter counter, + IReadOnlyDictionary context, + out int index, + out int length, + out string? replacement); + +/// +/// Determines if a line matches. The line excludes its terminator. +/// +public delegate bool LineMatch(CharSpan line); + +/// +/// Produces a for a line. The line excludes its terminator. +/// +public delegate LineResult LineReplace(CharSpan line); diff --git a/src/Verify/Serialization/Scrubbers/SharedScrubber_Dates.cs b/src/Verify/Serialization/Scrubbers/SharedScrubber_Dates.cs index 733f5bb50..10e3ad0e5 100644 --- a/src/Verify/Serialization/Scrubbers/SharedScrubber_Dates.cs +++ b/src/Verify/Serialization/Scrubbers/SharedScrubber_Dates.cs @@ -178,7 +178,8 @@ public bool TryConvertDateTime(CharSpan value, [NotNullWhen(true)] out string? r { if (ScrubDateTimes) { - if (TryParseDateTime(value, "yyyy-MM-ddTHH:mm:ss.FFFFFFFK", out var date)) + if (CanBeIsoDate(value) && + TryParseDateTime(value, "yyyy-MM-ddTHH:mm:ss.FFFFFFFK", out var date)) { result = Convert(date); return true; @@ -198,6 +199,15 @@ public bool TryConvertDateTime(CharSpan value, [NotNullWhen(true)] out string? r return false; } + // The ISO format is "yyyy-MM-ddTHH:mm:ss.FFFFFFFK": at least 19 chars, starting + // with a 4 digit year then '-'. TryParseExact with DateTimeStyles.None allows no + // whitespace, so values failing this shape can never parse and the attempt can + // be skipped. + static bool CanBeIsoDate(CharSpan value) => + value.Length >= 19 && + char.IsDigit(value[0]) && + value[4] == '-'; + static bool TryParseDateTime(CharSpan value, string format, out DateTime dateTime) => DateTime.TryParseExact(value, format, null, DateTimeStyles.None, out dateTime); @@ -205,7 +215,8 @@ public bool TryConvertDateTimeOffset(CharSpan value, [NotNullWhen(true)] out str { if (ScrubDateTimes) { - if (TryParseDateTimeOffset(value, "yyyy-MM-ddTHH:mm:ss.FFFFFFFK", out var date)) + if (CanBeIsoDate(value) && + TryParseDateTimeOffset(value, "yyyy-MM-ddTHH:mm:ss.FFFFFFFK", out var date)) { result = Convert(date); return true; diff --git a/src/Verify/Serialization/Scrubbers/SharedScrubber_Guids.cs b/src/Verify/Serialization/Scrubbers/SharedScrubber_Guids.cs index 1dbeeaa6a..038c7b107 100644 --- a/src/Verify/Serialization/Scrubbers/SharedScrubber_Guids.cs +++ b/src/Verify/Serialization/Scrubbers/SharedScrubber_Guids.cs @@ -28,7 +28,11 @@ public bool TryConvertGuid(CharSpan value, [NotNullWhen(true)] out string? resul { if (ScrubGuids) { - if (Guid.TryParse(value, out var guid)) + // The shortest parseable guid (the "N" format) is 32 chars. Whitespace + // padding, which Guid.TryParse trims, only adds length, so anything + // shorter can never parse and the attempt can be skipped. + if (value.Length >= 32 && + Guid.TryParse(value, out var guid)) { result = Convert(guid); return true; diff --git a/src/Verify/Serialization/Scrubbers/UserMachineScrubber.cs b/src/Verify/Serialization/Scrubbers/UserMachineScrubber.cs index eaddc697f..c3a0b0202 100644 --- a/src/Verify/Serialization/Scrubbers/UserMachineScrubber.cs +++ b/src/Verify/Serialization/Scrubbers/UserMachineScrubber.cs @@ -1,4 +1,4 @@ -static partial class UserMachineScrubber +static class UserMachineScrubber { static string machineName; static string userName; @@ -13,9 +13,9 @@ internal static void ResetReplacements(string machineName, string userName) UserMachineScrubber.userName = userName; } - public static void Machine(StringBuilder builder) => - PerformReplacements(builder, machineName, "TheMachineName"); + public static Scrubber MachineScrubber() => + Scrubber.Replace(machineName, "TheMachineName", StringComparison.Ordinal, requireWordBoundary: true); - public static void User(StringBuilder builder) => - PerformReplacements(builder, userName, "TheUserName"); -} \ No newline at end of file + public static Scrubber UserScrubber() => + Scrubber.Replace(userName, "TheUserName", StringComparison.Ordinal, requireWordBoundary: true); +} diff --git a/src/Verify/Serialization/Scrubbers/UserMachineScrubber_PerformReplacements.cs b/src/Verify/Serialization/Scrubbers/UserMachineScrubber_PerformReplacements.cs deleted file mode 100644 index 0ff848905..000000000 --- a/src/Verify/Serialization/Scrubbers/UserMachineScrubber_PerformReplacements.cs +++ /dev/null @@ -1,144 +0,0 @@ -static partial class UserMachineScrubber -{ - static bool IsValidWrapper(char ch) => - !char.IsLetterOrDigit(ch); - - public static void PerformReplacements(StringBuilder builder, string find, string replace) - { - if (builder.Length < find.Length) - { - return; - } - - var matches = FindMatches(builder, find); - - // Sort by position descending. In-place to avoid LINQ allocation - matches.Sort((a, b) => b.CompareTo(a)); - - // Apply matches - foreach (var match in matches) - { - builder.Overwrite(replace, match, find.Length); - } - } - - static List FindMatches(StringBuilder builder, string find) - { - var matches = new List(); - var absolutePosition = 0; - var carryoverSize = find.Length - 1; - - Span carryoverBuffer = stackalloc char[carryoverSize]; - Span combinedBuffer = stackalloc char[find.Length]; - var carryoverLength = 0; - var previousChunkAbsoluteEnd = 0; - - foreach (var chunk in builder.GetChunks()) - { - var chunkSpan = chunk.Span; - - // Check for matches spanning from previous chunk to current chunk - if (carryoverLength > 0) - { - for (var carryoverIndex = 0; carryoverIndex < carryoverLength; carryoverIndex++) - { - var remainingInCarryover = carryoverLength - carryoverIndex; - var neededFromCurrent = find.Length - remainingInCarryover; - - if (neededFromCurrent <= 0 || - neededFromCurrent > chunkSpan.Length) - { - continue; - } - - // Build combined buffer - carryoverBuffer.Slice(carryoverIndex, remainingInCarryover).CopyTo(combinedBuffer); - chunkSpan[..neededFromCurrent].CopyTo(combinedBuffer[remainingInCarryover..]); - - // Check if it matches - if (!combinedBuffer.SequenceEqual(find)) - { - continue; - } - - var startPosition = previousChunkAbsoluteEnd - carryoverLength + carryoverIndex; - - // Check preceding character - var validStart = startPosition == 0 || - IsValidWrapper(builder[startPosition - 1]); - - if (!validStart) - { - continue; - } - - // Check trailing character. Use the builder indexer rather - // than chunkSpan[neededFromCurrent], which is out of range when - // the match ends exactly at this chunk's boundary, so the check - // still works when there is a following chunk. - var endPosition = startPosition + find.Length; - var validEnd = endPosition >= builder.Length || - IsValidWrapper(builder[endPosition]); - - if (!validEnd) - { - continue; - } - - matches.Add(startPosition); - } - } - - // Process matches entirely within this chunk - if (chunk.Length >= find.Length) - { - var chunkIndex = 0; - while (true) - { - var value = chunkSpan; - var searchSpan = value[chunkIndex..]; - var foundIndex = searchSpan.IndexOf(find); - if (foundIndex == -1) - { - break; - } - - chunkIndex += foundIndex; - var end = chunkIndex + find.Length; - - if ((chunkIndex != 0 && !IsValidWrapper(value[chunkIndex - 1])) || - (end != value.Length && !IsValidWrapper(value[end]))) - { - chunkIndex++; - continue; - } - - matches.Add(absolutePosition + chunkIndex); - chunkIndex += find.Length; - } - } - - // Roll the carryover forward: keep the last carryoverSize chars of - // everything seen so far. Rebuilding it from the current chunk alone - // drops the prefix when a chunk is shorter than the search string, so - // a token spanning three or more chunks would never be found. - if (chunk.Length >= carryoverSize) - { - chunkSpan.Slice(chunk.Length - carryoverSize, carryoverSize).CopyTo(carryoverBuffer); - carryoverLength = carryoverSize; - } - else - { - var keep = Math.Min(carryoverLength, carryoverSize - chunk.Length); - carryoverBuffer.Slice(carryoverLength - keep, keep).CopyTo(carryoverBuffer); - chunkSpan.CopyTo(carryoverBuffer[keep..]); - carryoverLength = keep + chunk.Length; - } - - previousChunkAbsoluteEnd = absolutePosition + chunk.Length; - absolutePosition += chunk.Length; - } - - return matches; - } -} \ No newline at end of file diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs index 3ad5609b3..60085999e 100644 --- a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers.cs @@ -1,9 +1,26 @@ -namespace VerifyTests; +namespace VerifyTests; public static partial class VerifierSettings { internal static Dictionary>>> ExtensionMappedGlobalScrubbers = []; + internal static Dictionary> ExtensionMappedGlobalSpanScrubbers = []; + + /// + /// Add a that applies to all verified files with a matching extension. + /// + internal static void AddScrubber(string extension, Scrubber scrubber) + { + InnerVerifier.ThrowIfVerifyHasBeenRun(); + if (!ExtensionMappedGlobalSpanScrubbers.TryGetValue(extension, out var values)) + { + ExtensionMappedGlobalSpanScrubbers[extension] = values = []; + } + + values.Add(scrubber); + EngineScrubberSet.InvalidateGlobalCache(); + } + /// /// Modify the resulting test content using custom code. /// @@ -45,82 +62,93 @@ public static void AddScrubber(string extension, Action /// Remove any lines containing any of from the test results. /// - public static void ScrubLinesContaining(string extension, StringComparison comparison, params string[] stringToMatch) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - ScrubLinesContaining(extension, comparison, ScrubberLocation.First, stringToMatch); - } + public static void ScrubLinesContaining(string extension, StringComparison comparison, params string[] stringToMatch) => + AddScrubber(extension, Scrubber.RemoveLinesContaining(comparison, stringToMatch)); /// - /// Remove any lines containing any of from the test results. + /// Remove any lines matching from the test results. /// - public static void ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(extension, _ => _.RemoveLinesContaining(comparison, stringToMatch), location); - } + public static void ScrubLines(string extension, Func removeLine) => + AddScrubber(extension, Scrubber.RemoveLines(removeLine)); /// /// Remove any lines matching from the test results. + /// No per line string is allocated for span predicates. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ScrubLines(extension, (ReadOnlySpan<char> line) => ...). /// - public static void ScrubLines(string extension, Func removeLine, ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(extension, _ => _.FilterLines(removeLine), location); - } + [OverloadResolutionPriority(-1)] + public static void ScrubLines(string extension, LineMatch removeLine) => + AddScrubber(extension, Scrubber.RemoveLines(removeLine)); /// /// Remove any lines containing only whitespace from the test results. /// - public static void ScrubEmptyLines(string extension, ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(extension, _ => _.RemoveEmptyLines(), location); - } + public static void ScrubEmptyLines(string extension) => + AddScrubber(extension, Scrubber.RemoveEmptyLines()); /// /// Replace inline s with a placeholder. /// - public static void ScrubInlineGuids(string extension, ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(extension, GuidScrubber.ReplaceGuids, location); - } + public static void ScrubInlineGuids(string extension) => + AddScrubber(extension, GuidMatcher.Instance); /// /// Scrub lines with an optional replace. /// can return the input to ignore the line, or return a different string to replace it. /// - public static void ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(extension, _ => _.ReplaceLines(replaceLine), location); - } + public static void ScrubLinesWithReplace(string extension, Func replaceLine) => + AddScrubber(extension, Scrubber.ReplaceLines(replaceLine)); /// - /// Remove any lines containing any of from the test results. + /// Scrub lines with an optional replace. + /// No per line string is allocated for span based replacers. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ScrubLinesWithReplace(extension, (ReadOnlySpan<char> line) => ...). /// - public static void ScrubLinesContaining(string extension, ScrubberLocation location = ScrubberLocation.First, params string[] stringToMatch) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - ScrubLinesContaining(extension, StringComparison.OrdinalIgnoreCase, location, stringToMatch); - } + [OverloadResolutionPriority(-1)] + public static void ScrubLinesWithReplace(string extension, LineReplace replaceLine) => + AddScrubber(extension, Scrubber.ReplaceLines(replaceLine)); /// /// Remove the from the test results. /// - public static void ScrubMachineName(string extension, ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(extension, UserMachineScrubber.Machine, location); - } + public static void ScrubMachineName(string extension) => + AddScrubber(extension, UserMachineScrubber.MachineScrubber()); /// /// Remove the from the test results. /// - public static void ScrubUserName(string extension, ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(extension, UserMachineScrubber.User, location); - } -} \ No newline at end of file + public static void ScrubUserName(string extension) => + AddScrubber(extension, UserMachineScrubber.UserScrubber()); + + /// + /// Replace every occurrence of with . + /// must be or . + /// + public static void ScrubReplace(string extension, string find, string replacement, StringComparison comparison = StringComparison.Ordinal, bool requireWordBoundary = false) => + AddScrubber(extension, Scrubber.Replace(find, replacement, comparison, requireWordBoundary)); + + /// + /// Replace every occurrence of each Find with its Replacement. + /// At a given position the longest matching Find wins. + /// must be or . + /// + public static void ScrubReplace(string extension, StringComparison comparison, bool requireWordBoundary, params (string Find, string Replacement)[] pairs) => + AddScrubber(extension, Scrubber.Replace(comparison, requireWordBoundary, pairs)); + + /// + /// Match candidate windows of text between and characters. + /// At each position the engine tries the longest window first. + /// + public static void ScrubWindow(string extension, int minLength, int maxLength, WindowMatch matcher, bool requireWordBoundary = false) => + AddScrubber(extension, Scrubber.Window(minLength, maxLength, matcher, requireWordBoundary)); + + /// + /// Find matches using custom search logic. + /// : segments shorter than this are skipped (null scans everything). + /// : used for ordering only; null (unknown) runs before all known length scrubbers. + /// + public static void ScrubMatch(string extension, SegmentMatch matcher, int? minLength = null, int? maxLength = null) => + AddScrubber(extension, Scrubber.Match(matcher, minLength, maxLength)); +} diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers_Obsoletes.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers_Obsoletes.cs new file mode 100644 index 000000000..251bcfef3 --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_ExtensionMappedGlobalScrubbers_Obsoletes.cs @@ -0,0 +1,61 @@ +namespace VerifyTests; + +public static partial class VerifierSettings +{ + /// + /// Remove any lines containing any of from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(extension, comparison, stringToMatch); + + /// + /// Remove any lines matching from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubLines(string extension, Func removeLine, ScrubberLocation location) => + ScrubLines(extension, removeLine); + + /// + /// Remove any lines containing only whitespace from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubEmptyLines(string extension, ScrubberLocation location) => + ScrubEmptyLines(extension); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public static void ScrubInlineGuids(string extension, ScrubberLocation location) => + ScrubInlineGuids(extension); + + /// + /// Scrub lines with an optional replace. + /// can return the input to ignore the line, or return a different string to replace it. + /// + [Obsolete(locationObsolete)] + public static void ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(extension, replaceLine); + + /// + /// Remove any lines containing any of from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubLinesContaining(string extension, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(extension, StringComparison.OrdinalIgnoreCase, stringToMatch); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubMachineName(string extension, ScrubberLocation location) => + ScrubMachineName(extension); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubUserName(string extension, ScrubberLocation location) => + ScrubUserName(extension); +} diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs index cb6732bb4..ea66c5860 100644 --- a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers.cs @@ -1,9 +1,21 @@ -namespace VerifyTests; +namespace VerifyTests; public static partial class VerifierSettings { internal static List>> GlobalScrubbers = []; + internal static List GlobalSpanScrubbers = []; + + /// + /// Add a that applies to all verified files. + /// + internal static void AddScrubber(Scrubber scrubber) + { + InnerVerifier.ThrowIfVerifyHasBeenRun(); + GlobalSpanScrubbers.Add(scrubber); + EngineScrubberSet.InvalidateGlobalCache(); + } + /// /// Modify the resulting test content using custom code. /// @@ -36,38 +48,30 @@ public static void AddScrubber(Action /// Remove any lines containing any of from the test results. /// - public static void ScrubLinesContaining(StringComparison comparison, params string[] stringToMatch) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - ScrubLinesContaining(comparison, ScrubberLocation.First, stringToMatch); - } + public static void ScrubLinesContaining(StringComparison comparison, params string[] stringToMatch) => + AddScrubber(Scrubber.RemoveLinesContaining(comparison, stringToMatch)); /// - /// Remove any lines containing any of from the test results. + /// Remove any lines matching from the test results. /// - public static void ScrubLinesContaining(StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(_ => _.RemoveLinesContaining(comparison, stringToMatch), location); - } + public static void ScrubLines(Func removeLine) => + AddScrubber(Scrubber.RemoveLines(removeLine)); /// /// Remove any lines matching from the test results. + /// No per line string is allocated for span predicates. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ScrubLines((ReadOnlySpan<char> line) => ...). /// - public static void ScrubLines(Func removeLine, ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(_ => _.FilterLines(removeLine), location); - } + [OverloadResolutionPriority(-1)] + public static void ScrubLines(LineMatch removeLine) => + AddScrubber(Scrubber.RemoveLines(removeLine)); /// /// Remove any lines containing only whitespace from the test results. /// - public static void ScrubEmptyLines(ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(_ => _.RemoveEmptyLines(), location); - } + public static void ScrubEmptyLines() => + AddScrubber(Scrubber.RemoveEmptyLines()); internal static bool DateCountingEnabled { get; private set; } = true; @@ -82,22 +86,26 @@ public static void DisableDateCounting() => /// public static void ScrubInlineDateTimes( [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) => - AddScrubber( - DateScrubber.BuildDateTimeScrubber(format, culture), - location); + Culture? culture = null) + { + foreach (var scrubber in DateMatchers.DateTimes(format, culture)) + { + AddScrubber(scrubber); + } + } /// - /// Replace inline s with a placeholder. + /// Replace inline s with a placeholder. /// public static void ScrubInlineDateTimeOffsets( [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) => - AddScrubber( - DateScrubber.BuildDateTimeOffsetScrubber(format, culture), - location); + Culture? culture = null) + { + foreach (var scrubber in DateMatchers.DateTimeOffsets(format, culture)) + { + AddScrubber(scrubber); + } + } #if NET6_0_OR_GREATER @@ -106,66 +114,84 @@ public static void ScrubInlineDateTimeOffsets( /// public static void ScrubInlineDates( [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) => - AddScrubber( - DateScrubber.BuildDateScrubber(format, culture), - location); + Culture? culture = null) + { + foreach (var scrubber in DateMatchers.Dates(format, culture)) + { + AddScrubber(scrubber); + } + } #endif /// /// Replace inline s with a placeholder. /// - public static void ScrubInlineGuids(ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(GuidScrubber.ReplaceGuids, location); - } + public static void ScrubInlineGuids() => + AddScrubber(GuidMatcher.Instance); /// /// Scrub lines with an optional replace. /// can return the input to ignore the line, or return a different string to replace it. /// - public static void ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(_ => _.ReplaceLines(replaceLine), location); - } + public static void ScrubLinesWithReplace(Func replaceLine) => + AddScrubber(Scrubber.ReplaceLines(replaceLine)); /// - /// Remove any lines containing any of from the test results. + /// Scrub lines with an optional replace. + /// No per line string is allocated for span based replacers. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ScrubLinesWithReplace((ReadOnlySpan<char> line) => ...). /// - public static void ScrubLinesContaining(params string[] stringToMatch) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - ScrubLinesContaining(ScrubberLocation.First, stringToMatch); - } + [OverloadResolutionPriority(-1)] + public static void ScrubLinesWithReplace(LineReplace replaceLine) => + AddScrubber(Scrubber.ReplaceLines(replaceLine)); /// /// Remove any lines containing any of from the test results. /// - public static void ScrubLinesContaining(ScrubberLocation location = ScrubberLocation.First, params string[] stringToMatch) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, location, stringToMatch); - } + public static void ScrubLinesContaining(params string[] stringToMatch) => + ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, stringToMatch); /// /// Remove the from the test results. /// - public static void ScrubMachineName(ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(UserMachineScrubber.Machine, location); - } + public static void ScrubMachineName() => + AddScrubber(UserMachineScrubber.MachineScrubber()); /// /// Remove the from the test results. /// - public static void ScrubUserName(ScrubberLocation location = ScrubberLocation.First) - { - InnerVerifier.ThrowIfVerifyHasBeenRun(); - AddScrubber(UserMachineScrubber.User, location); - } -} \ No newline at end of file + public static void ScrubUserName() => + AddScrubber(UserMachineScrubber.UserScrubber()); + + /// + /// Replace every occurrence of with . + /// must be or . + /// + public static void ScrubReplace(string find, string replacement, StringComparison comparison = StringComparison.Ordinal, bool requireWordBoundary = false) => + AddScrubber(Scrubber.Replace(find, replacement, comparison, requireWordBoundary)); + + /// + /// Replace every occurrence of each Find with its Replacement. + /// At a given position the longest matching Find wins. + /// must be or . + /// + public static void ScrubReplace(StringComparison comparison, bool requireWordBoundary, params (string Find, string Replacement)[] pairs) => + AddScrubber(Scrubber.Replace(comparison, requireWordBoundary, pairs)); + + /// + /// Match candidate windows of text between and characters. + /// At each position the engine tries the longest window first. + /// + public static void ScrubWindow(int minLength, int maxLength, WindowMatch matcher, bool requireWordBoundary = false) => + AddScrubber(Scrubber.Window(minLength, maxLength, matcher, requireWordBoundary)); + + /// + /// Find matches using custom search logic. + /// : segments shorter than this are skipped (null scans everything). + /// : used for ordering only; null (unknown) runs before all known length scrubbers. + /// + public static void ScrubMatch(SegmentMatch matcher, int? minLength = null, int? maxLength = null) => + AddScrubber(Scrubber.Match(matcher, minLength, maxLength)); +} diff --git a/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers_Obsoletes.cs b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers_Obsoletes.cs new file mode 100644 index 000000000..d84249220 --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/VerifierSettings_GlobalScrubbers_Obsoletes.cs @@ -0,0 +1,97 @@ +namespace VerifyTests; + +public static partial class VerifierSettings +{ + const string locationObsolete = "ScrubberLocation is ignored; span scrubber ordering is engine determined. Use the overload without ScrubberLocation."; + + /// + /// Remove any lines containing any of from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubLinesContaining(StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(comparison, stringToMatch); + + /// + /// Remove any lines matching from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubLines(Func removeLine, ScrubberLocation location) => + ScrubLines(removeLine); + + /// + /// Remove any lines containing only whitespace from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubEmptyLines(ScrubberLocation location) => + ScrubEmptyLines(); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public static void ScrubInlineDateTimes( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimes(format, culture); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public static void ScrubInlineDateTimeOffsets( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimeOffsets(format, culture); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public static void ScrubInlineGuids(ScrubberLocation location) => + ScrubInlineGuids(); + + /// + /// Scrub lines with an optional replace. + /// can return the input to ignore the line, or return a different string to replace it. + /// + [Obsolete(locationObsolete)] + public static void ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(replaceLine); + + /// + /// Remove any lines containing any of from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubLinesContaining(ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, stringToMatch); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubMachineName(ScrubberLocation location) => + ScrubMachineName(); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public static void ScrubUserName(ScrubberLocation location) => + ScrubUserName(); + +#if NET6_0_OR_GREATER + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public static void ScrubInlineDates( + [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDates(format, culture); + +#endif +} diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs index 192538788..199a20841 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers.cs @@ -1,9 +1,26 @@ -namespace VerifyTests; +namespace VerifyTests; public partial class VerifySettings { internal Dictionary>>>? ExtensionMappedInstanceScrubbers = []; + internal Dictionary>? ExtensionMappedInstanceSpanScrubbers; + + /// + /// Add a that applies to verified files with a matching extension. + /// + internal void AddScrubber(string extension, Scrubber scrubber) + { + ExtensionMappedInstanceSpanScrubbers ??= []; + + if (!ExtensionMappedInstanceSpanScrubbers.TryGetValue(extension, out var values)) + { + ExtensionMappedInstanceSpanScrubbers[extension] = values = []; + } + + values.Add(scrubber); + } + /// /// Modify the resulting test content using custom code. /// @@ -43,55 +60,93 @@ public void AddScrubber(string extension, Action /// Remove the from the test results. /// - public void ScrubMachineName(string extension, ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(extension, UserMachineScrubber.Machine, location); + public void ScrubMachineName(string extension) => + AddScrubber(extension, UserMachineScrubber.MachineScrubber()); /// /// Remove the from the test results. /// - public void ScrubUserName(string extension, ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(extension, UserMachineScrubber.User, location); + public void ScrubUserName(string extension) => + AddScrubber(extension, UserMachineScrubber.UserScrubber()); /// /// Remove any lines containing any of from the test results. /// public void ScrubLinesContaining(string extension, StringComparison comparison, params string[] stringToMatch) => - ScrubLinesContaining(extension, comparison, ScrubberLocation.First, stringToMatch); + AddScrubber(extension, Scrubber.RemoveLinesContaining(comparison, stringToMatch)); /// - /// Remove any lines containing any of from the test results. + /// Replace inline s with a placeholder. /// - public void ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => - AddScrubber(extension, _ => _.RemoveLinesContaining(comparison, stringToMatch), location); + public void ScrubInlineGuids(string extension) => + AddScrubber(extension, GuidMatcher.Instance); /// - /// Replace inline s with a placeholder. + /// Remove any lines matching from the test results. /// - public void ScrubInlineGuids(string extension, ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(extension, GuidScrubber.ReplaceGuids, location); + public void ScrubLines(string extension, Func removeLine) => + AddScrubber(extension, Scrubber.RemoveLines(removeLine)); /// /// Remove any lines matching from the test results. + /// No per line string is allocated for span predicates. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ScrubLines(extension, (ReadOnlySpan<char> line) => ...). /// - public void ScrubLines(string extension, Func removeLine, ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(extension, _ => _.FilterLines(removeLine), location); + [OverloadResolutionPriority(-1)] + public void ScrubLines(string extension, LineMatch removeLine) => + AddScrubber(extension, Scrubber.RemoveLines(removeLine)); /// /// Scrub lines with an optional replace. /// can return the input to ignore the line, or return a different string to replace it. /// - public void ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(extension, _ => _.ReplaceLines(replaceLine), location); + public void ScrubLinesWithReplace(string extension, Func replaceLine) => + AddScrubber(extension, Scrubber.ReplaceLines(replaceLine)); + + /// + /// Scrub lines with an optional replace. + /// No per line string is allocated for span based replacers. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ScrubLinesWithReplace(extension, (ReadOnlySpan<char> line) => ...). + /// + [OverloadResolutionPriority(-1)] + public void ScrubLinesWithReplace(string extension, LineReplace replaceLine) => + AddScrubber(extension, Scrubber.ReplaceLines(replaceLine)); /// /// Remove any lines containing only whitespace from the test results. /// - public void ScrubEmptyLines(string extension, ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(extension, _ => _.RemoveEmptyLines(), location); + public void ScrubEmptyLines(string extension) => + AddScrubber(extension, Scrubber.RemoveEmptyLines()); /// - /// Remove any lines containing any of from the test results. + /// Replace every occurrence of with . + /// must be or . + /// + public void ScrubReplace(string extension, string find, string replacement, StringComparison comparison = StringComparison.Ordinal, bool requireWordBoundary = false) => + AddScrubber(extension, Scrubber.Replace(find, replacement, comparison, requireWordBoundary)); + + /// + /// Replace every occurrence of each Find with its Replacement. + /// At a given position the longest matching Find wins. + /// must be or . + /// + public void ScrubReplace(string extension, StringComparison comparison, bool requireWordBoundary, params (string Find, string Replacement)[] pairs) => + AddScrubber(extension, Scrubber.Replace(comparison, requireWordBoundary, pairs)); + + /// + /// Match candidate windows of text between and characters. + /// At each position the engine tries the longest window first. + /// + public void ScrubWindow(string extension, int minLength, int maxLength, WindowMatch matcher, bool requireWordBoundary = false) => + AddScrubber(extension, Scrubber.Window(minLength, maxLength, matcher, requireWordBoundary)); + + /// + /// Find matches using custom search logic. + /// : segments shorter than this are skipped (null scans everything). + /// : used for ordering only; null (unknown) runs before all known length scrubbers. /// - public void ScrubLinesContaining(string extension, ScrubberLocation location = ScrubberLocation.First, params string[] stringToMatch) => - ScrubLinesContaining(extension, StringComparison.OrdinalIgnoreCase, location, stringToMatch); -} \ No newline at end of file + public void ScrubMatch(string extension, SegmentMatch matcher, int? minLength = null, int? maxLength = null) => + AddScrubber(extension, Scrubber.Match(matcher, minLength, maxLength)); +} diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers_Obsoletes.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers_Obsoletes.cs new file mode 100644 index 000000000..5c3fc07de --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_ExtensionMappedInstanceScrubbers_Obsoletes.cs @@ -0,0 +1,61 @@ +namespace VerifyTests; + +public partial class VerifySettings +{ + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubMachineName(string extension, ScrubberLocation location) => + ScrubMachineName(extension); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubUserName(string extension, ScrubberLocation location) => + ScrubUserName(extension); + + /// + /// Remove any lines containing any of from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(extension, comparison, stringToMatch); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public void ScrubInlineGuids(string extension, ScrubberLocation location) => + ScrubInlineGuids(extension); + + /// + /// Remove any lines matching from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubLines(string extension, Func removeLine, ScrubberLocation location) => + ScrubLines(extension, removeLine); + + /// + /// Scrub lines with an optional replace. + /// can return the input to ignore the line, or return a different string to replace it. + /// + [Obsolete(locationObsolete)] + public void ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(extension, replaceLine); + + /// + /// Remove any lines containing only whitespace from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubEmptyLines(string extension, ScrubberLocation location) => + ScrubEmptyLines(extension); + + /// + /// Remove any lines containing any of from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubLinesContaining(string extension, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(extension, StringComparison.OrdinalIgnoreCase, stringToMatch); +} diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs index c6a79b090..281a099fa 100644 --- a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers.cs @@ -1,9 +1,25 @@ -namespace VerifyTests; +namespace VerifyTests; public partial class VerifySettings { internal List>>? InstanceScrubbers = []; + internal List? InstanceSpanScrubbers; + + // Cached merged set for the property value path, which runs once per + // serialized string value. Invalidated on registration. + internal EngineScrubberSet? PropertyValueSetCache; + + /// + /// Add a . + /// + internal void AddScrubber(Scrubber scrubber) + { + InstanceSpanScrubbers ??= []; + InstanceSpanScrubbers.Add(scrubber); + PropertyValueSetCache = null; + } + internal bool ScrubbersEnabled { get; private set; } = true; /// @@ -14,14 +30,14 @@ public partial class VerifySettings /// /// Remove the from the test results. /// - public void ScrubMachineName(ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(UserMachineScrubber.Machine, location); + public void ScrubMachineName() => + AddScrubber(UserMachineScrubber.MachineScrubber()); /// /// Remove the from the test results. /// - public void ScrubUserName(ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(UserMachineScrubber.User, location); + public void ScrubUserName() => + AddScrubber(UserMachineScrubber.UserScrubber()); /// /// Modify the resulting test content using custom code. @@ -64,25 +80,19 @@ public void AddScrubber(Action from the test results. /// public void ScrubLinesContaining(StringComparison comparison, params string[] stringToMatch) => - ScrubLinesContaining(comparison, ScrubberLocation.First, stringToMatch); - - /// - /// Remove any lines containing any of from the test results. - /// - public void ScrubLinesContaining(StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => - AddScrubber(_ => _.RemoveLinesContaining(comparison, stringToMatch), location); + AddScrubber(Scrubber.RemoveLinesContaining(comparison, stringToMatch)); /// /// Replace inline s with a placeholder. /// - public void ScrubInlineGuids(ScrubberLocation location = ScrubberLocation.First) + public void ScrubInlineGuids() { if (serialization.ScrubGuids == false) { throw new("ScrubGuids is disabled. Call .ScrubGuids() before calling .ScrubInlineGuids()."); } - AddScrubber(GuidScrubber.ReplaceGuids, location); + AddScrubber(GuidMatcher.Instance); } /// @@ -91,36 +101,36 @@ public void ScrubInlineGuids(ScrubberLocation location = ScrubberLocation.First) public void ScrubInlineDateTimes( [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) + Culture? culture = null) { if (serialization.ScrubDateTimes == false) { throw new("ScrubDateTimes is disabled. Call .ScrubDateTimes() before calling .ScrubInlineDateTimes()."); } - AddScrubber( - DateScrubber.BuildDateTimeScrubber(format, culture), - location); + foreach (var scrubber in DateMatchers.DateTimes(format, culture)) + { + AddScrubber(scrubber); + } } /// - /// Replace inline s with a placeholder. + /// Replace inline s with a placeholder. /// public void ScrubInlineDateTimeOffsets( [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) + Culture? culture = null) { if (serialization.ScrubDateTimes == false) { throw new("ScrubDateTimes is disabled. Call .ScrubDateTimes() before calling .ScrubInlineDateTimeOffsets()."); } - AddScrubber( - DateScrubber.BuildDateTimeOffsetScrubber(format, culture), - location); + foreach (var scrubber in DateMatchers.DateTimeOffsets(format, culture)) + { + AddScrubber(scrubber); + } } #if NET6_0_OR_GREATER @@ -130,17 +140,17 @@ public void ScrubInlineDateTimeOffsets( /// public void ScrubInlineDates( [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) + Culture? culture = null) { if (serialization.ScrubDateTimes == false) { throw new("ScrubDateTimes is disabled. Call .ScrubDateTimes() before calling .ScrubInlineDates()."); } - AddScrubber( - DateScrubber.BuildDateScrubber(format, culture), - location); + foreach (var scrubber in DateMatchers.Dates(format, culture)) + { + AddScrubber(scrubber); + } } #endif @@ -148,31 +158,75 @@ public void ScrubInlineDates( /// /// Remove any lines matching from the test results. /// - public void ScrubLines(Func removeLine, ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(_ => _.FilterLines(removeLine), location); + public void ScrubLines(Func removeLine) => + AddScrubber(Scrubber.RemoveLines(removeLine)); + + /// + /// Remove any lines matching from the test results. + /// No per line string is allocated for span predicates. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ScrubLines((ReadOnlySpan<char> line) => ...). + /// + [OverloadResolutionPriority(-1)] + public void ScrubLines(LineMatch removeLine) => + AddScrubber(Scrubber.RemoveLines(removeLine)); /// /// Scrub lines with an optional replace. /// can return the input to ignore the line, or return a different string to replace it. /// - public void ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(_ => _.ReplaceLines(replaceLine), location); + public void ScrubLinesWithReplace(Func replaceLine) => + AddScrubber(Scrubber.ReplaceLines(replaceLine)); + + /// + /// Scrub lines with an optional replace. + /// No per line string is allocated for span based replacers. + /// Use an explicitly typed lambda parameter to select this overload, + /// e.g. ScrubLinesWithReplace((ReadOnlySpan<char> line) => ...). + /// + [OverloadResolutionPriority(-1)] + public void ScrubLinesWithReplace(LineReplace replaceLine) => + AddScrubber(Scrubber.ReplaceLines(replaceLine)); /// /// Remove any lines containing only whitespace from the test results. /// - public void ScrubEmptyLines(ScrubberLocation location = ScrubberLocation.First) => - AddScrubber(_ => _.RemoveEmptyLines(), location); + public void ScrubEmptyLines() => + AddScrubber(Scrubber.RemoveEmptyLines()); /// /// Remove any lines containing any of from the test results. /// public void ScrubLinesContaining(params string[] stringToMatch) => - ScrubLinesContaining(ScrubberLocation.First, stringToMatch); + ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, stringToMatch); /// - /// Remove any lines containing any of from the test results. + /// Replace every occurrence of with . + /// must be or . + /// + public void ScrubReplace(string find, string replacement, StringComparison comparison = StringComparison.Ordinal, bool requireWordBoundary = false) => + AddScrubber(Scrubber.Replace(find, replacement, comparison, requireWordBoundary)); + + /// + /// Replace every occurrence of each Find with its Replacement. + /// At a given position the longest matching Find wins. + /// must be or . + /// + public void ScrubReplace(StringComparison comparison, bool requireWordBoundary, params (string Find, string Replacement)[] pairs) => + AddScrubber(Scrubber.Replace(comparison, requireWordBoundary, pairs)); + + /// + /// Match candidate windows of text between and characters. + /// At each position the engine tries the longest window first. + /// + public void ScrubWindow(int minLength, int maxLength, WindowMatch matcher, bool requireWordBoundary = false) => + AddScrubber(Scrubber.Window(minLength, maxLength, matcher, requireWordBoundary)); + + /// + /// Find matches using custom search logic. + /// : segments shorter than this are skipped (null scans everything). + /// : used for ordering only; null (unknown) runs before all known length scrubbers. /// - public void ScrubLinesContaining(ScrubberLocation location = ScrubberLocation.First, params string[] stringToMatch) => - ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, location, stringToMatch); -} \ No newline at end of file + public void ScrubMatch(SegmentMatch matcher, int? minLength = null, int? maxLength = null) => + AddScrubber(Scrubber.Match(matcher, minLength, maxLength)); +} diff --git a/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers_Obsoletes.cs b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers_Obsoletes.cs new file mode 100644 index 000000000..f37870534 --- /dev/null +++ b/src/Verify/Serialization/Scrubbers/VerifySettings_InstanceScrubbers_Obsoletes.cs @@ -0,0 +1,99 @@ +namespace VerifyTests; + +public partial class VerifySettings +{ + const string locationObsolete = "ScrubberLocation is ignored; span scrubber ordering is engine determined. Use the overload without ScrubberLocation."; + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubMachineName(ScrubberLocation location) => + ScrubMachineName(); + + /// + /// Remove the from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubUserName(ScrubberLocation location) => + ScrubUserName(); + + /// + /// Remove any lines containing any of from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubLinesContaining(StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(comparison, stringToMatch); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public void ScrubInlineGuids(ScrubberLocation location) => + ScrubInlineGuids(); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public void ScrubInlineDateTimes( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] + string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimes(format, culture); + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public void ScrubInlineDateTimeOffsets( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] + string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimeOffsets(format, culture); + + /// + /// Remove any lines matching from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubLines(Func removeLine, ScrubberLocation location) => + ScrubLines(removeLine); + + /// + /// Scrub lines with an optional replace. + /// can return the input to ignore the line, or return a different string to replace it. + /// + [Obsolete(locationObsolete)] + public void ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(replaceLine); + + /// + /// Remove any lines containing only whitespace from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubEmptyLines(ScrubberLocation location) => + ScrubEmptyLines(); + + /// + /// Remove any lines containing any of from the test results. + /// + [Obsolete(locationObsolete)] + public void ScrubLinesContaining(ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(StringComparison.OrdinalIgnoreCase, stringToMatch); + +#if NET6_0_OR_GREATER + + /// + /// Replace inline s with a placeholder. + /// + [Obsolete(locationObsolete)] + public void ScrubInlineDates( + [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDates(format, culture); + +#endif +} diff --git a/src/Verify/Serialization/VerifierSettings.cs b/src/Verify/Serialization/VerifierSettings.cs index 93e2e96b9..ae9799c20 100644 --- a/src/Verify/Serialization/VerifierSettings.cs +++ b/src/Verify/Serialization/VerifierSettings.cs @@ -167,6 +167,10 @@ internal static void Reset() addAttachments = true; excludedTargets = null; GlobalScrubbers.Clear(); + ExtensionMappedGlobalScrubbers.Clear(); + GlobalSpanScrubbers.Clear(); + ExtensionMappedGlobalSpanScrubbers.Clear(); + EngineScrubberSet.InvalidateGlobalCache(); GlobalIgnoredParameters = null; GlobalIgnoreConstructorParameters = false; } diff --git a/src/Verify/Serialization/VerifyJsonWriter.cs b/src/Verify/Serialization/VerifyJsonWriter.cs index e4bbe941e..bb0587b3e 100644 --- a/src/Verify/Serialization/VerifyJsonWriter.cs +++ b/src/Verify/Serialization/VerifyJsonWriter.cs @@ -57,8 +57,18 @@ public override void WriteValue(char value) base.WriteRawValue(value); } - public void WriteRawValueWithScrubbers(string value) => - WriteRawValueWithScrubbers(value.AsSpan()); + public void WriteRawValueWithScrubbers(string value) + { + if (value.Length == 0) + { + WriteRawValueIfNoStrict(value); + return; + } + + // The string overload returns the same instance when nothing changed, so + // the value the caller already holds is not duplicated + WriteRawValueIfNoStrict(ApplyScrubbers.ApplyForPropertyValue(value, settings, Counter)); + } public void WriteRawValueWithScrubbers(CharSpan value) { @@ -100,7 +110,21 @@ public override void WriteValue(string? value) return; } - WriteValue(value.AsSpan()); + if (value.Length == 0) + { + WriteRawValueIfNoStrict(value); + return; + } + + if (Counter.TryConvert(value.AsSpan(), out var result)) + { + WriteRawValueIfNoStrict(result); + return; + } + + // The string overload returns the same instance when nothing changed, so + // the value the caller already holds is not duplicated + WriteScrubbed(ApplyScrubbers.ApplyForPropertyValue(value, settings, Counter)); } public override void WriteValue(StringBuilder? value) => @@ -121,7 +145,11 @@ public override void WriteValue(CharSpan value) return; } - value = ApplyScrubbers.ApplyForPropertyValue(value, settings, Counter); + WriteScrubbed(ApplyScrubbers.ApplyForPropertyValue(value, settings, Counter)); + } + + void WriteScrubbed(CharSpan value) + { if (settings.StrictJson) { base.WriteValue(value); diff --git a/src/Verify/SettingsTask_Scrubbing.cs b/src/Verify/SettingsTask_Scrubbing.cs index 947666d54..6de2afef6 100644 --- a/src/Verify/SettingsTask_Scrubbing.cs +++ b/src/Verify/SettingsTask_Scrubbing.cs @@ -1,4 +1,4 @@ -namespace VerifyTests; +namespace VerifyTests; public partial class SettingsTask { @@ -10,6 +10,22 @@ public SettingsTask DisableScrubbers() return this; } + /// + [Pure] + internal SettingsTask AddScrubber(Scrubber scrubber) + { + CurrentSettings.AddScrubber(scrubber); + return this; + } + + /// + [Pure] + internal SettingsTask AddScrubber(string extension, Scrubber scrubber) + { + CurrentSettings.AddScrubber(extension, scrubber); + return this; + } + /// [Pure] public SettingsTask AddScrubber(Action scrubber, ScrubberLocation location = ScrubberLocation.First) @@ -26,11 +42,11 @@ public SettingsTask AddScrubber(string extension, Action scrubber return this; } - /// + /// [Pure] - public SettingsTask ScrubInlineGuids(ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubInlineGuids() { - CurrentSettings.ScrubInlineGuids(location); + CurrentSettings.ScrubInlineGuids(); return this; } @@ -50,11 +66,11 @@ public SettingsTask ScrubNumericIds() return this; } - /// + /// [Pure] - public SettingsTask ScrubInlineGuids(string extension, ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubInlineGuids(string extension) { - CurrentSettings.ScrubInlineGuids(extension, location); + CurrentSettings.ScrubInlineGuids(extension); return this; } @@ -74,72 +90,69 @@ public SettingsTask ScrubDateTimes() return this; } - /// + /// [Pure] public SettingsTask ScrubInlineDateTimes( [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) + Culture? culture = null) { - CurrentSettings.ScrubInlineDateTimes(format, culture, location); + CurrentSettings.ScrubInlineDateTimes(format, culture); return this; } - /// + /// [Pure] public SettingsTask ScrubInlineDateTimeOffsets( [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) + Culture? culture = null) { - CurrentSettings.ScrubInlineDateTimeOffsets(format, culture, location); + CurrentSettings.ScrubInlineDateTimeOffsets(format, culture); return this; } #if NET6_0_OR_GREATER - /// + /// [Pure] public SettingsTask ScrubInlineDates( [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, - Culture? culture = null, - ScrubberLocation location = ScrubberLocation.First) + Culture? culture = null) { - CurrentSettings.ScrubInlineDates(format, culture, location); + CurrentSettings.ScrubInlineDates(format, culture); return this; } #endif - /// + /// [Pure] - public SettingsTask ScrubMachineName(ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubMachineName() { - CurrentSettings.ScrubMachineName(location); + CurrentSettings.ScrubMachineName(); return this; } - /// + /// [Pure] - public SettingsTask ScrubMachineName(string extension, ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubMachineName(string extension) { - CurrentSettings.ScrubMachineName(extension, location); + CurrentSettings.ScrubMachineName(extension); return this; } - /// + /// [Pure] - public SettingsTask ScrubUserName(ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubUserName() { - CurrentSettings.ScrubUserName(location); + CurrentSettings.ScrubUserName(); return this; } - /// + /// [Pure] - public SettingsTask ScrubUserName(string extension,ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubUserName(string extension) { - CurrentSettings.ScrubUserName(extension, location); + CurrentSettings.ScrubUserName(extension); return this; } @@ -159,67 +172,87 @@ public SettingsTask ScrubLinesContaining(string extension, StringComparison comp return this; } - /// + /// + [Pure] + public SettingsTask ScrubLines(Func removeLine) + { + CurrentSettings.ScrubLines(removeLine); + return this; + } + + /// + [OverloadResolutionPriority(-1)] + [Pure] + public SettingsTask ScrubLines(LineMatch removeLine) + { + CurrentSettings.ScrubLines(removeLine); + return this; + } + + /// [Pure] - public SettingsTask ScrubLinesContaining(StringComparison comparison, ScrubberLocation location = ScrubberLocation.First, params string[] stringToMatch) + public SettingsTask ScrubLines(string extension, Func removeLine) { - CurrentSettings.ScrubLinesContaining(comparison, location, stringToMatch); + CurrentSettings.ScrubLines(extension, removeLine); return this; } - /// + /// + [OverloadResolutionPriority(-1)] [Pure] - public SettingsTask ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location = ScrubberLocation.First, params string[] stringToMatch) + public SettingsTask ScrubLines(string extension, LineMatch removeLine) { - CurrentSettings.ScrubLinesContaining(extension, comparison, location, stringToMatch); + CurrentSettings.ScrubLines(extension, removeLine); return this; } - /// + /// [Pure] - public SettingsTask ScrubLines(Func removeLine, ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubLinesWithReplace(Func replaceLine) { - CurrentSettings.ScrubLines(removeLine, location); + CurrentSettings.ScrubLinesWithReplace(replaceLine); return this; } - /// + /// + [OverloadResolutionPriority(-1)] [Pure] - public SettingsTask ScrubLines(string extension, Func removeLine, ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubLinesWithReplace(LineReplace replaceLine) { - CurrentSettings.ScrubLines(extension, removeLine, location); + CurrentSettings.ScrubLinesWithReplace(replaceLine); return this; } - /// + /// [Pure] - public SettingsTask ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubLinesWithReplace(string extension, Func replaceLine) { - CurrentSettings.ScrubLinesWithReplace(replaceLine, location); + CurrentSettings.ScrubLinesWithReplace(extension, replaceLine); return this; } - /// + /// + [OverloadResolutionPriority(-1)] [Pure] - public SettingsTask ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubLinesWithReplace(string extension, LineReplace replaceLine) { - CurrentSettings.ScrubLinesWithReplace(extension, replaceLine, location); + CurrentSettings.ScrubLinesWithReplace(extension, replaceLine); return this; } - /// + /// [Pure] - public SettingsTask ScrubEmptyLines(ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubEmptyLines() { - CurrentSettings.ScrubEmptyLines(location); + CurrentSettings.ScrubEmptyLines(); return this; } - /// + /// [Pure] - public SettingsTask ScrubEmptyLines(string extension, ScrubberLocation location = ScrubberLocation.First) + public SettingsTask ScrubEmptyLines(string extension) { - CurrentSettings.ScrubEmptyLines(extension, location); + CurrentSettings.ScrubEmptyLines(extension); return this; } @@ -231,19 +264,67 @@ public SettingsTask ScrubLinesContaining(params string[] stringToMatch) return this; } - /// + /// + [Pure] + public SettingsTask ScrubReplace(string find, string replacement, StringComparison comparison = StringComparison.Ordinal, bool requireWordBoundary = false) + { + CurrentSettings.ScrubReplace(find, replacement, comparison, requireWordBoundary); + return this; + } + + /// + [Pure] + public SettingsTask ScrubReplace(string extension, string find, string replacement, StringComparison comparison = StringComparison.Ordinal, bool requireWordBoundary = false) + { + CurrentSettings.ScrubReplace(extension, find, replacement, comparison, requireWordBoundary); + return this; + } + + /// + [Pure] + public SettingsTask ScrubReplace(StringComparison comparison, bool requireWordBoundary, params (string Find, string Replacement)[] pairs) + { + CurrentSettings.ScrubReplace(comparison, requireWordBoundary, pairs); + return this; + } + + /// + [Pure] + public SettingsTask ScrubReplace(string extension, StringComparison comparison, bool requireWordBoundary, params (string Find, string Replacement)[] pairs) + { + CurrentSettings.ScrubReplace(extension, comparison, requireWordBoundary, pairs); + return this; + } + + /// + [Pure] + public SettingsTask ScrubWindow(int minLength, int maxLength, WindowMatch matcher, bool requireWordBoundary = false) + { + CurrentSettings.ScrubWindow(minLength, maxLength, matcher, requireWordBoundary); + return this; + } + + /// + [Pure] + public SettingsTask ScrubWindow(string extension, int minLength, int maxLength, WindowMatch matcher, bool requireWordBoundary = false) + { + CurrentSettings.ScrubWindow(extension, minLength, maxLength, matcher, requireWordBoundary); + return this; + } + + /// [Pure] - public SettingsTask ScrubLinesContaining(ScrubberLocation location = ScrubberLocation.First, params string[] stringToMatch) + public SettingsTask ScrubMatch(SegmentMatch matcher, int? minLength = null, int? maxLength = null) { - CurrentSettings.ScrubLinesContaining(location, stringToMatch); + CurrentSettings.ScrubMatch(matcher, minLength, maxLength); return this; } - /// + /// [Pure] - public SettingsTask ScrubLinesContaining(string extension, ScrubberLocation location = ScrubberLocation.First, params string[] stringToMatch) + public SettingsTask ScrubMatch(string extension, SegmentMatch matcher, int? minLength = null, int? maxLength = null) { - CurrentSettings.ScrubLinesContaining(extension, location, stringToMatch); + CurrentSettings.ScrubMatch(extension, matcher, minLength, maxLength); return this; } -} \ No newline at end of file +} diff --git a/src/Verify/SettingsTask_Scrubbing_Obsoletes.cs b/src/Verify/SettingsTask_Scrubbing_Obsoletes.cs new file mode 100644 index 000000000..4692d5423 --- /dev/null +++ b/src/Verify/SettingsTask_Scrubbing_Obsoletes.cs @@ -0,0 +1,138 @@ +namespace VerifyTests; + +public partial class SettingsTask +{ + const string locationObsolete = "ScrubberLocation is ignored; span scrubber ordering is engine determined. Use the overload without ScrubberLocation."; + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubInlineGuids(ScrubberLocation location) => + ScrubInlineGuids(); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubInlineGuids(string extension, ScrubberLocation location) => + ScrubInlineGuids(extension); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubInlineDateTimes( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] + string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimes(format, culture); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubInlineDateTimeOffsets( + [StringSyntax(StringSyntaxAttribute.DateTimeFormat)] + string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDateTimeOffsets(format, culture); + +#if NET6_0_OR_GREATER + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubInlineDates( + [StringSyntax(StringSyntaxAttribute.DateOnlyFormat)] string format, + Culture? culture, + ScrubberLocation location) => + ScrubInlineDates(format, culture); + +#endif + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubMachineName(ScrubberLocation location) => + ScrubMachineName(); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubMachineName(string extension, ScrubberLocation location) => + ScrubMachineName(extension); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubUserName(ScrubberLocation location) => + ScrubUserName(); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubUserName(string extension, ScrubberLocation location) => + ScrubUserName(extension); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLinesContaining(StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(comparison, stringToMatch); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLinesContaining(string extension, StringComparison comparison, ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(extension, comparison, stringToMatch); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLines(Func removeLine, ScrubberLocation location) => + ScrubLines(removeLine); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLines(string extension, Func removeLine, ScrubberLocation location) => + ScrubLines(extension, removeLine); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLinesWithReplace(Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(replaceLine); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLinesWithReplace(string extension, Func replaceLine, ScrubberLocation location) => + ScrubLinesWithReplace(extension, replaceLine); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubEmptyLines(ScrubberLocation location) => + ScrubEmptyLines(); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubEmptyLines(string extension, ScrubberLocation location) => + ScrubEmptyLines(extension); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLinesContaining(ScrubberLocation location, params string[] stringToMatch) => + ScrubLinesContaining(stringToMatch); + + /// + [Obsolete(locationObsolete)] + [Pure] + public SettingsTask ScrubLinesContaining(string extension, ScrubberLocation location, params string[] stringToMatch) + { + CurrentSettings.ScrubLinesContaining(extension, StringComparison.OrdinalIgnoreCase, stringToMatch); + return this; + } +} diff --git a/src/Verify/TempDirectory.cs b/src/Verify/TempDirectory.cs index 4b56fa14a..510bef0d1 100644 --- a/src/Verify/TempDirectory.cs +++ b/src/Verify/TempDirectory.cs @@ -63,22 +63,8 @@ public static void Init() }, after: () => asyncPaths.Value = null); - VerifierSettings.GlobalScrubbers.Add((scrubber, _, _) => - { - var pathsValue = asyncPaths.Value; - if (pathsValue == null) - { - return; - } - - lock (pathsLock) - { - foreach (var path in pathsValue) - { - scrubber.Replace(path, "{TempDirectory}"); - } - } - }); + VerifierSettings.AddScrubber( + TempPathScrubber.Build(asyncPaths, pathsLock, "{TempDirectory}", RootDirectory.Length)); Cleanup(); } diff --git a/src/Verify/TempFile.cs b/src/Verify/TempFile.cs index 612745f37..54f4043b8 100644 --- a/src/Verify/TempFile.cs +++ b/src/Verify/TempFile.cs @@ -65,22 +65,8 @@ public static void Init() }, after: () => asyncPaths.Value = null); - VerifierSettings.GlobalScrubbers.Add((scrubber, _, _) => - { - var pathsValue = asyncPaths.Value; - if (pathsValue == null) - { - return; - } - - lock (pathsLock) - { - foreach (var path in pathsValue) - { - scrubber.Replace(path, "{TempFile}"); - } - } - }); + VerifierSettings.AddScrubber( + TempPathScrubber.Build(asyncPaths, pathsLock, "{TempFile}", RootDirectory.Length)); Cleanup(); } diff --git a/src/Verify/TempPathScrubber.cs b/src/Verify/TempPathScrubber.cs new file mode 100644 index 000000000..1de6316f1 --- /dev/null +++ b/src/Verify/TempPathScrubber.cs @@ -0,0 +1,51 @@ +namespace VerifyTests; + +// The scrubber shared by TempDirectory and TempFile. Each tracks the paths it has +// created for the current async context, so the paths are only known at scrub time +// and cannot be expressed as fixed Replace pairs. +static class TempPathScrubber +{ + // Replaces any tracked path with token. The leftmost match in the segment wins, + // matching the engine's leftmost first scan. + // All tracked paths start with the root directory, so segments shorter than it + // are skipped without invoking the matcher. + public static Scrubber Build( + AsyncLocal?> paths, + object pathsLock, + string token, + int rootDirectoryLength) => + Scrubber.Match( + (CharSpan segment, Counter _, IReadOnlyDictionary _, out int index, out int length, out string? replacement) => + { + index = -1; + length = 0; + replacement = token; + var pathsValue = paths.Value; + if (pathsValue == null) + { + return false; + } + + lock (pathsLock) + { + foreach (var path in pathsValue) + { + var found = segment.IndexOf(path.AsSpan(), StringComparison.Ordinal); + if (found < 0) + { + continue; + } + + if (index < 0 || + found < index) + { + index = found; + length = path.Length; + } + } + } + + return index >= 0; + }, + minLength: rootDirectoryLength); +} diff --git a/src/Verify/Verifier/InnerVerifier_Xml.cs b/src/Verify/Verifier/InnerVerifier_Xml.cs index b04b44406..b36d20ab4 100644 --- a/src/Verify/Verifier/InnerVerifier_Xml.cs +++ b/src/Verify/Verifier/InnerVerifier_Xml.cs @@ -103,13 +103,12 @@ Task VerifyXml(XContainer? target) string ConvertValue(string value) { - var span = value.AsSpan(); - if (counter.TryConvert(span, out var result)) + if (counter.TryConvert(value.AsSpan(), out var result)) { return result; } - return ApplyScrubbers.ApplyForPropertyValue(span, settings, counter); + return ApplyScrubbers.ApplyForPropertyValue(value, settings, counter); } void ScrubAttributes(XElement node, SerializationSettings serialization) @@ -132,14 +131,14 @@ void ScrubAttributes(XElement node, SerializationSettings serialization) continue; } - var span = attribute.Value.AsSpan(); - if (counter.TryConvert(span, out var result)) + var value = attribute.Value; + if (counter.TryConvert(value.AsSpan(), out var result)) { attribute.Value = result; } else { - attribute.Value = ApplyScrubbers.ApplyForPropertyValue(span, settings, counter); + attribute.Value = ApplyScrubbers.ApplyForPropertyValue(value, settings, counter); } } } diff --git a/src/Verify/VerifySettings.cs b/src/Verify/VerifySettings.cs index cff19d559..338033d27 100644 --- a/src/Verify/VerifySettings.cs +++ b/src/Verify/VerifySettings.cs @@ -36,6 +36,18 @@ public VerifySettings(VerifySettings? settings) .ToDictionary(_ => _.Key, _ => _.Value.ToList()); } + if (settings.InstanceSpanScrubbers != null) + { + InstanceSpanScrubbers = [..settings.InstanceSpanScrubbers]; + } + + if (settings.ExtensionMappedInstanceSpanScrubbers != null) + { + // Deep copy: the inner lists are mutated in place by AddScrubber. + ExtensionMappedInstanceSpanScrubbers = settings.ExtensionMappedInstanceSpanScrubbers + .ToDictionary(_ => _.Key, _ => _.Value.ToList()); + } + diffEnabled = settings.diffEnabled; MethodName = settings.MethodName; TypeName = settings.TypeName; diff --git a/src/VerifyCore.slnf b/src/VerifyCore.slnf index e521ab1c5..859263d30 100644 --- a/src/VerifyCore.slnf +++ b/src/VerifyCore.slnf @@ -2,6 +2,7 @@ "solution": { "path": "Verify.slnx", "projects": [ + "ApplyScrubbersTests\\ApplyScrubbersTests.csproj", "DisableScrubbersTests\\DisableScrubbersTests.csproj", "Benchmarks\\Benchmarks.csproj", "RawTempUsage\\RawTempUsage.csproj", diff --git a/src/appveyor.yml b/src/appveyor.yml index b8ae3c54c..016253767 100644 --- a/src/appveyor.yml +++ b/src/appveyor.yml @@ -28,6 +28,7 @@ before_build: - dotnet tool restore --tool-manifest src/.config/dotnet-tools.json build_script: - dotnet build src/Verify.slnx --configuration Release --verbosity minimal +- dotnet test src/ApplyScrubbersTests --configuration Release --no-build --no-restore --verbosity minimal - dotnet test src/DeterministicTests --configuration Release --no-build --no-restore --verbosity minimal - dotnet test src/FSharpTests --configuration Release --no-build --no-restore --verbosity minimal - dotnet test src/StaticSettingsTests --configuration Release --no-build --no-restore --verbosity minimal