diff --git a/Dummies.UnitTests/AnyPatternTests.cs b/Dummies.UnitTests/AnyPatternTests.cs index 6eef52b2..44541716 100644 --- a/Dummies.UnitTests/AnyPatternTests.cs +++ b/Dummies.UnitTests/AnyPatternTests.cs @@ -26,6 +26,18 @@ private static string Display(string value) { return "\"" + value.Replace("\t", "\\t").Replace("\n", "\\n").Replace("\r", "\\r") + "\""; } + // The oracle again, as a predicate: does the real .NET engine compile this pattern at all? Used to assert the + // rejection taxonomy — a pattern the engine accepts must never be reported as malformed (ArgumentException). + private static bool IsCompiledByTheRealEngine(string pattern) { + try { + _ = new Regex(pattern); + + return true; + } catch (ArgumentException) { + return false; + } + } + #endregion [Theory(DisplayName = "Every generated value is fully matched by the real .NET regex engine.")] @@ -50,6 +62,14 @@ private static string Display(string value) { [InlineData(@"(?<10>ab)")] // a multi-digit group number stays valid [InlineData(@"(?xy)")] // a named group whose name merely contains digits stays valid [InlineData(@"^a$|^b$")] + [InlineData(@"^^abc")] // a run of boundary anchors is a no-op, exactly as in the real engine + [InlineData(@"abc$$")] + [InlineData(@"^*abc")] // a quantifier on a zero-width anchor is a no-op too + [InlineData(@"abc$*")] + [InlineData(@"^{2}xy")] + [InlineData(@"^?abc$?")] + [InlineData(@"[-[x]]")] // a leading '-' is an ordinary hyphen, not a subtraction operator + [InlineData(@"[-[abc]]")] [InlineData(@"[\d]{3}")] [InlineData(@"[-a-z]{2}")] [InlineData(@"[a-z-]{2}")] @@ -157,6 +177,73 @@ public void AnchorsAreNoOps() { } } + [Fact(DisplayName = "Repeated and quantified boundary anchors are no-ops, consistently for '^' and '$'.")] + public void RepeatedAndQuantifiedAnchorsAreNoOps() { + // '^^abc' was already accepted while the symmetric 'abc$$' was refused — an avoidable asymmetry, now closed. + // Repeating or quantifying a zero-width boundary assertion never changes which strings match, so all of + // these are no-ops, exactly as the real engine treats them. + Check.That(Any.StringMatching(@"^^abc$$").Generate()).IsEqualTo("abc"); + Check.That(Any.StringMatching(@"^*abc$*").Generate()).IsEqualTo("abc"); + Check.That(Any.StringMatching(@"^{2}abc").Generate()).IsEqualTo("abc"); + Check.That(Any.StringMatching(@"^?abc$?").Generate()).IsEqualTo("abc"); + } + + [Fact(DisplayName = "A pattern that overruns the generation ceiling fails with an honest message, naming no false cause.")] + public void OverLimitPatternFailsWithoutAFalseCause() { + // '(a{1000}){1000}' deterministically asks for 1,000,000 characters — every quantifier is bounded, there is + // no unbounded quantifier at all. The pattern parses fine (a resource ceiling is not a satisfiability + // conflict); generation is what overruns the limit, and the message must not blame a quantifier that is absent. + AnyPattern generator = Any.StringMatching(@"(a{1000}){1000}"); + + AnyGenerationException caught = Assert.Throws(() => generator.Generate()); + Check.That(caught.Message).Contains("generation limit"); + Check.That(caught.Message).Contains("bounded quantifiers"); // the real cause is offered + Check.That(caught.Message).Not.Contains("is expanding without bound"); // the old, false assertion is gone + } + + [Fact(DisplayName = "A negated class that excludes the whole printable universe is refused as unsupported, not malformed.")] + public void NegatedClassExcludingTheUniverseIsUnsupported() { + // Well-formed and regular — the real engine accepts both — but no printable-ASCII character survives the + // negation, so Dummies cannot draw a value. That is a universe limit (unsupported), not a caller mistake + // (malformed): the pattern is not broken, Dummies simply does not reach outside printable ASCII. + Check.ThatCode(() => Any.StringMatching(@"[^\x20-\x7E]")).Throws(); + Check.ThatCode(() => Any.StringMatching(@"[^\s\S]")).Throws(); + + UnsupportedRegexException caught = Assert.Throws(() => Any.StringMatching(@"[^\x20-\x7E]")); + Check.That(caught.Message).Contains("printable ASCII"); + } + + [Theory(DisplayName = "A pattern the real .NET engine accepts is never rejected as malformed — it generates, or is refused as unsupported.")] + [InlineData(@"^^abc")] + [InlineData(@"abc$$")] + [InlineData(@"^*abc")] + [InlineData(@"abc$*")] + [InlineData(@"^{2}xy")] + [InlineData(@"^?a$?")] + [InlineData(@"[-[x]]")] + [InlineData(@"[a-[x]]")] // subtraction (member precedes): .NET-valid, refused as UNSUPPORTED, not malformed + [InlineData(@"[ab-[b]]")] + [InlineData(@"[^\x20-\x7E]")] // negated-empty: .NET-valid, refused as UNSUPPORTED + [InlineData(@"[^\s\S]")] + [InlineData(@"(a{1000}){1000}")] // over-limit: .NET-valid, fails at generation time, never as malformed + public void PatternsAcceptedByTheRealEngineAreNeverMalformed(string pattern) { + // The advertised taxonomy (see RegexParser's summary): ArgumentException == "the real engine rejects this + // pattern as malformed". So a pattern the real engine COMPILES must never surface here as ArgumentException — + // Dummies must either generate a matching value, or refuse it as UnsupportedRegexException, or fail the + // generation itself. This guards the whole channel, not just the individual edges #210 corrected. + Assert.True(IsCompiledByTheRealEngine(pattern), $"test precondition: /{pattern}/ must be accepted by .NET"); + + try { + Any.WithSeed(1).StringMatching(pattern).Generate(); + } catch (ArgumentException e) { + Assert.Fail($"/{pattern}/ is accepted by the real engine but Dummies rejected it as malformed: {e.Message}"); + } catch (UnsupportedRegexException) { + // acceptable: refused as unsupported (a construct or universe Dummies declines), not as malformed + } catch (AnyGenerationException) { + // acceptable: a resource-limit overrun, not a verdict that the pattern is malformed + } + } + [Fact(DisplayName = "A Regex with IgnoreCase generates either case.")] public void IgnoreCaseHonoured() { Regex pattern = new("^[a-z]{5}$", RegexOptions.IgnoreCase); @@ -218,8 +305,12 @@ public void NotGeneratableConstructsAreRefused() { Check.ThatCode(() => Any.StringMatching(@"x(^a)")).Throws(); Check.ThatCode(() => Any.StringMatching(@"(a$)x")).Throws(); - // .NET class subtraction removes a nested class; parsing '-[' as members would generate outside the set. + // .NET class subtraction removes a nested class; parsing '-[' as members would generate outside the set. It + // is subtraction only when a base member precedes the '-[' — after a range ('[a-z-[aeiou]]') or a single + // member ('[a-[x]]', '[ab-[b]]'). A leading '-[' is an ordinary hyphen and IS accepted (see the oracle theory). Check.ThatCode(() => Any.StringMatching(@"[a-z-[aeiou]]")).Throws(); + Check.ThatCode(() => Any.StringMatching(@"[a-[x]]")).Throws(); + Check.ThatCode(() => Any.StringMatching(@"[ab-[b]]")).Throws(); // IgnorePatternWhitespace changes how the pattern text itself is read: "^A B$" matches "AB", not "A B". Check.ThatCode(() => Any.StringMatching(new Regex("^A B$", RegexOptions.IgnorePatternWhitespace))).Throws(); diff --git a/Dummies/RegexNode.cs b/Dummies/RegexNode.cs index d6a55c78..1076b772 100644 --- a/Dummies/RegexNode.cs +++ b/Dummies/RegexNode.cs @@ -8,9 +8,10 @@ namespace Dummies; /// /// The carrier of a single generation: the seeded random generator to draw from, and the buffer the nodes write -/// into. enforces a hard length ceiling so a nested unbounded quantifier -/// ((a+)+ and the like) can never expand without bound — the value is built directly, never generated then -/// retried, but the buffer is still guarded. +/// into. enforces a hard length ceiling so no pattern can expand the buffer without bound — +/// whether through a nested unbounded quantifier ((a+)+ and the like) or through bounded quantifiers whose +/// product is very large ((a{1000}){1000}). The value is built directly, never generated then retried, but +/// the buffer is still guarded. /// internal sealed class RegexGenerationContext { @@ -30,7 +31,7 @@ internal RegexGenerationContext(Random random, int limit) { internal void Append(char character) { if (_builder.Length >= _limit) { - throw new AnyGenerationException($"The pattern produced a string longer than the {_limit}-character generation limit; a nested unbounded quantifier is expanding without bound."); + throw new AnyGenerationException($"The pattern produced a string longer than the {_limit}-character generation limit. This ceiling guards against runaway expansion; a pattern can reach it either through a nested unbounded quantifier (such as \"(a+)+\") or through bounded quantifiers whose product is very large (such as \"(a{{1000}}){{1000}}\")."); } _builder.Append(character); diff --git a/Dummies/RegexParser.cs b/Dummies/RegexParser.cs index b76b5991..79955fb9 100644 --- a/Dummies/RegexParser.cs +++ b/Dummies/RegexParser.cs @@ -14,14 +14,15 @@ namespace Dummies; /// ? * + {n} {n,} {n,m} (a lazy ? marker is accepted and ignored — it changes matching order, never /// which strings match; possessive quantifiers do not exist in .NET and are rejected); alternation; grouping /// (capturing, non-capturing and named — the name is validated as the real engine would, then ignored); the dot; -/// and the anchors ^ $ at the start -/// and end of the pattern or of a top-level alternation branch (no-ops there, since a whole matching string is -/// generated; anywhere else they are refused, because the pattern could never be matched by a whole generated -/// string). A brace that does not form a well-formed quantifier is a literal, as in the real engine, and groups -/// may nest at most 256 levels deep. A well-formed but non-regular or out-of-scope construct — a lookaround, a -/// backreference, a balancing group (it pops the capture stack, the backreference family), a Unicode category, a -/// word boundary, an atomic group (its first-branch commit is not language-equivalent to plain alternation), a -/// class subtraction — is refused with an +/// and the anchors ^ $ at the start and end of the pattern or of a top-level alternation branch — including +/// a run of them (^^, $$) or a quantified one (^*, $?), all no-ops there since a whole +/// matching string is generated; anywhere else they are refused, because the pattern could never be matched by a +/// whole generated string. A brace that does not form a well-formed quantifier is a literal, as in the real engine, +/// and groups may nest at most 256 levels deep. A well-formed but non-regular or out-of-scope construct — a +/// lookaround, a backreference, a balancing group (it pops the capture stack, the backreference family), a Unicode +/// category, a word boundary, an atomic group (its first-branch commit is not language-equivalent to plain +/// alternation), a class subtraction, or a negated class that excludes the whole printable-ASCII universe Dummies +/// draws from — is refused with an /// rather than silently mis-generated; a malformed pattern (including an /// escape the real engine rejects) raises an . /// @@ -128,12 +129,15 @@ private RegexNode ParseSequence() { List parts = new(); while (!AtEnd && Peek() != '|' && Peek() != ')') { // Anchors are no-ops for a whole-string generator, but only where they are guaranteed to match: - // '^' at the start and '$' at the end of the pattern or of a top-level alternation branch. Anywhere - // else ('a^', '$a', inside a group) the pattern can never be matched by a whole generated string, - // so it is refused instead of silently mis-generated. + // '^' at the start and '$' at the end of the pattern or of a top-level alternation branch. A run of + // them ('^^', '$$') and a quantified one ('^*', '$?', '^{2}') are equally no-ops there — the real + // engine accepts and matches all of these — so they are consumed and ignored. Anywhere else ('a^', + // '$a', inside a group) the pattern can never be matched by a whole generated string, so it is + // refused instead of silently mis-generated. if (Peek() == '^') { if (_depth > 0 || parts.Count > 0) { throw Unsupported("an anchor '^' away from the start of the pattern or of a top-level alternation branch", _index); } _index++; + SkipAnchorQuantifier(); continue; } @@ -141,7 +145,10 @@ private RegexNode ParseSequence() { if (Peek() == '$') { int position = _index; _index++; - if (_depth > 0 || (!AtEnd && Peek() != '|')) { throw Unsupported("an anchor '$' away from the end of the pattern or of a top-level alternation branch", position); } + SkipAnchorQuantifier(); + // A '$' is a no-op only at the very end: what follows must be end-of-pattern, a branch bar, or + // another end-anchor '$'. Inside a group, or before anything else, it can never match a whole string. + if (_depth > 0 || (!AtEnd && Peek() != '|' && Peek() != '$')) { throw Unsupported("an anchor '$' away from the end of the pattern or of a top-level alternation branch", position); } continue; } @@ -152,6 +159,17 @@ private RegexNode ParseSequence() { return parts.Count == 1 ? parts[0] : new RegexSequence(parts.ToArray()); } + /// + /// Consumes a quantifier applied to a boundary anchor (^*, $?, ^{2,3}, a lazy marker + /// included) when one is present. Repeating a zero-width no-op leaves it a no-op — the real engine accepts and + /// matches these — so the quantifier is read and discarded, never turned into a repeat node. + /// + private void SkipAnchorQuantifier() { + if (AtEnd) { return; } // an anchor at the very end carries no quantifier + if (TryReadQuantifier() is null) { return; } + if (!AtEnd && Peek() == '?') { _index++; } // a lazy marker on a no-op is immaterial + } + private RegexNode ParseQuantified() { RegexNode atom = ParseAtom(); if (AtEnd) { return atom; } @@ -417,6 +435,7 @@ private char ReadOctalTail(int firstDigit) { } private RegexNode ParseClass() { + int position = _index; _index++; // consume '[' bool negated = Eat('^'); HashSet set = new(); @@ -431,9 +450,11 @@ private RegexNode ParseClass() { } first = false; - // .NET's class subtraction ([a-z-[aeiou]]) removes a nested class; parsing the '-[' as members - // would close the class early and generate values outside it, so the construct is refused. - if (Peek() == '-' && PeekAt(1) == '[') { throw Unsupported("a character-class subtraction '[…-[…]]'", _index); } + // .NET's class subtraction ([a-z-[aeiou]]) removes a nested class; parsing the '-[' as members would + // close the class early and generate values outside it, so the construct is refused. But '-[' is + // subtraction only when a base member precedes it: the real engine reads a leading '-' (as in '[-[x]]') + // as an ordinary hyphen, so it is refused here only once the set already holds a member. + if (Peek() == '-' && PeekAt(1) == '[' && set.Count > 0) { throw Unsupported("a character-class subtraction '[…-[…]]'", _index); } if (Peek() == '\\' && IsClassShorthand(PeekAt(1))) { _index++; // consume '\' AddClassShorthand(set, Next()); @@ -442,6 +463,9 @@ private RegexNode ParseClass() { } char low = ReadClassChar(); + // A '-[' immediately after a member is subtraction as well ('[a-[x]]'): the real engine never reads it + // as a range whose upper bound is '[', so intercepting it here can never turn away a valid range. + if (!AtEnd && Peek() == '-' && PeekAt(1) == '[') { throw Unsupported("a character-class subtraction '[…-[…]]'", _index); } if (!AtEnd && Peek() == '-' && PeekAt(1) != ']' && PeekAt(1) != '\0') { _index++; // consume '-' char high = ReadClassChar(); @@ -456,7 +480,13 @@ private RegexNode ParseClass() { if (_ignoreCase) { set = ExpandCase(set); } char[] choices = negated ? RegexAlphabet.Negate(set) : set.ToArray(); - if (choices.Length == 0) { throw Malformed("character class matches no printable character"); } + // Only a negated class can end up empty — a plain class always holds the member it just read. Such a class + // is well-formed and regular (the real engine accepts it), but it excludes every character Dummies draws + // from, so it is refused as unsupported (a universe limit), not as malformed. Routing it through Malformed + // would claim the caller wrote a broken pattern for one the real engine compiles. + if (choices.Length == 0) { + throw new UnsupportedRegexException($"The regular expression pattern \"{_pattern}\" uses a negated character class that excludes every character Dummies can generate (printable ASCII U+0020 to U+007E) at position {position}, which Dummies cannot generate from. It draws values from printable ASCII; express the requirement with characters inside that range, or generate the value another way."); + } return new RegexCharacters(choices); }