|
6 | 6 | keeps runs reproducible on shared CI runners -- this layer guards |
7 | 7 | against regressions; exploratory fuzzing happened during review. |
8 | 8 | """ |
| 9 | +import json |
| 10 | +from pathlib import Path |
| 11 | + |
| 12 | +import pytest |
9 | 13 | from hypothesis import given, settings |
10 | 14 | from hypothesis import strategies as st |
11 | 15 |
|
12 | 16 | from nameparser import Lexicon, Policy, parse |
13 | 17 | from nameparser._pipeline import run |
14 | 18 | from nameparser._pipeline._state import ParseState |
| 19 | +from nameparser._types import AmbiguityKind, Role |
15 | 20 |
|
16 | 21 | _ALPHABET = st.sampled_from( |
17 | 22 | 'abcdefgh ABC 12 .,،,\'"()«»\U0001f600éñßЖ-') |
18 | 23 |
|
| 24 | +# Same one-JSON-name-per-line convention test_locales.py reads; see the |
| 25 | +# note there on why tools/ is not imported. |
| 26 | +_FORK_CORPUS = [ |
| 27 | + json.loads(line) |
| 28 | + for line in (Path(__file__).parents[2] / "tools" / "differential" |
| 29 | + / "corpus.jsonl").read_text().splitlines() |
| 30 | + if line.strip() |
| 31 | +] |
| 32 | + |
19 | 33 |
|
20 | 34 | @given(st.text(alphabet=_ALPHABET, max_size=200)) |
21 | 35 | @settings(max_examples=300, deadline=None, derandomize=True) |
@@ -76,3 +90,86 @@ def test_every_original_char_is_accounted_for(text: str) -> None: |
76 | 90 | continue |
77 | 91 | raise AssertionError( |
78 | 92 | f"char {ch!r} at {i} in {text!r} is unaccounted for") |
| 93 | + |
| 94 | + |
| 95 | +_NAME_ROLES = (Role.GIVEN, Role.MIDDLE, Role.FAMILY) |
| 96 | + |
| 97 | + |
| 98 | +def _fork_count(state: ParseState) -> int: |
| 99 | + return sum(a.kind is AmbiguityKind.PARTICLE_OR_GIVEN |
| 100 | + for a in state.ambiguities) |
| 101 | + |
| 102 | + |
| 103 | +def test_a_leading_ambiguous_particle_is_reported_once_and_only_once( |
| 104 | +) -> None: |
| 105 | + """PARTICLE_OR_GIVEN is the one kind two stages emit: _group takes |
| 106 | + the particle branch when a title shifts the particle off index 0 and |
| 107 | + the chain claims something ("Dr. Van Johnson"), _assign takes the |
| 108 | + given branch when it stays a lone leading piece ("Van Johnson"). |
| 109 | + Each reports the side it decides -- see the ParseState docstring -- |
| 110 | + but they coordinate only through _group's `j > k + 1` guard, which |
| 111 | + mirrors _assign's reachability by hand. Nothing checked the mirror. |
| 112 | +
|
| 113 | + The shape that separates them is the one real-name corpora have |
| 114 | + least reason to contain: a suffix straight after the particle, where |
| 115 | + the chain is a no-op and only _assign should speak. Generate it, over |
| 116 | + every ambiguous particle, so a vocabulary addition is covered too. |
| 117 | +
|
| 118 | + Deliberately asserts the COUNT and not which stage spoke. Whether a |
| 119 | + given fork was decided by a vocabulary merge or by position is not |
| 120 | + recoverable from the finished parse -- 'Dr. aan Johnson Jr.' and |
| 121 | + 'أبو بكر أحمد' end with the same roles and the same tags, and only |
| 122 | + one is a fork -- so any reconstruction here would have to |
| 123 | + re-implement _group rather than check it. |
| 124 | + """ |
| 125 | + lex = Lexicon.default() |
| 126 | + # bound-given prefixes are excluded, not overlooked: 'abu' is both |
| 127 | + # an ambiguous particle and a bound given prefix, so whether it |
| 128 | + # forks depends on whether the bound join fired -- a second rule, |
| 129 | + # covered by the case corpus rather than by this sweep |
| 130 | + particles = sorted(lex.particles_ambiguous - lex.bound_given_names) |
| 131 | + assert particles, "no ambiguous particles to exercise" |
| 132 | + failures = [] |
| 133 | + for particle in particles: |
| 134 | + for lead in ("", "Dr. ", "Dr. Ann "): |
| 135 | + for body in ("", "Johnson ", "Johnson Smith "): |
| 136 | + for tail in ("", "Jr.", "MD", "III"): |
| 137 | + text = f"{lead}{particle} {body}{tail}".strip() |
| 138 | + state = run(ParseState(original=text, |
| 139 | + lexicon=lex, policy=Policy())) |
| 140 | + names = [t for t in state.tokens |
| 141 | + if t.role in _NAME_ROLES] |
| 142 | + # Leading = no name part precedes it. With a given |
| 143 | + # name in front ("Dr. Ann van Johnson") the particle |
| 144 | + # sits mid-name, where nothing has to choose -- the |
| 145 | + # decision-not-a-word rule, so no report. And a lone |
| 146 | + # name part is the whole name, not a coin flip. |
| 147 | + leads = bool(names) and "vocab:particle-ambiguous" \ |
| 148 | + in names[0].tags |
| 149 | + want = 1 if leads and len(names) >= 2 else 0 |
| 150 | + got = _fork_count(state) |
| 151 | + if got != want: |
| 152 | + failures.append( |
| 153 | + f"{text!r}: {got} report(s), expected {want} " |
| 154 | + f"({len(names)} name tokens, " |
| 155 | + f"leading={leads})") |
| 156 | + assert not failures, ( |
| 157 | + f"{len(failures)} shape(s) disagree:\n" + "\n".join(failures[:15])) |
| 158 | + |
| 159 | + |
| 160 | +@pytest.mark.parametrize("text", _FORK_CORPUS) |
| 161 | +def test_a_fork_is_never_reported_twice_on_a_real_name(text: str) -> None: |
| 162 | + state = run(ParseState(original=text, lexicon=Lexicon.default(), |
| 163 | + policy=Policy())) |
| 164 | + assert _fork_count(state) <= 1 |
| 165 | + |
| 166 | + |
| 167 | +@given(st.text(alphabet=_ALPHABET, max_size=120)) |
| 168 | +@settings(max_examples=400, deadline=None, derandomize=True) |
| 169 | +def test_particle_fork_is_never_double_reported(text: str) -> None: |
| 170 | + # The half of the invariant that needs no reconstruction: whatever |
| 171 | + # the two emitters decide, they must not both fire on one parse. |
| 172 | + state = run(ParseState(original=text, lexicon=Lexicon.default(), |
| 173 | + policy=Policy())) |
| 174 | + assert _fork_count(state) <= 1, ( |
| 175 | + f"{text!r} reported the same fork more than once") |
0 commit comments