Skip to content

Commit 7fbcd9e

Browse files
derek73claude
andcommitted
Pin the two-emitter invariant for PARTICLE_OR_GIVEN
_group and _assign each report one branch of this fork and coordinate only through _group's `j > k + 1` guard, which mirrors _assign's reachability by hand. Nothing checked the mirror: reachability drift in either stage would over- or under-report, and only a case pinning that exact name would notice. Generate the shapes rather than trusting a corpus. The boundary is a suffix straight after the particle ("Dr. Van Jr."), where the chain is a no-op and only _assign should speak -- a shape plausible-name corpora have little reason to hold, and indeed the 486-name differential corpus does not. Swept over every ambiguous particle, so a vocabulary addition is covered without editing the test. Verified to fail in both directions by loosening and tightening the guard. Asserts the count, not which stage spoke, on purpose: whether a fork was decided by a vocabulary merge or by position is NOT recoverable from the finished parse. 'Dr. aan Johnson Jr.' and 'أبو بكر أحمد' end with the same roles and the same tags, and only one is a fork -- so a reconstruction would have to re-implement _group instead of checking it. That is also why the two emitters stay where the decisions are made. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent d9a07f7 commit 7fbcd9e

1 file changed

Lines changed: 97 additions & 0 deletions

File tree

tests/v2/test_properties.py

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,30 @@
66
keeps runs reproducible on shared CI runners -- this layer guards
77
against regressions; exploratory fuzzing happened during review.
88
"""
9+
import json
10+
from pathlib import Path
11+
12+
import pytest
913
from hypothesis import given, settings
1014
from hypothesis import strategies as st
1115

1216
from nameparser import Lexicon, Policy, parse
1317
from nameparser._pipeline import run
1418
from nameparser._pipeline._state import ParseState
19+
from nameparser._types import AmbiguityKind, Role
1520

1621
_ALPHABET = st.sampled_from(
1722
'abcdefgh ABC 12 .,،,\'"()«»‏‏\U0001f600éñßЖ-')
1823

24+
# Same one-JSON-name-per-line convention test_locales.py reads; see the
25+
# note there on why tools/ is not imported.
26+
_FORK_CORPUS = [
27+
json.loads(line)
28+
for line in (Path(__file__).parents[2] / "tools" / "differential"
29+
/ "corpus.jsonl").read_text().splitlines()
30+
if line.strip()
31+
]
32+
1933

2034
@given(st.text(alphabet=_ALPHABET, max_size=200))
2135
@settings(max_examples=300, deadline=None, derandomize=True)
@@ -76,3 +90,86 @@ def test_every_original_char_is_accounted_for(text: str) -> None:
7690
continue
7791
raise AssertionError(
7892
f"char {ch!r} at {i} in {text!r} is unaccounted for")
93+
94+
95+
_NAME_ROLES = (Role.GIVEN, Role.MIDDLE, Role.FAMILY)
96+
97+
98+
def _fork_count(state: ParseState) -> int:
99+
return sum(a.kind is AmbiguityKind.PARTICLE_OR_GIVEN
100+
for a in state.ambiguities)
101+
102+
103+
def test_a_leading_ambiguous_particle_is_reported_once_and_only_once(
104+
) -> None:
105+
"""PARTICLE_OR_GIVEN is the one kind two stages emit: _group takes
106+
the particle branch when a title shifts the particle off index 0 and
107+
the chain claims something ("Dr. Van Johnson"), _assign takes the
108+
given branch when it stays a lone leading piece ("Van Johnson").
109+
Each reports the side it decides -- see the ParseState docstring --
110+
but they coordinate only through _group's `j > k + 1` guard, which
111+
mirrors _assign's reachability by hand. Nothing checked the mirror.
112+
113+
The shape that separates them is the one real-name corpora have
114+
least reason to contain: a suffix straight after the particle, where
115+
the chain is a no-op and only _assign should speak. Generate it, over
116+
every ambiguous particle, so a vocabulary addition is covered too.
117+
118+
Deliberately asserts the COUNT and not which stage spoke. Whether a
119+
given fork was decided by a vocabulary merge or by position is not
120+
recoverable from the finished parse -- 'Dr. aan Johnson Jr.' and
121+
'أبو بكر أحمد' end with the same roles and the same tags, and only
122+
one is a fork -- so any reconstruction here would have to
123+
re-implement _group rather than check it.
124+
"""
125+
lex = Lexicon.default()
126+
# bound-given prefixes are excluded, not overlooked: 'abu' is both
127+
# an ambiguous particle and a bound given prefix, so whether it
128+
# forks depends on whether the bound join fired -- a second rule,
129+
# covered by the case corpus rather than by this sweep
130+
particles = sorted(lex.particles_ambiguous - lex.bound_given_names)
131+
assert particles, "no ambiguous particles to exercise"
132+
failures = []
133+
for particle in particles:
134+
for lead in ("", "Dr. ", "Dr. Ann "):
135+
for body in ("", "Johnson ", "Johnson Smith "):
136+
for tail in ("", "Jr.", "MD", "III"):
137+
text = f"{lead}{particle} {body}{tail}".strip()
138+
state = run(ParseState(original=text,
139+
lexicon=lex, policy=Policy()))
140+
names = [t for t in state.tokens
141+
if t.role in _NAME_ROLES]
142+
# Leading = no name part precedes it. With a given
143+
# name in front ("Dr. Ann van Johnson") the particle
144+
# sits mid-name, where nothing has to choose -- the
145+
# decision-not-a-word rule, so no report. And a lone
146+
# name part is the whole name, not a coin flip.
147+
leads = bool(names) and "vocab:particle-ambiguous" \
148+
in names[0].tags
149+
want = 1 if leads and len(names) >= 2 else 0
150+
got = _fork_count(state)
151+
if got != want:
152+
failures.append(
153+
f"{text!r}: {got} report(s), expected {want} "
154+
f"({len(names)} name tokens, "
155+
f"leading={leads})")
156+
assert not failures, (
157+
f"{len(failures)} shape(s) disagree:\n" + "\n".join(failures[:15]))
158+
159+
160+
@pytest.mark.parametrize("text", _FORK_CORPUS)
161+
def test_a_fork_is_never_reported_twice_on_a_real_name(text: str) -> None:
162+
state = run(ParseState(original=text, lexicon=Lexicon.default(),
163+
policy=Policy()))
164+
assert _fork_count(state) <= 1
165+
166+
167+
@given(st.text(alphabet=_ALPHABET, max_size=120))
168+
@settings(max_examples=400, deadline=None, derandomize=True)
169+
def test_particle_fork_is_never_double_reported(text: str) -> None:
170+
# The half of the invariant that needs no reconstruction: whatever
171+
# the two emitters decide, they must not both fire on one parse.
172+
state = run(ParseState(original=text, lexicon=Lexicon.default(),
173+
policy=Policy()))
174+
assert _fork_count(state) <= 1, (
175+
f"{text!r} reported the same fork more than once")

0 commit comments

Comments
 (0)