From 4f5b95633bd7ee3ff97c93ca45c9f2e6bb682c38 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 20:53:37 -0700 Subject: [PATCH 01/36] Add the glued-honorific tail vocabulary (#308) --- nameparser/config/suffixes.py | 72 +++++++++++++++++++----- tests/v2/cases.py | 16 ++++++ tools/differential/corpus_cjk.jsonl | 2 + tools/differential/expected_changes.toml | 2 +- 4 files changed, 77 insertions(+), 15 deletions(-) diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index c592b8d..0cef66f 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -37,17 +37,19 @@ 'שליט"א', # honorific for a living rabbi, ASCII quote 'שליט״א', # same, U+05F4 gershayim - # #307: CJK postnominal honorifics and degrees, whole-token - # matched. That reaches the spaced forms directly, and glued - # forms in exactly ONE shape: surname+honorific (김씨; 王先生 - # under the zh pack), where segmentation splits off the surname - # before suffix classification and the honorific is what remains. - # A glued honorific after a GIVEN name (김민준씨, 王小明先生) and - # glued kana (山田太郎様; 田中さん additionally ships no さん - # entry at all) stay out of reach, tracked as #308. - # Self-selecting like the Korean surnames: a Han or hangul entry - # can only ever match CJK text. Vetting per the мл/ст standard - # above -- three entries worth naming: + # #307/#308: CJK postnominal honorifics and degrees. Matched + # whole-token here, which reaches the SPACED forms; the GLUED + # forms (田中さん, 김민준씨) are reached by the peel in + # _pipeline/_script_segment.py, which splits a listed tail off the + # last name token and hands the piece back to this vocabulary -- + # so every GLUED_HONORIFICS entry below is also an entry here, + # asserted at the foot of this module. + # Self-selecting like the Korean surnames: a Han, kana or hangul + # entry can only ever match CJK text. Vetting per the мл/ст + # standard above, and note the two bars are different -- an entry + # can be safe in the spaced position and unsafe glued, never the + # other way round, because the spaced position is a token boundary + # its writer drew: # - 氏: a rare Japanese surname reading exists, but a bare # trailing 氏 after a name is the news-style honorific, and a # 氏-surnamed person writes it FIRST. @@ -57,17 +59,18 @@ # final position both are vanishingly rare in practice. # - 博士: an attested Japanese given name (ひろし, Hiroshi) -- # the doctorate reading vastly dominates the spaced trailing - # position this set matches, and a glued 田中博士 is untouched. + # position this set matches. # Bare hangul 선생/교수 are deliberately absent (only the -님 # honorific forms ship): the bare forms read as common nouns as - # readily as address terms. Further candidates (여사, 太太, 殿) - # wait on the same case-by-case argument. + # readily as address terms. Further candidates (여사, 太太) wait on + # the same case-by-case argument. '씨', # ko Mr./Ms. -- standardly spaced in Korean orthography '박사', # ko doctorate holder ("Dr.") '선생님', # ko teacher/respected elder '교수님', # ko professor (honorific form) '군', # ko young man ("Master") '양', # ko young woman ("Miss") + '님', # ko -- the bare honorific of online/formal address '先生', # zh Mr. / ja teacher-master -- honorific in both '女士', # zh Ms./Madam '小姐', # zh Miss @@ -75,12 +78,47 @@ '教授', # zh+ja professor, shared Han '様', # ja formal Mr./Ms. (the mail-addressing honorific) '氏', # ja news-style Mr. (田中氏) + '殿', # ja formal, official/rank-flavored + 'さん', # ja the everyday honorific, kana + 'さま', # ja the kana spelling of 様 + 'くん', # ja the kana spelling of 君 + 'ちゃん', # ja familiar/diminutive } """ Post-nominal pieces that are not acronyms. The parser does not remove periods when matching against these pieces. +""" +GLUED_HONORIFICS = { + # #308: the entries above that may also be peeled off the END of a + # name token -- 田中さん, 山田太郎様, 김민준씨. A separate set, not + # SUFFIX_NOT_ACRONYMS reused, because the glued position has no + # token boundary to lean on: the vetting question is not "is this + # a name?" but "can this END a name?", and only entries that can + # never end one belong here. + # kana -- name-final never, in any of the four: + 'さん', 'さま', 'くん', 'ちゃん', + # kanji: + '様', '先生', '殿', '教授', + # hangul (the -님 compounds too: longest-first peels 선생님 off + # 김선생님 whole, rather than leaving 김선생 to segment into a + # family 김 and a given 선생): + '씨', '님', '선생님', '교수님', +} +""" + +The subset of :data:`SUFFIX_NOT_ACRONYMS` a name token may end WITH, peeled +off as its own token before segmentation (#308). Deliberately harsher than +the spaced set, entry by entry: + +* 양, 군 -- 김지양 and 김지군 are given names ending in these syllables, and + 양 is a top-tier surname besides. Spaced entries stay. +* 君 -- 王君 is a complete Chinese name (君 is a common given-name final). +* 氏 -- 王氏 is a historical name form ("the Wang woman"). +* 博士 -- glued 田中博士 IS Tanaka Hiroshi, an attested given name. +* bare 선생, 교수 -- read as common nouns; only the -님 forms ship at all. + """ SUFFIX_ACRONYMS_AMBIGUOUS = { # Suffix acronyms that also commonly work as given-name nicknames on @@ -769,4 +807,10 @@ "an ambiguous acronym must not also be a suffix word (the word " \ "branch bypasses its period gate): " \ f"{sorted(SUFFIX_ACRONYMS_AMBIGUOUS & SUFFIX_NOT_ACRONYMS)}" +# The peel splits its tail off as a TOKEN and suffix classification is +# what claims it downstream, so a tail that is not also a suffix word +# would split the name and then leave the piece sitting in it. +assert GLUED_HONORIFICS <= SUFFIX_NOT_ACRONYMS, \ + "GLUED_HONORIFICS must stay a subset of SUFFIX_NOT_ACRONYMS: " \ + f"{sorted(GLUED_HONORIFICS - SUFFIX_NOT_ACRONYMS)}" assert_normalized("suffix", SUFFIX_ACRONYMS | SUFFIX_NOT_ACRONYMS) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 167f980..8c2ed4d 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -733,6 +733,22 @@ def __post_init__(self) -> None: notes="the spaced 様 of forms and databases; the glued " "mail-addressing form 山田太郎様 is #308's mechanism, " "out of reach of whole-token matching"), + Case("ja_san_spaced", "田中 さん", + {"family": "田中", "suffix": "さん"}, + classification="fix(#308)", + notes="the kana honorifics ship as suffix vocabulary so the " + "glued peel has somewhere to hand its tail; spaced " + "recognition falls out of the same entry -- until this " + "change さん read as the given name under the " + "family-first default"), + Case("ko_honorific_nim_spaced", "김민준 님", + {"family": "김", "given": "민준", "suffix": "님"}, + classification="fix(#308)", + notes="님 is new in both sets -- #307 shipped only the -님 " + "compounds 선생님/교수님. Standardly glued in online " + "address, spaced too, and never the end of a Korean " + "given name, which is what qualifies it for the " + "harsher glued vetting as well"), Case("ko_honorific_glued_via_segmentation", "김씨", {"family": "김", "suffix": "씨"}, classification="fix(#307)", diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index 4de386d..2d21638 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -25,6 +25,7 @@ "王先生" "王小明 先生" "王小明先生" +"田中 さん" "田中 太郎 様" "田中『ハナ』花子" "諸葛亮" @@ -36,6 +37,7 @@ "高橋一郎" "김 민준" "김민준" +"김민준 님" "김민준 박사" "김민준 박사 씨" "김민준 씨" diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index 3e4fea9..548f925 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -203,7 +203,7 @@ issue = "fix(cjk-honorific-suffix) postnominal honorifics recognized, compoundin # {first, last, suffix} shape (v1 held the whole token in first, so # last moves too), which is field-honest even though that rule's # prose describes the two-token Latin case. -name_regex = "(?:^| )(?:씨|박사|선생님|교수님|군|양|先生|女士|小姐|博士|教授|様|氏)$" +name_regex = "(?:^| )(?:씨|박사|선생님|교수님|군|양|님|先生|女士|小姐|博士|教授|様|氏|殿|さん|さま|くん|ちゃん)$" fields = ["first", "middle", "last", "suffix"] [[change]] From 2b0f007b609873662005196db48827b307fb0f77 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 21:11:30 -0700 Subject: [PATCH 02/36] Vet the glued tail set against real surname data (#308) --- nameparser/config/suffixes.py | 48 +++++++++++++++++++++-------- tests/v2/cases.py | 8 +++++ tools/differential/corpus_cjk.jsonl | 1 + 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index 0cef66f..37810b3 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -39,8 +39,8 @@ # #307/#308: CJK postnominal honorifics and degrees. Matched # whole-token here, which reaches the SPACED forms; the GLUED - # forms (田中さん, 김민준씨) are reached by the peel in - # _pipeline/_script_segment.py, which splits a listed tail off the + # forms (田中さん, 김민준씨) are reached by the peel #308 adds to + # _pipeline/_script_segment.py: it splits a listed tail off the # last name token and hands the piece back to this vocabulary -- # so every GLUED_HONORIFICS entry below is also an entry here, # asserted at the foot of this module. @@ -60,6 +60,12 @@ # - 博士: an attested Japanese given name (ひろし, Hiroshi) -- # the doctorate reading vastly dominates the spaced trailing # position this set matches. + # - 殿: some ninety Japanese surnames end in it (鵜殿, 真殿), which + # is what keeps it out of GLUED_HONORIFICS below -- but the + # surname-LEADS argument that clears 양/군 clears it here too. + # - 님: a single hangul syllable, the risk class 양/군 are in, but + # it has no hanja reading at all, so it cannot sit inside a + # Sino-Korean given name. # Bare hangul 선생/교수 are deliberately absent (only the -님 # honorific forms ship): the bare forms read as common nouns as # readily as address terms. Further candidates (여사, 太太) wait on @@ -97,28 +103,43 @@ # token boundary to lean on: the vetting question is not "is this # a name?" but "can this END a name?", and only entries that can # never end one belong here. - # kana -- name-final never, in any of the four: + # kana -- name-final never, in any of the four, and the kana/kanji + # split is itself a vetting result: くん ships where 君 cannot, + # since 王君 is a complete Chinese name while hiragana spells no + # name character at all. 'さん', 'さま', 'くん', 'ちゃん', - # kanji: - '様', '先生', '殿', '教授', - # hangul (the -님 compounds too: longest-first peels 선생님 off + # Han -- two-character honorifics with no name-final reading in + # either language, plus 様. 殿 is deliberately absent though it + # ships spaced above; see the exclusions below. + '様', '先生', '教授', '女士', '小姐', + # hangul -- the -님 compounds too: longest-first peels 선생님 off # 김선생님 whole, rather than leaving 김선생 to segment into a - # family 김 and a given 선생): - '씨', '님', '선생님', '교수님', + # family 김 and a given 선생. 박사 is safe glued where its Han twin + # 博士 is not: that collision is Japanese (博士 = ひろし) and the + # hangul spelling carries none of it. + '씨', '님', '선생님', '교수님', '박사', } """ The subset of :data:`SUFFIX_NOT_ACRONYMS` a name token may end WITH, peeled off as its own token before segmentation (#308). Deliberately harsher than -the spaced set, entry by entry: +the spaced set, because a glued tail has no writer-drawn token boundary to +lean on -- these entries ship spaced only: * 양, 군 -- 김지양 and 김지군 are given names ending in these syllables, and - 양 is a top-tier surname besides. Spaced entries stay. -* 君 -- 王君 is a complete Chinese name (君 is a common given-name final). + 양 is a top-tier surname besides. * 氏 -- 王氏 is a historical name form ("the Wang woman"). * 博士 -- glued 田中博士 IS Tanaka Hiroshi, an attested given name. +* 殿 -- some ninety Japanese surnames END in it, 鵜殿 (Udono) and 真殿 + (Madono) with four-figure populations, so peeling it would cut a real + family name in two. Spaced 殿 is safe for the reason 양/군 are: a + 殿-surnamed person's name LEADS, and the suffix gate is trailing-only. * bare 선생, 교수 -- read as common nouns; only the -님 forms ship at all. +君 is in NEITHER set: 王君 is a complete Chinese name (君 is a common +given-name final), so the honorific reading never gets the benefit of the +doubt -- while its kana spelling くん ships glued, above. + """ SUFFIX_ACRONYMS_AMBIGUOUS = { # Suffix acronyms that also commonly work as given-name nicknames on @@ -809,7 +830,10 @@ f"{sorted(SUFFIX_ACRONYMS_AMBIGUOUS & SUFFIX_NOT_ACRONYMS)}" # The peel splits its tail off as a TOKEN and suffix classification is # what claims it downstream, so a tail that is not also a suffix word -# would split the name and then leave the piece sitting in it. +# would split the name and then leave the piece sitting in it. The +# reverse direction is deliberately unguarded: a suffix word that is +# not a tail is the ordinary case, and an empty tail set is inert +# rather than wrong. assert GLUED_HONORIFICS <= SUFFIX_NOT_ACRONYMS, \ "GLUED_HONORIFICS must stay a subset of SUFFIX_NOT_ACRONYMS: " \ f"{sorted(GLUED_HONORIFICS - SUFFIX_NOT_ACRONYMS)}" diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 8c2ed4d..14ef330 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -741,6 +741,14 @@ def __post_init__(self) -> None: "recognition falls out of the same entry -- until this " "change さん read as the given name under the " "family-first default"), + Case("ja_dono_spaced", "田中 殿", + {"family": "田中", "suffix": "殿"}, + classification="fix(#308)", + notes="殿 waited on an argument in #307 and gets one here: " + "spaced it is safe for the reason 양/군 are -- a " + "殿-surnamed person's name LEADS, and the suffix gate " + "is trailing-only -- while glued it would cut 鵜殿 and " + "真殿 in two, so it ships spaced only"), Case("ko_honorific_nim_spaced", "김민준 님", {"family": "김", "given": "민준", "suffix": "님"}, classification="fix(#308)", diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index 2d21638..b3ecb3a 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -27,6 +27,7 @@ "王小明先生" "田中 さん" "田中 太郎 様" +"田中 殿" "田中『ハナ』花子" "諸葛亮" "阿明" From b9c58a29a1da890ba6fc4a92464165b348130ce9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 21:15:01 -0700 Subject: [PATCH 03/36] Add Lexicon.honorific_tails, the glued-peel vocabulary (#308) --- nameparser/_config_shim.py | 4 ++++ nameparser/_lexicon.py | 14 +++++++++++++- tests/v2/test_lexicon.py | 10 ++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index 331809c..9481ac3 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -986,6 +986,7 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: removal path. """ from nameparser.config.maiden_markers import MAIDEN_MARKERS + from nameparser.config.suffixes import GLUED_HONORIFICS from nameparser.config.surnames import KOREAN_SURNAMES acronyms = frozenset(self.suffix_acronyms) particles = frozenset(self.prefixes) @@ -1068,6 +1069,9 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: # Unwrapped where maiden_markers above is wrapped: this # module is born frozen (#293), so no wrap surnames=KOREAN_SURNAMES, + # likewise no v1 manager: the glued-honorific tail set is + # 2.1 behavior (#308), so it rides in the snapshot only + honorific_tails=frozenset(GLUED_HONORIFICS), # TupleManager is dict[str, object] (v1 parity: values were # never statically str-typed); every real entry is a str, # same assumption _DelimiterManager's sentinel lookup makes diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 7190e73..d3612a6 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -24,6 +24,7 @@ "titles", "given_name_titles", "suffix_acronyms", "suffix_words", "suffix_acronyms_ambiguous", "particles", "particles_ambiguous", "conjunctions", "bound_given_names", "maiden_markers", "surnames", + "honorific_tails", ) #: (marker, base, why) triples. Each marker narrows how entries of its @@ -335,6 +336,15 @@ class Lexicon: #: (:data:`~nameparser.config.surnames.KOREAN_SURNAMES`); Chinese #: surnames ship in locales.ZH because Han segmentation is opt-in. surnames: frozenset[str] = frozenset() + #: Honorifics that may be peeled off the END of a name token + #: (#308), matched longest-first: 田中さん splits into 田中 and さん + #: before anything else reads the name. Every entry must also be a + #: :attr:`suffix_words` entry -- the peeled tail is claimed by + #: suffix classification like any other post-nominal. Strictly + #: narrower than the spaced honorific vocabulary, since a glued + #: tail has no token boundary to lean on. Full default list: + #: :data:`~nameparser.config.suffixes.GLUED_HONORIFICS`. + honorific_tails: frozenset[str] = frozenset() #: Lowercase word -> exact-cased replacement used by capitalized() #: ("phd" -> "Ph.D."). Pair-valued: change it with #: dataclasses.replace(), not add()/remove(); read it as a mapping @@ -577,7 +587,8 @@ def _default_lexicon() -> Lexicon: from nameparser.config.maiden_markers import MAIDEN_MARKERS from nameparser.config.prefixes import NON_FIRST_NAME_PREFIXES, PREFIXES from nameparser.config.suffixes import ( - SUFFIX_ACRONYMS, SUFFIX_ACRONYMS_AMBIGUOUS, SUFFIX_NOT_ACRONYMS, + GLUED_HONORIFICS, SUFFIX_ACRONYMS, SUFFIX_ACRONYMS_AMBIGUOUS, + SUFFIX_NOT_ACRONYMS, ) from nameparser.config.surnames import KOREAN_SURNAMES from nameparser.config.titles import FIRST_NAME_TITLES, TITLES @@ -602,6 +613,7 @@ def _default_lexicon() -> Lexicon: # surnames.py is born frozen (#293) -- no call-site wrap needed, # unlike the v1 modules above (their wraps drop when #293 lands) surnames=KOREAN_SURNAMES, + honorific_tails=frozenset(GLUED_HONORIFICS), # pass canonical pair-tuples so this strictly-typed call site never # feeds a Mapping to the tuple-annotated field; __post_init__ # still tolerates a Mapping at runtime for interactive use diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 6988cdd..ceeb3a3 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -577,3 +577,13 @@ def test_default_lexicon_ships_korean_surnames_only() -> None: hangul = _SCRIPT_RANGES[Script.HANGUL] assert all(all(any(lo <= ord(c) <= hi for lo, hi in hangul) for c in s) and 1 <= len(s) <= 2 for s in lex.surnames) + + +def test_honorific_tails_is_a_vocab_field() -> None: + lex = Lexicon.empty().add(honorific_tails={"씨", "さん"}) + assert lex.honorific_tails == frozenset({"씨", "さん"}) + assert lex.remove(honorific_tails={"씨"}).honorific_tails == \ + frozenset({"さん"}) + merged = (Lexicon(honorific_tails=frozenset({"씨"})) + | Lexicon(honorific_tails=frozenset({"様"}))) + assert merged.honorific_tails == frozenset({"씨", "様"}) From b72150a2ffe5915dbb157e772a8a07a801555524 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 21:31:49 -0700 Subject: [PATCH 04/36] Enforce the honorific-tail subset invariant (#308) --- nameparser/_config_shim.py | 35 +++++++++++++++++++++-------------- nameparser/_lexicon.py | 16 ++++++++++++---- tests/v2/test_lexicon.py | 31 +++++++++++++++++++++++++------ tests/v2/test_properties.py | 6 +++++- 4 files changed, 63 insertions(+), 25 deletions(-) diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index 9481ac3..dbc5988 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -992,6 +992,17 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: particles = frozenset(self.prefixes) bound = frozenset(self.bound_first_names) ambiguous_acronyms = frozenset(self.suffix_acronyms_ambiguous) & acronyms + # Drop any ambiguous acronym from the word set rather than the + # other way round. Lexicon forbids the overlap because the word + # branch bypasses the period gate, and adding an ambiguous + # acronym to suffix_not_acronyms is INERT in v1 anyway: + # is_suffix already accepts it via the acronym branch, and + # reserve_last keeps it as the surname. So ignoring the + # addition reproduces v1 ("Jack Ma" keeps last='Ma'), where + # dropping it from the AMBIGUOUS set instead ungated the word + # and lost the family name -- a silent misparse worse than the + # raise it avoided. + suffix_words = frozenset(self.suffix_not_acronyms) - ambiguous_acronyms # keep in sync with _lexicon._default_lexicon() (pinned by # tests/v2/test_config_shim.py::test_snapshot_field_translation) lexicon = Lexicon( @@ -1018,18 +1029,7 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: if e == " ".join(e.split()) ) if t), suffix_acronyms=acronyms, - # Drop any ambiguous acronym from the word set rather than - # the other way round. Lexicon forbids the overlap because - # the word branch bypasses the period gate, and adding an - # ambiguous acronym to suffix_not_acronyms is INERT in v1 - # anyway: is_suffix already accepts it via the acronym - # branch, and reserve_last keeps it as the surname. So - # ignoring the addition reproduces v1 ("Jack Ma" keeps - # last='Ma'), where dropping it from the AMBIGUOUS set - # instead ungated the word and lost the family name -- - # a silent misparse worse than the raise it avoided. - suffix_words=frozenset( - self.suffix_not_acronyms) - ambiguous_acronyms, + suffix_words=suffix_words, # Intersect with acronyms: Lexicon enforces ambiguous <= # acronyms; v1 behaves the same when an acronym is deleted # but its ambiguous entry lingers (the entry stops @@ -1070,8 +1070,15 @@ def _build_snapshot(self) -> tuple[Lexicon, Policy, _RenderDefaults]: # module is born frozen (#293), so no wrap surnames=KOREAN_SURNAMES, # likewise no v1 manager: the glued-honorific tail set is - # 2.1 behavior (#308), so it rides in the snapshot only - honorific_tails=frozenset(GLUED_HONORIFICS), + # 2.1 behavior (#308), so it rides in the snapshot only. + # Wrapped, unlike surnames above: suffixes.py is still a + # mutable v1 module, not born-frozen like surnames.py + # (#293). Intersect with the word set: Lexicon enforces + # tails <= suffix_words, and v1 semantics are that deleting + # a suffix word turns the behavior off -- a lingering tail + # simply stops mattering, the same rule ambiguous_acronyms + # gets against suffix_acronyms above. + honorific_tails=frozenset(GLUED_HONORIFICS) & suffix_words, # TupleManager is dict[str, object] (v1 parity: values were # never statically str-typed); every real entry is a str, # same assumption _DelimiterManager's sentinel lookup makes diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index d3612a6..67a5e3a 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -38,6 +38,9 @@ #: * suffix_acronyms_ambiguous: _vocab returns True on the ambiguous #: set before testing suffix_acronyms, so an orphan silently turns a #: word into a period-gated suffix. +#: * honorific_tails: script_segment peels the tail into its own token +#: before classify ever runs, so an orphan splits the name and leaves +#: the fragment stranded inside it -- worse than not peeling at all. #: #: given_name_titles is deliberately NOT here and has no check of its #: own -- see the note in __post_init__ for why every attempt at one @@ -47,6 +50,8 @@ "an orphan emits a spurious particle-or-given ambiguity"), ("suffix_acronyms_ambiguous", "suffix_acronyms", "an orphan silently becomes a period-gated suffix"), + ("honorific_tails", "suffix_words", + "an orphan splits the name and leaves the tail inside it"), ) @@ -338,11 +343,14 @@ class Lexicon: surnames: frozenset[str] = frozenset() #: Honorifics that may be peeled off the END of a name token #: (#308), matched longest-first: 田中さん splits into 田中 and さん - #: before anything else reads the name. Every entry must also be a + #: before the tokens are classified. Every entry must also be a #: :attr:`suffix_words` entry -- the peeled tail is claimed by - #: suffix classification like any other post-nominal. Strictly - #: narrower than the spaced honorific vocabulary, since a glued - #: tail has no token boundary to lean on. Full default list: + #: suffix classification like any other post-nominal. Deliberately + #: NOT gated on :attr:`Policy.segment_scripts + #: ` (unlike :attr:`surnames` + #: above): 田中さん peels under the default policy, where HAN is in + #: no activation set, because a tail entry carries its own license + #: to fire. Full default list: #: :data:`~nameparser.config.suffixes.GLUED_HONORIFICS`. honorific_tails: frozenset[str] = frozenset() #: Lowercase word -> exact-cased replacement used by capitalized() diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index ceeb3a3..f07144b 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -325,7 +325,9 @@ def test_given_name_titles_is_not_constrained_against_titles() -> None: "particles_ambiguous", "particles"), (lambda w: Lexicon(suffix_acronyms_ambiguous=w), "suffix_acronyms_ambiguous", "suffix_acronyms"), -], ids=["particles_ambiguous", "suffix_acronyms_ambiguous"]) + (lambda w: Lexicon(honorific_tails=w), + "honorific_tails", "suffix_words"), +], ids=["particles_ambiguous", "suffix_acronyms_ambiguous", "honorific_tails"]) def test_subset_error_names_the_fix( make: Callable[[frozenset[str]], Lexicon], marker: str, base: str ) -> None: @@ -476,10 +478,12 @@ def test_every_lexicon_entry_point_rejects_a_buffer_with_a_decode_hint( def test_multiword_entry_warns_per_field(field: str) -> None: # Guard the whole family: every per-word field warns, not one. with pytest.warns(UserWarning, match="matched one word at a time"): - if field in ("particles_ambiguous", "suffix_acronyms_ambiguous"): + if field in ("particles_ambiguous", "suffix_acronyms_ambiguous", + "honorific_tails"): # subset fields need their base populated to construct base = {"particles_ambiguous": "particles", - "suffix_acronyms_ambiguous": "suffix_acronyms"}[field] + "suffix_acronyms_ambiguous": "suffix_acronyms", + "honorific_tails": "suffix_words"}[field] Lexicon.empty().add(**{base: ["zqx zqy"], field: ["zqx zqy"]}) else: Lexicon.empty().add(**{field: ["zqx zqy"]}) @@ -580,10 +584,25 @@ def test_default_lexicon_ships_korean_surnames_only() -> None: def test_honorific_tails_is_a_vocab_field() -> None: - lex = Lexicon.empty().add(honorific_tails={"씨", "さん"}) + # every tail is also a suffix word -- the peeled piece is claimed + # by suffix classification -- so each construction carries both + lex = Lexicon.empty().add(honorific_tails={"씨", "さん"}, + suffix_words={"씨", "さん"}) assert lex.honorific_tails == frozenset({"씨", "さん"}) assert lex.remove(honorific_tails={"씨"}).honorific_tails == \ frozenset({"さん"}) - merged = (Lexicon(honorific_tails=frozenset({"씨"})) - | Lexicon(honorific_tails=frozenset({"様"}))) + merged = (Lexicon(honorific_tails=frozenset({"씨"}), + suffix_words=frozenset({"씨"})) + | Lexicon(honorific_tails=frozenset({"様"}), + suffix_words=frozenset({"様"}))) assert merged.honorific_tails == frozenset({"씨", "様"}) + + +def test_default_lexicon_honorific_tails_are_all_suffix_words() -> None: + # Not redundant with suffixes.py's GLUED_HONORIFICS <= SUFFIX_NOT_ + # ACRONYMS assert: that one relates the raw constants at import + # time (and is gone under python -O); this one relates the + # normalized, post-ambiguous-subtraction Lexicon fields, and + # survives -O. Structural check, not a content pin. + d = Lexicon.default() + assert d.honorific_tails <= d.suffix_words diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index e66b4b4..6622abf 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -214,13 +214,17 @@ def _fix_invariants(**fields: frozenset[str]) -> dict[str, frozenset[str]]: Drawing dependent subsets directly (particles_ambiguous from whatever particles happened to be drawn) makes the strategy tree deep and mostly rejects; intersecting after the fact keeps every - draw usable and still reaches every shape. The four rules are + draw usable and still reaches every shape. The five rules are Lexicon's own, restated here on purpose -- if one changes, this fails loudly rather than silently fuzzing a narrower space. """ fields["particles_ambiguous"] &= fields["particles"] fields["suffix_acronyms_ambiguous"] &= fields["suffix_acronyms"] fields["suffix_words"] -= fields["suffix_acronyms_ambiguous"] + # order matters: must run AFTER the suffix_words subtraction above, + # or that later subtraction could re-orphan a tail this repair just + # fixed + fields["honorific_tails"] &= fields["suffix_words"] fields["bound_given_names"] -= ( fields["particles"] - fields["particles_ambiguous"]) return fields From 9075fdef18c79512b27d753e1d633d23f348a039 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 21:37:15 -0700 Subject: [PATCH 05/36] Peel a glued CJK honorific off the last name token (#308) --- nameparser/_pipeline/_script_segment.py | 82 ++++++++++++++--- tests/v2/cases.py | 110 ++++++++++++++++++++--- tests/v2/pipeline/test_script_segment.py | 95 ++++++++++++++++++++ tools/differential/corpus_cjk.jsonl | 11 +++ 4 files changed, 273 insertions(+), 25 deletions(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 37e01c2..d2bd227 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -17,6 +17,15 @@ nothing -- spans still index the original exactly, so the anti-#100 invariant holds by construction. +A second, independent split runs first (#308): a listed honorific +glued to the END of the name part's last token is peeled off as its +own token -- 田中さん -> 田中 + さん -- so that suffix classification +can claim it and the surname match or segmenter consult below sees the +name rather than the name plus an honorific. It is gated by the +structural opt-outs (comma, 间隔号) but NOT by segment_scripts: the +vocabulary of tails is licensed by the entries themselves, each of +which can never end a name, so no per-script trust question arises. + Where the VOCABULARY declines -- no prefix matched -- an optional Parser(segmenter=...) gets the token (#272 amendment 2026-07-29). Vocabulary first, segmenter on decline, so parser_for(ZH, JA, @@ -179,20 +188,56 @@ def _split(state: ParseState, i: int, splits: tuple[int, ...], ambiguities=ambiguities) -@functools.lru_cache(maxsize=8) -def _longest_entry(surnames: frozenset[str]) -> int: - """The longest surname in a vocabulary, cached per-vocabulary - rather than recomputed per parse (Lexicon is frozen and slotted, - so it cannot carry a cached_property of its own). The frozenset is +@functools.lru_cache(maxsize=16) +def _longest_entry(entries: frozenset[str]) -> int: + """The longest entry in a vocabulary, cached per-vocabulary rather + than recomputed per parse (Lexicon is frozen and slotted, so it + cannot carry a cached_property of its own). The frozenset is hashable and a process holds only a handful of distinct vocabularies -- the default one, plus one per constructed pack - parser -- so maxsize=8 bounds pathological many-lexicon churn - without ever evicting in normal use. + parser, times the two FIELDS that call this (surnames and + honorific_tails) -- so maxsize=16 bounds pathological many-lexicon + churn without ever evicting in normal use. Callers must pass a NON-EMPTY vocabulary: max() of an empty set - raises, and the only call site sits under the stage's `if surnames` - match guard, so it cannot reach that.""" - return max(map(len, surnames)) + raises, and both call sites sit under a match guard that cannot + reach it with one.""" + return max(map(len, entries)) + + +def _peel_honorific_tail(state: ParseState) -> ParseState: + """#308: split a listed honorific off the END of the name part's + last token -- 田中さん -> 田中 + さん -- and let the existing + machinery do the rest. Suffix classification claims the tail + downstream (every honorific_tails entry is a suffix word too, + enforced by Lexicon), and the segmentation half below then sees + the remainder rather than the glued whole, so 김민준씨 splits + 김 + 민준 and a configured segmenter is handed 山田太郎 rather + than 山田太郎様. + + Longest-first, and capped so the remainder is never empty: a token + that IS a tail (씨, さん) has nothing to peel off and stays whole + -- the analogue of the `text in surnames` guard below. ONE peel, + never recursive: 김민준박사님 gives up its 님 and keeps its glued + 박사, though 박사 is itself a listed tail, which is accepted + rather than chased. No script precondition on the remainder + either, since the tail alone is the license: Andersonさん peels. + + Emits no ambiguity, on the 〆 and 间隔号 rule -- vocabulary plus + orthography decided this, nothing was chosen between.""" + tails = state.lexicon.honorific_tails + if not tails: + return state + i = state.segments[0][-1] + text = state.tokens[i].text + # range/cap construction identical to the surname match below, and + # for the same two reasons: longest-first, and a len-1 cap that + # makes the offset interior by construction (_split's contract). + cap = min(_longest_entry(tails), len(text) - 1) + for length in range(cap, 0, -1): + if text[-length:] in tails: + return _split(state, i, (len(text) - length,), None) + return state def script_segment(state: ParseState) -> ParseState: @@ -201,9 +246,7 @@ def script_segment(state: ParseState) -> ParseState: # so an ASCII original has only ASCII tokens: nothing here is # in any script's ranges return state - scripts = state.policy.segment_scripts - # an empty VOCABULARY deliberately does not bail here -- see below - if not scripts or not state.segments: + if not state.segments: return state if state.structure is Structure.FAMILY_COMMA: return state # the comma already drew the boundary @@ -217,6 +260,19 @@ def script_segment(state: ParseState) -> ParseState: # listing, so even an un-dotted hangul token beside a dotted # one stays whole. return state + # #308: the honorific peel runs after the structural gates above + # (a comma-divided or dot-divided name opts out of this stage + # whole) but BEFORE the activation gate below, and independently + # of it -- 田中さん must peel under the DEFAULT policy, where HAN + # is in no activation set. The activation gate is about which + # scripts a SURNAME VOCABULARY may be trusted to divide; the peel + # asks nothing of the remainder, only whether the token ends in a + # word that can never end a name. + state = _peel_honorific_tail(state) + scripts = state.policy.segment_scripts + # an empty VOCABULARY deliberately does not bail here -- see below + if not scripts: + return state # segments[0] is the NAME part under both remaining structures # (everything, under NO_COMMA); later segments are suffixes. Its # members are main-stream token indices by construction, so diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 14ef330..1511a54 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -741,6 +741,79 @@ def __post_init__(self) -> None: "recognition falls out of the same entry -- until this " "change さん read as the given name under the " "family-first default"), + Case("ja_san_glued", "田中さん", + {"family": "田中", "suffix": "さん"}, + classification="fix(#308)", + notes="the everyday glued form, and the one that also " + "corrupted classification: 田中さん is Han plus " + "hiragana, so the kana license read the whole string " + "as a Japanese name. The peel runs before the license " + "is consulted, so it now sees 田中 alone"), + Case("ja_sama_glued", "山田太郎様", + {"family": "山田太郎", "suffix": "様"}, + classification="fix(#308)", + notes="the mail-addressing form. Undivided without a " + "segmenter -- no surname list divides a kanji name -- " + "so the family name is the whole 山田太郎; " + "tests/v2/test_locales.py pins the divided twin under " + "locales.JA"), + Case("ko_honorific_nim_glued", "김민준님", + {"family": "김", "given": "민준", "suffix": "님"}, + classification="fix(#308)", + notes="the online/formal glued address form, 씨's twin"), + Case("ko_honorific_glued_teacher", "김선생님", + {"family": "김", "suffix": "선생님"}, + classification="fix(#307)", + notes="longest-first, end to end: 선생님 peels whole where " + "님 alone would have left 김선생 to segment into a " + "family 김 and a given 선생. Classified to #307 " + "because the fields do not move in this change -- " + "segmentation already delivered this shape (김 is a " + "listed surname, so the split reached it); #308 " + "changes which mechanism gets there first"), + Case("latin_stem_glued_kana_honorific", "Andersonさん", + {"given": "Anderson", "suffix": "さん"}, + classification="fix(#308)", + notes="no script precondition on the remainder -- the tail " + "is the license. Japanese text about a foreigner, and " + "the Latin remainder keeps the positional default"), + Case("ko_glued_stack_peels_once", "김민준박사님", + {"family": "김", "given": "민준박사", "suffix": "님"}, + classification="fix(#308)", + notes="one peel, no recursion -- and the remainder 김민준박사 " + "ends in a listed tail of its own, so this is the " + "shape that would recurse if anything did. 님 comes " + "off, the glued 박사 stays in the given name. Accepted " + "and pinned rather than chased"), + Case("ko_glued_tail_alone_never_peels", "씨", + {"family": "씨"}, + classification="fix(#271)", + notes="the empty-remainder guard: a token that IS a tail has " + "nothing to peel off. Classified to #271 because that " + "is what moves the lone token from first to family; " + "the row exists for the guard"), + Case("ja_glued_tail_alone_never_peels", "さん", + {"family": "さん"}, + classification="fix(#272)", + notes="the kana twin of the row above; the field placement " + "is the kana-licensed order default, not the peel"), + Case("ja_glued_degree_stays", "田中博士", + {"family": "田中博士"}, + classification="fix(#271)", + notes="the exclusion pinned: glued 田中博士 IS Tanaka " + "Hiroshi, an attested given name, so 博士 never peels " + "-- it stays recognized in the SPACED position only, " + "where the writer's own token boundary settles it"), + Case("zh_glued_jun_stays", "王君", + {"family": "王君"}, + classification="fix(#271)", + notes="likewise: 君 is a common Chinese given-name final, so " + "王君 is a complete name, not Mr. Wang"), + Case("zh_glued_shi_stays", "王氏", + {"family": "王氏"}, + classification="fix(#271)", + notes="likewise: 王氏 is a historical name form ('the Wang " + "woman'). The spaced 田中氏 keeps its entry"), Case("ja_dono_spaced", "田中 殿", {"family": "田中", "suffix": "殿"}, classification="fix(#308)", @@ -771,13 +844,15 @@ def __post_init__(self) -> None: classification="fix(#307)", notes="the post-comma lenient gate admits the honorific too; " "the comma disables segmentation per the comma " - "doctrine, so 김민준 stays whole"), Case("ko_honorific_glued_given_stays", "김민준씨", - {"family": "김", "given": "민준씨"}, - notes="the boundary of the glued reach: segmentation splits " - "off 김 and the REMAINDER is 민준씨, not a listed " - "honorific token -- the common full-name glued shape " - "is #308's mechanism, pinned here so the docs' scoped " - "claim stays true"), + "doctrine, so 김민준 stays whole"), + Case("ko_honorific_glued_given", "김민준씨", + {"family": "김", "given": "민준", "suffix": "씨"}, + classification="fix(#308)", + notes="the common full-name glued shape, and the row this " + "replaces (ko_honorific_glued_given_stays) pinned the " + "old boundary: 씨 peels off the last token first, and " + "the remainder 김민준 then segments as usual -- peel " + "and split compose, in that order"), Case("zh_honorific_glued_surname", "王先生", {"family": "王", "suffix": "先生"}, locale="zh", @@ -785,17 +860,28 @@ def __post_init__(self) -> None: notes="the Han twin of 김씨: the zh pack's segmentation " "splits off the surname and the remaining 先生 is the " "honorific token"), - Case("zh_honorific_glued_given_stays", "王小明先生", - {"family": "王", "given": "小明先生"}, + Case("zh_honorific_glued_given", "王小明先生", + {"family": "王", "given": "小明", "suffix": "先生"}, locale="zh", - notes="the Han boundary twin: 小明先生 is the remainder, not " - "an honorific token -- glued full names stay #308"), + classification="fix(#308)", + notes="the Han twin, replacing zh_honorific_glued_given_stays: " + "先生 peels, and the zh pack's surname vocabulary then " + "divides the remainder 王小明"), + Case("zh_honorific_glued_given_default", "王小明先生", + {"family": "王小明", "suffix": "先生"}, + classification="fix(#308)", + notes="the same input WITHOUT the pack: the peel is default-on " + "and script-independent, so the honorific still routes " + "to suffix -- only the surname split needs the opt-in, " + "so the undivided 王小明 stays one family name"), Case("ko_suffix_matching_is_whole_token", "김지양", {"family": "김", "given": "지양"}, notes="지양 ENDS with the honorific 양 but is a given name: " "suffix matching is whole-token, never endswith -- the " "pin the differential rule's anchor mirrors at the " - "name-string level"), + "name-string level. #308 leaves it alone too: 양 is " + "excluded from the glued tail set for exactly this " + "name"), Case("ko_surname_yang_leads", "양 미선", {"family": "양", "given": "미선"}, notes="양 is both a top-tier surname (Yang) and a shipped " diff --git a/tests/v2/pipeline/test_script_segment.py b/tests/v2/pipeline/test_script_segment.py index 80f88fc..59a8aa7 100644 --- a/tests/v2/pipeline/test_script_segment.py +++ b/tests/v2/pipeline/test_script_segment.py @@ -26,6 +26,15 @@ # "jr" so the comma cases can reach SUFFIX_COMMA. _LEX = Lexicon(surnames=frozenset({"毛", "欧", "欧阳", "김", "남", "남궁"}), suffix_words=frozenset({"jr"})) +# The peel's own vocabulary (#308). Tails are also suffix words here, +# as the shipped config asserts they are: the neighbour rule in the +# segmenter tests below reads that membership, so a stage lexicon that +# omitted it would pin a shape no real configuration can have. +_TAILS = frozenset({"씨", "님", "선생님", "박사", "さん", "先生"}) +_LEX_TAILS = Lexicon( + surnames=frozenset({"毛", "欧", "欧阳", "김", "남", "남궁"}), + suffix_words=frozenset({"jr"}) | _TAILS, + honorific_tails=_TAILS) def _run(text: str, policy: Policy = _HAN, lexicon: Lexicon = _LEX, @@ -399,6 +408,92 @@ def test_a_neighbour_in_an_UNACTIVATED_script_still_blocks_the_consult() -> None assert out.ambiguities == (), name +# -- the glued honorific peel (#308) ----------------------------------- + + +def test_peels_a_listed_tail_without_any_activation() -> None: + # The peel is licensed by the TAIL, not by the script, so it runs + # under a policy that activates nothing -- 田中さん must peel under + # the DEFAULT policy, where HAN is in no activation set. Spans stay + # sub-slices of the original (anti-#100), like every split here. + off = Policy(segment_scripts=()) # type: ignore[arg-type] + out = _run("田中さん", policy=off, lexicon=_LEX_TAILS) + assert _texts(out) == ["田中", "さん"] + assert [(t.span.start, t.span.end) for t in out.tokens] == [ + (0, 2), (2, 4)] + assert all(out.original[t.span.start:t.span.end] == t.text + for t in out.tokens) + assert out.ambiguities == () # decisive vocabulary, no fork + + +def test_peel_then_segmentation_compose() -> None: + # the peel runs first, so the segmentation half sees the REMAINDER + assert _texts(_run("김민준씨", policy=_HANGUL, + lexicon=_LEX_TAILS)) == ["김", "민준", "씨"] + + +def test_a_token_that_is_a_tail_never_peels() -> None: + # nothing to split off: the empty-remainder guard, the analogue of + # the whole-token surname guard above + assert _texts(_run("씨", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["씨"] + assert _texts(_run("さん", policy=_HANGUL, + lexicon=_LEX_TAILS)) == ["さん"] + + +def test_longest_tail_wins() -> None: + # 님 and 선생님 both match by endswith; peeling the shorter one + # would leave 김선생 for the vocabulary to split into a family 김 + # and a given 선생. Same longest-first discipline as the surname + # match, and 김 is itself a listed surname, so the remainder then + # declines to split further (the whole-token guard). + assert _texts(_run("김선생님", policy=_HANGUL, + lexicon=_LEX_TAILS)) == ["김", "선생님"] + + +def test_one_peel_never_a_stack() -> None: + # The remainder here ENDS in another listed tail (박사) and is + # still not peeled again: one peel, then the stage moves on to + # segmentation, which splits the surname off what is left + assert _texts(_run("김민준박사님", policy=_HANGUL, + lexicon=_LEX_TAILS)) == ["김", "민준박사", "님"] + + +def test_peel_needs_no_script_on_the_remainder() -> None: + # the tail is the license: Japanese text about a foreigner + assert _texts(_run("Andersonさん", policy=_HANGUL, + lexicon=_LEX_TAILS)) == ["Anderson", "さん"] + + +def test_an_unlisted_tail_never_peels() -> None: + # endswith against the vocabulary, never against a shape: 지양 ends + # in a shipped SPACED honorific that is deliberately not a tail + assert _texts(_run("김지양", policy=_HANGUL, + lexicon=_LEX_TAILS)) == ["김", "지양"] + + +def test_family_comma_skips_the_peel() -> None: + # the comma doctrine covers the peel with the rest of the stage: + # the writer already said where the family name ends + assert _texts(_run("田中さん, 太郎", lexicon=_LEX_TAILS)) == [ + "田中さん", "太郎"] + + +def test_interpunct_divided_name_never_peels() -> None: + # the transcription gate likewise: its pieces are syllable groups + state = segment(tokenize(ParseState( + original="威廉·莎士比亚先生", lexicon=_LEX_TAILS, policy=_HAN))) + assert state.interpunct_offsets + assert script_segment(state).tokens == state.tokens + + +def test_the_peel_only_looks_at_the_last_name_token() -> None: + # a tail anywhere but the end is somebody's name, not an + # honorific: only the trailing position is an honorific site, the + # same reasoning the trailing-only suffix gate follows + assert _texts(_run("田中さん 太郎", policy=_HANGUL, + lexicon=_LEX_TAILS)) == ["田中さん", "太郎"] + + # -- the 间隔号 gate: a divided name is a transcription (#298) ---------- diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index b3ecb3a..d2dcf5b 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -1,8 +1,10 @@ +"Andersonさん" "Dr 김민준, Jr." "John 王" "〆木 ひろ" "〆木 太郎" "〆木太郎" +"さん" "みなみ" "マイケル" "マイケル ジャクソン" @@ -17,18 +19,23 @@ "山田 エミ" "山田 太郎 (マイケル・ジャクソン)" "山田「タロ」太郎" +"山田太郎様" "张伟" "毛 泽东" "毛 김" "毛泽东" "王·Smith" "王先生" +"王君" "王小明 先生" "王小明先生" +"王氏" "田中 さん" "田中 太郎 様" "田中 殿" "田中『ハナ』花子" +"田中さん" +"田中博士" "諸葛亮" "阿明" "马丁·路德·金" @@ -43,11 +50,15 @@ "김민준 박사 씨" "김민준 씨" "김민준, 씨" +"김민준님" +"김민준박사님" "김민준씨" +"김선생님" "김씨" "김지양" "남궁" "남궁민수" "남궁민수, 지훈" "마이클·잭슨" +"씨" "양 미선" From 17f9c39d531b897ce007e73ad56dea32fe903e82 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 22:03:30 -0700 Subject: [PATCH 06/36] Never treat a post-nominal as a name site (#308) --- nameparser/_pipeline/_script_segment.py | 96 +++++++++++++++++------- tests/v2/cases.py | 25 ++++++ tests/v2/pipeline/test_script_segment.py | 38 +++++++--- tools/differential/corpus_cjk.jsonl | 3 + 4 files changed, 124 insertions(+), 38 deletions(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index d2bd227..df23b8b 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -18,13 +18,15 @@ invariant holds by construction. A second, independent split runs first (#308): a listed honorific -glued to the END of the name part's last token is peeled off as its -own token -- 田中さん -> 田中 + さん -- so that suffix classification -can claim it and the surname match or segmenter consult below sees the -name rather than the name plus an honorific. It is gated by the -structural opt-outs (comma, 间隔号) but NOT by segment_scripts: the -vocabulary of tails is licensed by the entries themselves, each of -which can never end a name, so no per-script trust question arises. +glued to the END of the name part's last non-post-nominal token is +peeled off as its own token -- 田中さん -> 田中 + さん -- so that +suffix classification can claim it and the surname match or segmenter +consult below sees the name rather than the name plus an honorific. It +is gated by the FAMILY comma and by 间隔号 but NOT by segment_scripts: +the vocabulary of tails is licensed by the entries themselves, each of +which can never end a name, so no per-script trust question arises. A +suffix comma does not gate it -- "Dr 김민준씨, Jr." peels within its +name part like any other. Where the VOCABULARY declines -- no prefix matched -- an optional Parser(segmenter=...) gets the token (#272 amendment 2026-07-29). @@ -83,7 +85,13 @@ Only the FIRST activated-script token is considered, match or no match: family-first traditions put the surname at the front of the name, and a match deeper in the token stream would be a given name or -an ordinary word, not a surname site. +an ordinary word, not a surname site. A token the vocabulary reads as +a POST-NOMINAL is passed over rather than taken as that site, whoever +wrote it: an honorific is not part of the name, so it cannot be the +name's front. The peel above manufactures such a token and would +otherwise have its own product dissected -- the 선생님 of Anderson선생님 +opens on the listed surname 선 -- and a spaced one was mis-split the +same way before the peel existed ("Anderson 선생님"). """ from __future__ import annotations @@ -93,7 +101,7 @@ from nameparser._pipeline._state import ( ParseState, PendingAmbiguity, Structure, WorkToken, ) -from nameparser._pipeline._vocab import effective_script +from nameparser._pipeline._vocab import effective_script, is_suffix_strict from nameparser._types import AmbiguityKind, Segmentation, Span #: Segmenter answers scoring below this attach a SEGMENTATION report @@ -205,30 +213,58 @@ def _longest_entry(entries: frozenset[str]) -> int: return max(map(len, entries)) +def _is_post_nominal(state: ParseState, i: int) -> bool: + """Whether token `i` is vocabulary the parse will read as a + post-nominal. Two of this stage's decisions ask it and neither is + about script: an honorific is not part of the name, so it is + neither a surname SITE nor the token a glued honorific hangs off. + (#308; the segmenter's neighbour test joins them in the next + change, for the same reason.)""" + return is_suffix_strict(state.tokens[i].text, state.lexicon) + + def _peel_honorific_tail(state: ParseState) -> ParseState: """#308: split a listed honorific off the END of the name part's - last token -- 田中さん -> 田中 + さん -- and let the existing - machinery do the rest. Suffix classification claims the tail - downstream (every honorific_tails entry is a suffix word too, - enforced by Lexicon), and the segmentation half below then sees - the remainder rather than the glued whole, so 김민준씨 splits + last NON-POST-NOMINAL token -- 田中さん -> 田中 + さん -- and let + the existing machinery do the rest. Suffix classification claims + the tail downstream (every honorific_tails entry is a suffix word + too, enforced by Lexicon), and the segmentation half below then + sees the remainder rather than the glued whole, so 김민준씨 splits 김 + 민준 and a configured segmenter is handed 山田太郎 rather than 山田太郎様. - Longest-first, and capped so the remainder is never empty: a token - that IS a tail (씨, さん) has nothing to peel off and stays whole - -- the analogue of the `text in surnames` guard below. ONE peel, - never recursive: 김민준박사님 gives up its 님 and keeps its glued - 박사, though 박사 is itself a listed tail, which is accepted - rather than chased. No script precondition on the remainder - either, since the tail alone is the license: Andersonさん peels. + Scanning back over post-nominals rather than taking the last token + outright does three things at once. An unrelated trailing suffix + cannot hide the peel site, so "김민준씨 Jr." answers as the + comma-written "Dr 김민준씨, Jr." does -- one name, two spellings, + one parse. A token that IS a tail (씨, さん, and the nested 선생님, + which the cap alone would peel to 선생 + 님) is skipped as a site + and stays whole: every tail is a suffix word by the Lexicon + invariant, which is what makes that guard hold, and the cap below + only keeps the offset interior for _split. And segments[0] is + reachable while empty -- a leading comma yields one -- so the scan + that returns None where the outright index would raise removes an + unpinned reliance on the FAMILY_COMMA gate landing first. + + Longest-first. ONE peel, never recursive: 김민준박사님 gives up + its 님 and keeps its glued 박사, though 박사 is itself a listed + tail, which is accepted rather than chased. No script precondition + on the remainder either, since the tail alone is the license: + Andersonさん peels. - Emits no ambiguity, on the 〆 and 间隔号 rule -- vocabulary plus - orthography decided this, nothing was chosen between.""" + Emits no ambiguity, unlike the surname fork below, though + longest-first does CHOOSE here too -- 김선생님 gives 선생님 where + 님 also matches. The difference is what the runner-up is: a second + matching surname is a competing READING of the name, which a + caller may prefer, while a shorter tail leaves a remainder that is + not a name at all (김선생), so there is nothing to adjudicate.""" tails = state.lexicon.honorific_tails if not tails: return state - i = state.segments[0][-1] + i = next((j for j in reversed(state.segments[0]) + if not _is_post_nominal(state, j)), None) + if i is None: + return state text = state.tokens[i].text # range/cap construction identical to the surname match below, and # for the same two reasons: longest-first, and a len-1 cap that @@ -261,10 +297,11 @@ def script_segment(state: ParseState) -> ParseState: # one stays whole. return state # #308: the honorific peel runs after the structural gates above - # (a comma-divided or dot-divided name opts out of this stage - # whole) but BEFORE the activation gate below, and independently - # of it -- 田中さん must peel under the DEFAULT policy, where HAN - # is in no activation set. The activation gate is about which + # (a FAMILY-comma or dot-divided name opts out of this stage + # whole; a suffix comma does not -- its name part still peels) + # but BEFORE the activation gate below, and independently of it -- + # 田中さん must peel under the DEFAULT policy, where HAN is in no + # activation set. The activation gate is about which # scripts a SURNAME VOCABULARY may be trusted to divide; the peel # asks nothing of the remainder, only whether the token ends in a # word that can never end a name. @@ -279,7 +316,8 @@ def script_segment(state: ParseState) -> ParseState: # extracted nickname/maiden content is unreachable from here -- # no input can produce it, so no test pins it. i = next((i for i in state.segments[0] - if effective_script(state.tokens[i].text) in scripts), None) + if effective_script(state.tokens[i].text) in scripts + and not _is_post_nominal(state, i)), None) if i is None: return state token = state.tokens[i] diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 1511a54..d81ec3f 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -777,6 +777,15 @@ def __post_init__(self) -> None: notes="no script precondition on the remainder -- the tail " "is the license. Japanese text about a foreigner, and " "the Latin remainder keeps the positional default"), + Case("latin_stem_glued_hangul_honorific", "Anderson선생님", + {"given": "Anderson", "suffix": "선생님"}, + classification="fix(#308)", + notes="the hangul twin of latin_stem_glued_kana_honorific, " + "and the one that shows why a post-nominal is not a " + "surname site: 선 is a listed census surname, so the " + "peeled 선생님 would otherwise be split into 선 + 생님 " + "-- the stage dissecting the honorific it had just " + "manufactured"), Case("ko_glued_stack_peels_once", "김민준박사님", {"family": "김", "given": "민준박사", "suffix": "님"}, classification="fix(#308)", @@ -792,6 +801,14 @@ def __post_init__(self) -> None: "nothing to peel off. Classified to #271 because that " "is what moves the lone token from first to family; " "the row exists for the guard"), + Case("ko_honorific_token_alone_stays_whole", "선생님", + {"family": "선생님"}, + classification="fix(#308)", + notes="a lone honorific is not a name to be taken apart: it " + "is not a peel site (every tail is a suffix word) and " + "not a surname site, though 선 is listed and the " + "default segmentation split it 선 + 생님 before this " + "change"), Case("ja_glued_tail_alone_never_peels", "さん", {"family": "さん"}, classification="fix(#272)", @@ -853,6 +870,14 @@ def __post_init__(self) -> None: "old boundary: 씨 peels off the last token first, and " "the remainder 김민준 then segments as usual -- peel " "and split compose, in that order"), + Case("ko_honorific_glued_given_trailing_suffix", "김민준씨 Jr.", + {"family": "김", "given": "민준", "suffix": "씨, Jr."}, + classification="fix(#308)", + notes="the peel site is the last token that is not itself a " + "post-nominal, so an unrelated trailing suffix cannot " + "hide it -- this now agrees with the comma-written " + "'Dr 김민준씨, Jr.', where the suffix comma had " + "already put 씨 within reach"), Case("zh_honorific_glued_surname", "王先生", {"family": "王", "suffix": "先生"}, locale="zh", diff --git a/tests/v2/pipeline/test_script_segment.py b/tests/v2/pipeline/test_script_segment.py index 59a8aa7..7c5112a 100644 --- a/tests/v2/pipeline/test_script_segment.py +++ b/tests/v2/pipeline/test_script_segment.py @@ -433,21 +433,21 @@ def test_peel_then_segmentation_compose() -> None: def test_a_token_that_is_a_tail_never_peels() -> None: - # nothing to split off: the empty-remainder guard, the analogue of - # the whole-token surname guard above + # nothing to split off, and the guard is the peel site's scan past + # post-nominals, not the length cap: 선생님 ENDS in the shorter + # tail 님, so a cap-based guard would peel it to 선생 + 님 assert _texts(_run("씨", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["씨"] assert _texts(_run("さん", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["さん"] + assert _texts(_run("선생님", policy=_HANGUL, + lexicon=_LEX_TAILS)) == ["선생님"] def test_longest_tail_wins() -> None: - # 님 and 선생님 both match by endswith; peeling the shorter one - # would leave 김선생 for the vocabulary to split into a family 김 - # and a given 선생. Same longest-first discipline as the surname - # match, and 김 is itself a listed surname, so the remainder then - # declines to split further (the whole-token guard). - assert _texts(_run("김선생님", policy=_HANGUL, - lexicon=_LEX_TAILS)) == ["김", "선생님"] + # 님 and 선생님 both match by endswith, and only the longer one + # leaves a remainder that is a name (박선생 is not one) + assert _texts(_run("박선생님", policy=_HANGUL, + lexicon=_LEX_TAILS)) == ["박", "선생님"] def test_one_peel_never_a_stack() -> None: @@ -486,6 +486,26 @@ def test_interpunct_divided_name_never_peels() -> None: assert script_segment(state).tokens == state.tokens +def test_a_peeled_tail_is_not_a_surname_site() -> None: + # The peel manufactures a token, and the site scan below must not + # then treat it as a name: 남궁 opens with the listed surname 남, + # so without the post-nominal guard the stage would split the + # honorific it had just created. + lex = Lexicon(surnames=frozenset({"남"}), + suffix_words=frozenset({"남궁"}), + honorific_tails=frozenset({"남궁"})) + assert _texts(_run("Anderson남궁", policy=_HANGUL, + lexicon=lex)) == ["Anderson", "남궁"] + + +def test_a_trailing_post_nominal_does_not_hide_the_peel_site() -> None: + # the site is the last token that is not ITSELF a post-nominal, so + # an unrelated trailing suffix cannot put the glued one out of + # reach -- "김민준씨 jr" must answer as "Dr 김민준씨, Jr." does + assert _texts(_run("김민준씨 jr", policy=_HANGUL, + lexicon=_LEX_TAILS)) == ["김", "민준", "씨", "jr"] + + def test_the_peel_only_looks_at_the_last_name_token() -> None: # a tail anywhere but the end is somebody's name, not an # honorific: only the trailing position is an honorific site, the diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index d2dcf5b..b2c4c45 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -1,4 +1,5 @@ "Andersonさん" +"Anderson선생님" "Dr 김민준, Jr." "John 王" "〆木 ひろ" @@ -53,6 +54,7 @@ "김민준님" "김민준박사님" "김민준씨" +"김민준씨 Jr." "김선생님" "김씨" "김지양" @@ -60,5 +62,6 @@ "남궁민수" "남궁민수, 지훈" "마이클·잭슨" +"선생님" "씨" "양 미선" From be97aec9eef9c458feb6ac33cef48581915df267 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 22:08:33 -0700 Subject: [PATCH 07/36] Decline rather than scan past a post-nominal surname site (#308) --- nameparser/_pipeline/_script_segment.py | 32 ++++++++++++++++-------- tests/v2/cases.py | 12 +++++++++ tests/v2/pipeline/test_script_segment.py | 10 ++++++++ tools/differential/corpus_cjk.jsonl | 1 + 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index df23b8b..153993e 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -85,13 +85,17 @@ Only the FIRST activated-script token is considered, match or no match: family-first traditions put the surname at the front of the name, and a match deeper in the token stream would be a given name or -an ordinary word, not a surname site. A token the vocabulary reads as -a POST-NOMINAL is passed over rather than taken as that site, whoever -wrote it: an honorific is not part of the name, so it cannot be the -name's front. The peel above manufactures such a token and would -otherwise have its own product dissected -- the 선생님 of Anderson선생님 -opens on the listed surname 선 -- and a spaced one was mis-split the -same way before the peel existed ("Anderson 선생님"). +an ordinary word, not a surname site. Where that first token is one +the vocabulary reads as a POST-NOMINAL, the stage DECLINES rather than +looking further along: an honorific is not part of the name, and a +surname leads, so an honorific in the surname's own position means +there is no surname here to find. Deciding includes deciding that. +Looking further would reach the given name the rule already refuses to +touch -- "양 지훈" would split its own given name, 지 being listed too +-- while declining leaves the peel's manufactured tail intact, since +in "Anderson선생님" that tail is the first and only script-written +token (선생님 opens on the listed surname 선, and a spaced +"Anderson 선생님" was mis-split that way before the peel existed). """ from __future__ import annotations @@ -316,9 +320,17 @@ def script_segment(state: ParseState) -> ParseState: # extracted nickname/maiden content is unreachable from here -- # no input can produce it, so no test pins it. i = next((i for i in state.segments[0] - if effective_script(state.tokens[i].text) in scripts - and not _is_post_nominal(state, i)), None) - if i is None: + if effective_script(state.tokens[i].text) in scripts), None) + # A post-nominal in the surname's own position is not a site to + # skip past but an answer: a surname LEADS, so if the leading + # script-written token is an honorific there is no surname here to + # find. Scanning ON would reach the given name, which is exactly + # what the first-token rule exists to prevent -- 지 is a listed + # surname, so "양 지훈" (양 is a surname AND a shipped honorific) + # would have its own given name split in half. Declining also + # covers the token the peel above manufactures, which is the first + # and only script-written one in "Anderson선생님". + if i is None or _is_post_nominal(state, i): return state token = state.tokens[i] text = token.text diff --git a/tests/v2/cases.py b/tests/v2/cases.py index d81ec3f..6c27753 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -912,6 +912,18 @@ def __post_init__(self) -> None: notes="양 is both a top-tier surname (Yang) and a shipped " "honorific: position decides, and a surname LEADS -- " "the trailing-only suffix gate never sees it here"), + Case("ko_surname_yang_leads_a_segmentable_given", "양 지훈", + {"family": "양", "given": "지훈"}, + classification="fix(#271)", + notes="양 is a surname AND a shipped honorific, and 지 is a " + "listed surname too -- so this is the name that " + "catches a segmentation site scanning PAST the " + "honorific reading of 양 into the given name. The " + "first script-written token decides, and deciding " + "includes deciding there is no surname site. " + "Classified to #271, not parity: 1.4 gave first 양, " + "last 지훈, and it is the CJK order flip that puts 양 " + "in family -- nothing in #308 moves these fields"), Case("ko_honorific_stack", "김민준 박사 씨", {"family": "김", "given": "민준", "suffix": "박사, 씨"}, classification="fix(#307)", diff --git a/tests/v2/pipeline/test_script_segment.py b/tests/v2/pipeline/test_script_segment.py index 7c5112a..3a36c5e 100644 --- a/tests/v2/pipeline/test_script_segment.py +++ b/tests/v2/pipeline/test_script_segment.py @@ -498,6 +498,16 @@ def test_a_peeled_tail_is_not_a_surname_site() -> None: lexicon=lex)) == ["Anderson", "남궁"] +def test_a_leading_post_nominal_is_not_a_surname_site() -> None: + # A surname LEADS, so an honorific in the surname's own position + # means there is no surname to find -- decline rather than scan + # on. Scanning on would reach the given name, which is what the + # first-token rule above exists to prevent: 지 is a listed + # surname, so 양 지훈 would split its own given name in half. + assert _texts(_run("씨 김민준", policy=_HANGUL, + lexicon=_LEX_TAILS)) == ["씨", "김민준"] + + def test_a_trailing_post_nominal_does_not_hide_the_peel_site() -> None: # the site is the last token that is not ITSELF a post-nominal, so # an unrelated trailing suffix cannot put the glued one out of diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index b2c4c45..3160314 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -65,3 +65,4 @@ "선생님" "씨" "양 미선" +"양 지훈" From 90be6ba5f685871bea8eadc76e205668e462c520 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 22:16:19 -0700 Subject: [PATCH 08/36] Pin the suffix-comma peel and name the maiden reach (#308) --- nameparser/_pipeline/_script_segment.py | 9 +++++++++ tests/v2/cases.py | 10 ++++++++++ tests/v2/pipeline/test_script_segment.py | 2 +- tools/differential/corpus_cjk.jsonl | 1 + 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 153993e..272a24d 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -250,6 +250,15 @@ def _peel_honorific_tail(state: ParseState) -> ParseState: that returns None where the outright index would raise removes an unpinned reliance on the FAMILY_COMMA gate landing first. + That last token is the last of the NAME PART, which under NO_COMMA + reaches into a maiden clause: maiden tokens are still main-stream + here (extract_delimited has masked only bracketed content), so + "김민준 née 박씨" peels 씨 off the MAIDEN name 박씨 and hands it to + the person's suffix list -- "née Ms. Park". Intended rather than + incidental: the honorific is the reader's regardless of which of + her names it was glued to, and a name-part-final honorific is + exactly what this peels. + Longest-first. ONE peel, never recursive: 김민준박사님 gives up its 님 and keeps its glued 박사, though 박사 is itself a listed tail, which is accepted rather than chased. No script precondition diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 6c27753..dd3c018 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -878,6 +878,16 @@ def __post_init__(self) -> None: "hide it -- this now agrees with the comma-written " "'Dr 김민준씨, Jr.', where the suffix comma had " "already put 씨 within reach"), + Case("ko_honorific_glued_given_suffix_comma", "Dr 김민준씨, Jr.", + {"title": "Dr", "family": "김", "given": "민준", + "suffix": "씨, Jr."}, + classification="fix(#308)", + notes="the peel indexes the NAME part, not the token stream: " + "under a suffix comma segments[0] is a strict subset, " + "and the peel site is found within it. Pairs with " + "ko_honorific_glued_given_trailing_suffix, whose " + "comma-less spelling of the same name reaches the same " + "answer by the scan-back instead"), Case("zh_honorific_glued_surname", "王先生", {"family": "王", "suffix": "先生"}, locale="zh", diff --git a/tests/v2/pipeline/test_script_segment.py b/tests/v2/pipeline/test_script_segment.py index 3a36c5e..86eefd7 100644 --- a/tests/v2/pipeline/test_script_segment.py +++ b/tests/v2/pipeline/test_script_segment.py @@ -516,7 +516,7 @@ def test_a_trailing_post_nominal_does_not_hide_the_peel_site() -> None: lexicon=_LEX_TAILS)) == ["김", "민준", "씨", "jr"] -def test_the_peel_only_looks_at_the_last_name_token() -> None: +def test_the_peel_only_looks_at_the_last_non_post_nominal_token() -> None: # a tail anywhere but the end is somebody's name, not an # honorific: only the trailing position is an honorific site, the # same reasoning the trailing-only suffix gate follows diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index 3160314..96274f4 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -1,6 +1,7 @@ "Andersonさん" "Anderson선생님" "Dr 김민준, Jr." +"Dr 김민준씨, Jr." "John 王" "〆木 ひろ" "〆木 太郎" From 59aac1fc5c3b922642ac8e08c68e1b4e89f0f159 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 22:21:51 -0700 Subject: [PATCH 09/36] Let a recognized honorific reach the segmenter, not block it (#308) --- nameparser/_pipeline/_script_segment.py | 19 +++++++++++--- tests/v2/pipeline/test_script_segment.py | 17 +++++++++++- tests/v2/test_locales.py | 33 ++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 272a24d..103c48e 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -46,10 +46,13 @@ so it is consulted only when the gated token is the name part's ONLY script-written one -- "山田 太郎" was divided by its writer and must not have its family divided again (a Latin title or suffix draws no -such boundary). A segmenter's own exceptions PROPAGATE -- the single -declared exception to parse totality (locales spec section 4): a -user-supplied callable's error is a user-code error, not a content -error. +such boundary). The neighbour counts whatever script it is written +in, ACTIVATED or not -- effective_script is merely non-None -- unless +the vocabulary already knows it is a post-nominal, which no writer +draws a name boundary with (#308). A segmenter's own exceptions +PROPAGATE -- the single declared exception to parse totality (locales +spec section 4): a user-supplied callable's error is a user-code +error, not a content error. Placed AFTER segment, on the comma doctrine that script-conditional behavior is ignored where a comma already decides the family (the @@ -398,7 +401,15 @@ def script_segment(state: ParseState) -> ParseState: # Jr." still reaches the segmenter. Vocabulary keeps its own rule # -- a listed surname is a certainty about that exact string, # whoever else stands beside it. + # That carve-out is about being VOCABULARY, not about being Latin, + # and it was written in terms of script only because every suffix + # WAS Latin when it was written. #307 shipped CJK honorifics and + # #308's peel manufactures one, so the test asks the vocabulary: + # 様 beside 山田太郎 -- glued or spaced -- says nothing about where + # the kanji name divides, and reading it as the writer's own + # boundary left the commonest addressed form undivided. if any(j != i and effective_script(state.tokens[j].text) is not None + and not _is_post_nominal(state, j) for j in state.segments[0]): return state # No try/except around the call: the module docstring's totality diff --git a/tests/v2/pipeline/test_script_segment.py b/tests/v2/pipeline/test_script_segment.py index 86eefd7..801f4ca 100644 --- a/tests/v2/pipeline/test_script_segment.py +++ b/tests/v2/pipeline/test_script_segment.py @@ -30,7 +30,7 @@ # as the shipped config asserts they are: the neighbour rule in the # segmenter tests below reads that membership, so a stage lexicon that # omitted it would pin a shape no real configuration can have. -_TAILS = frozenset({"씨", "님", "선생님", "박사", "さん", "先生"}) +_TAILS = frozenset({"씨", "님", "선생님", "박사", "さん", "先生", "様"}) _LEX_TAILS = Lexicon( surnames=frozenset({"毛", "欧", "欧阳", "김", "남", "남궁"}), suffix_words=frozenset({"jr"}) | _TAILS, @@ -408,6 +408,21 @@ def test_a_neighbour_in_an_UNACTIVATED_script_still_blocks_the_consult() -> None assert out.ambiguities == (), name +def test_a_recognized_suffix_neighbour_does_not_block_the_consult() -> None: + # The carve-out the docstring already makes for a Latin title or + # suffix, asked of the VOCABULARY instead of the script: an + # honorific is not a boundary its writer drew between family and + # given, whatever script it is written in. Both spellings reach + # the segmenter with the NAME alone -- the glued one after the + # #308 peel manufactured the neighbour, the spaced one because its + # writer wrote it that way. + for name in ("山田太郎様", "山田太郎 様"): + asked, seg = _capture() + out = _run(name, policy=_JA, lexicon=_LEX_TAILS, segmenter=seg) + assert asked == ["山田太郎"], name + assert _texts(out) == ["山田", "太郎", "様"], name + + # -- the glued honorific peel (#308) ----------------------------------- diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index c3dcfaa..e250fd6 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -442,6 +442,39 @@ def test_ja_leaves_spaced_names_to_the_default_order() -> None: assert p.parse("山田 太郎").family == "山田" +@_needs_ja +def test_ja_divides_a_name_carrying_an_honorific() -> None: + # #308 end to end, with the real divider: the honorific peels off + # first, so namedivider is handed 山田太郎 rather than 山田太郎様 + # -- which it would have divided somewhere wrong, answering for + # any string it is given. The spaced spelling composes the same + # way: a recognized honorific is not a boundary its writer drew. + p = _PACKED["ja"] + for name in ("山田太郎様", "山田太郎 様"): + n = p.parse(name) + assert (n.family, n.given, n.suffix) == ("山田", "太郎", "様"), name + # the kana-licensed composite too, whose division is rule-based + n = p.parse("高橋みなみ様") + assert (n.family, n.given, n.suffix) == ("高橋", "みなみ", "様") + + +@_needs_ja +def test_ja_divides_a_two_character_name_under_an_honorific() -> None: + # The consequence of the rule above on the commonest addressed + # form, pinned so it is a recorded decision rather than a side + # effect: with the honorific no longer blocking the consult, a + # two-character remainder reaches namedivider, which divides it + # one character each way BY RULE (score 1.0, so no SEGMENTATION + # report). That is the same presumption locales/ja.py already + # documents and test_ja_reports_statistical_divisions... pins for + # 原恵 -- 田中さん is simply the shortest route to it. A caller + # who wants 田中 kept whole is asking not to have unspaced names + # divided, which is what declining the pack means. + n = _PACKED["ja"].parse("田中さん") + assert (n.family, n.given, n.suffix) == ("田", "中", "さん") + assert n.ambiguities == () # rule-based, so nothing to report + + @_needs_ja def test_ja_reports_statistical_divisions_and_not_rule_based_ones() -> None: # the confidence floor's measured consequence (_script_segment.py's From f71a2e4b0167ab82077cebda62c82cad2f319c89 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 22:37:11 -0700 Subject: [PATCH 10/36] Pin that an honorific does not excuse a real boundary (#308) --- nameparser/_pipeline/_script_segment.py | 41 ++++++++++++++---------- tests/v2/pipeline/test_script_segment.py | 14 ++++++++ tests/v2/test_locales.py | 3 +- 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 103c48e..c518263 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -45,11 +45,10 @@ has no twin of: a segmenter answers where an UNDIVIDED name divides, so it is consulted only when the gated token is the name part's ONLY script-written one -- "山田 太郎" was divided by its writer and must -not have its family divided again (a Latin title or suffix draws no -such boundary). The neighbour counts whatever script it is written -in, ACTIVATED or not -- effective_script is merely non-None -- unless -the vocabulary already knows it is a post-nominal, which no writer -draws a name boundary with (#308). A segmenter's own exceptions +not have its family divided again (a title or post-nominal draws no +such boundary, whatever script it is in). The neighbour test reads +effective_script -- merely non-None -- with a post-nominal excluded by +vocabulary rather than by script (#308). A segmenter's own exceptions PROPAGATE -- the single declared exception to parse totality (locales spec section 4): a user-supplied callable's error is a user-code error, not a content error. @@ -222,11 +221,18 @@ def _longest_entry(entries: frozenset[str]) -> int: def _is_post_nominal(state: ParseState, i: int) -> bool: """Whether token `i` is vocabulary the parse will read as a - post-nominal. Two of this stage's decisions ask it and neither is - about script: an honorific is not part of the name, so it is - neither a surname SITE nor the token a glued honorific hangs off. - (#308; the segmenter's neighbour test joins them in the next - change, for the same reason.)""" + post-nominal. Three of this stage's decisions ask it and none of + them is about script: an honorific is neither a surname SITE, nor + the token a glued honorific hangs off, nor a boundary its writer + drew between family and given (#308). + + Suffixes only, deliberately. The neighbour rule's prose covers "a + Latin title or suffix", but a Latin title needs no test here -- + effective_script gates it out first. Should a CJK entry ever join + titles, the surname site and the neighbour test would both want it + excluded while the peel site must not: a title never trails, so + widening the peel's scan-back would move it off the real last name + token. Split the predicate then, not before.""" return is_suffix_strict(state.tokens[i].text, state.lexicon) @@ -401,13 +407,14 @@ def script_segment(state: ParseState) -> ParseState: # Jr." still reaches the segmenter. Vocabulary keeps its own rule # -- a listed surname is a certainty about that exact string, # whoever else stands beside it. - # That carve-out is about being VOCABULARY, not about being Latin, - # and it was written in terms of script only because every suffix - # WAS Latin when it was written. #307 shipped CJK honorifics and - # #308's peel manufactures one, so the test asks the vocabulary: - # 様 beside 山田太郎 -- glued or spaced -- says nothing about where - # the kanji name divides, and reading it as the writer's own - # boundary left the commonest addressed form undivided. + # The Latin carve-out is about being VOCABULARY, not about being + # Latin, and it was written in terms of script only because every + # suffix WAS Latin when it was written. #307 shipped CJK honorifics + # and #308's peel manufactures one, so the test asks the + # vocabulary: 様 beside 山田太郎 -- glued or spaced -- says nothing + # about where the kanji name divides, and reading it as the + # writer's own boundary left the commonest addressed form + # undivided. if any(j != i and effective_script(state.tokens[j].text) is not None and not _is_post_nominal(state, j) for j in state.segments[0]): diff --git a/tests/v2/pipeline/test_script_segment.py b/tests/v2/pipeline/test_script_segment.py index 801f4ca..558e06d 100644 --- a/tests/v2/pipeline/test_script_segment.py +++ b/tests/v2/pipeline/test_script_segment.py @@ -423,6 +423,20 @@ def test_a_recognized_suffix_neighbour_does_not_block_the_consult() -> None: assert _texts(out) == ["山田", "太郎", "様"], name +def test_an_honorific_does_not_excuse_a_real_boundary() -> None: + # The honorific stops counting, and nothing else does: 太郎 is + # still a boundary its writer drew, so the name is already + # divided and the segmenter must not be asked. Without this the + # rule reads as "a post-nominal anywhere disables the neighbour + # test", which passes every other test in this file and divides + # 山田 into 山 + 田. + asked, seg = _capture() + out = _run("山田 太郎 様", policy=_JA, lexicon=_LEX_TAILS, + segmenter=seg) + assert asked == [] + assert _texts(out) == ["山田", "太郎", "様"] + + # -- the glued honorific peel (#308) ----------------------------------- diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index e250fd6..a25e356 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -437,7 +437,8 @@ def test_ja_leaves_spaced_names_to_the_default_order() -> None: # would wreck the commonest written form -- 山田 + 太郎 divided # again into 山 + 田 + 太郎. p = _PACKED["ja"] - for name in ("山田 太郎", "高橋 みなみ", "高橋 エミ", "山田 太郎 Jr."): + for name in ("山田 太郎", "高橋 みなみ", "高橋 エミ", "山田 太郎 Jr.", + "山田 太郎 様"): assert p.parse(name).as_dict() == _default_parse(name), name assert p.parse("山田 太郎").family == "山田" From d56fff2ece984d06ff08a89506252fb5593494af Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 22:40:33 -0700 Subject: [PATCH 11/36] Classify the glued-honorific diffs in the differential (#308) --- tools/differential/expected_changes.toml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index 548f925..f6df8c9 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -198,11 +198,16 @@ issue = "fix(cjk-honorific-suffix) postnominal honorifics recognized, compoundin # 김지양, the spaced 김 지양 -- would match and could have a real # regression absorbed as intentional (the case table pins the # parser-side twin: suffix matching is whole-token, never endswith). -# Glued surname+honorific diffs (김씨 -- reached via segmentation) -# fall to the fields-only suffix-routing rule by their -# {first, last, suffix} shape (v1 held the whole token in first, so -# last moves too), which is field-honest even though that rule's -# prose describes the two-token Latin case. +# GLUED diffs (田中さん, 김민준씨, and the 김씨 that segmentation +# already reached) never match this rule -- the anchor is judged on +# the name STRING, and a glued form has no space before its +# honorific. They fall to the fields-only suffix-routing rule by +# their {first, last, suffix} shape (v1 held the whole token in +# first, so last moves too), which is field-honest even though that +# rule's prose describes the two-token Latin case. Verified by the +# 2026-07-30 run, not assumed: #308's peel put a dozen glued names +# in the corpus (Latin+glued like Andersonさん included) and every +# one classified there -- none carried a middle-field diff. name_regex = "(?:^| )(?:씨|박사|선생님|교수님|군|양|님|先生|女士|小姐|博士|教授|様|氏|殿|さん|さま|くん|ちゃん)$" fields = ["first", "middle", "last", "suffix"] From aba6c035b871ecce94905c940b78048cc085e62d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 22:48:22 -0700 Subject: [PATCH 12/36] Document the glued honorific peel (#308) --- AGENTS.md | 4 ++-- docs/customize.rst | 3 ++- docs/release_log.rst | 4 +++- docs/usage.rst | 34 +++++++++++++++++++++++++--------- tests/v2/cases.py | 12 ++++++++++-- 5 files changed, 42 insertions(+), 15 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 537bd09..268e6e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,14 +75,14 @@ logging.getLogger('HumanName').setLevel(logging.DEBUG) The library has two layers: `nameparser/config/` (data) and `nameparser/parser.py` (logic). -**Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271/#272): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Five defaults fall out of it. Wholly-Han, wholly-Hangul and kana-licensed names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Hiragana licenses Japanese (#272) — a name whose characters stay inside Han∪kana while carrying at least one kana cannot be Chinese (the kana rules it out) and is not a transcription (foreign names are transcribed in katakana ALONE, マイケル has no kanji), so 高橋みなみ and 山田 エミ read family-first too; mechanically they resolve to the HIRAGANA entry, the license's carrier key. PURE katakana is excluded and keeps the positional default: マイケル・ジャクソン is a transcribed foreign name in its source order. And the 间隔号 U+00B7 (#298) is the transcription marker for scripts that HAVE no transcription script: a name it divides (威廉·莎士比亚 — flanked by classified characters on both sides, so Catalan's Gal·la is untouched) keeps its source order and never segments — the orthography names the convention, exactly as pure katakana does, with the divider carrying the signal instead of the script. The nakaguro belongs to the same doctrine but is decided a layer down: U+30FB and its halfwidth twin U+FF65 separate tokens like whitespace, unconditionally and in tokenize, so neither policy opt-out (`script_orders={}`, `segment_scripts=()`) reaches it — the codepoints are CJK-only and appear in no other script's names, which is what licenses a tokenize-level rule where U+00B7 (also the Catalan punt volat, interior to Gal·la) needs the flanked-by-classified-script guard `_tokenize_region` gives it (#298). Han segmentation stays OPT-IN (`locales.ZH` for Chinese, `locales.JA` for Japanese) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎; no surname list divides a kanji name at all, so `locales.JA` activates the stage and a pluggable `Parser(segmenter=...)` does the dividing. Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. +**Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271/#272): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Six defaults fall out of it. Wholly-Han, wholly-Hangul and kana-licensed names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Hiragana licenses Japanese (#272) — a name whose characters stay inside Han∪kana while carrying at least one kana cannot be Chinese (the kana rules it out) and is not a transcription (foreign names are transcribed in katakana ALONE, マイケル has no kanji), so 高橋みなみ and 山田 エミ read family-first too; mechanically they resolve to the HIRAGANA entry, the license's carrier key. PURE katakana is excluded and keeps the positional default: マイケル・ジャクソン is a transcribed foreign name in its source order. And the 间隔号 U+00B7 (#298) is the transcription marker for scripts that HAVE no transcription script: a name it divides (威廉·莎士比亚 — flanked by classified characters on both sides, so Catalan's Gal·la is untouched) keeps its source order and never segments — the orthography names the convention, exactly as pure katakana does, with the divider carrying the signal instead of the script. And a listed CJK honorific glued to the END of a name token is split off it (#308) — 田中さん is 田中 plus さん — on the same orthography-settles-it test, narrowed for the glued position: an entry peels only where it can never end a name, so 씨/님/さん/様/先生 peel while 양/군/氏/博士/殿 stay spaced-only (김지양 and 田中博士 are names, and ~90 Japanese surnames end in 殿) and 君 is in NEITHER set (王君 is a complete Chinese name), though its kana spelling くん peels. Like the nakaguro's tokenize-level separation described next, it is reached by neither policy opt-out — but for its own reason: the vocabulary carries the license itself rather than borrowing the script's, so `segment_scripts` has nothing to say about it. The nakaguro belongs to the same doctrine but is decided a layer down: U+30FB and its halfwidth twin U+FF65 separate tokens like whitespace, unconditionally and in tokenize, so neither policy opt-out (`script_orders={}`, `segment_scripts=()`) reaches it — the codepoints are CJK-only and appear in no other script's names, which is what licenses a tokenize-level rule where U+00B7 (also the Catalan punt volat, interior to Gal·la) needs the flanked-by-classified-script guard `_tokenize_region` gives it (#298). Han segmentation stays OPT-IN (`locales.ZH` for Chinese, `locales.JA` for Japanese) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎; no surname list divides a kanji name at all, so `locales.JA` activates the stage and a pluggable `Parser(segmenter=...)` does the dividing. Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. ### Configuration layer (`nameparser/config/`) Most modules define a plain Python set of known name pieces; `capitalization.py` and `regexes.py` define dicts: - `titles.py` — `TITLES` (prenominals) and `FIRST_NAME_TITLES` (e.g. "Sir", which treat the following name as first, not last) -- `suffixes.py` — `SUFFIX_ACRONYMS` (with periods, e.g. "M.D.") and `SUFFIX_NOT_ACRONYMS` (e.g. "Jr.") +- `suffixes.py` — `SUFFIX_ACRONYMS` (with periods, e.g. "M.D.") and `SUFFIX_NOT_ACRONYMS` (e.g. "Jr."), plus `GLUED_HONORIFICS` (#308), the subset of `SUFFIX_NOT_ACRONYMS` the peel may split off the END of a name token — a separate, harsher set, since the glued position has no writer-drawn boundary to lean on - `prefixes.py` — `PREFIXES` (lastname particles, e.g. "de", "van") - `bound_first_names.py` — `BOUND_FIRST_NAMES` (bound given-name prefixes, e.g. "abdul", "abu"); `_join_bound_first_name` joins the first non-title piece to its following piece before the main assignment loop - `conjunctions.py` — `CONJUNCTIONS` (e.g. "and", "of") used to chain multi-word titles diff --git a/docs/customize.rst b/docs/customize.rst index 2f29d70..7e79593 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -44,7 +44,8 @@ Removing works the same way, and drops the word from recognition: A few fields mark a subset of another — ``given_name_titles`` over ``titles``, ``particles_ambiguous`` over ``particles``, -``suffix_acronyms_ambiguous`` over ``suffix_acronyms``. Entries must +``suffix_acronyms_ambiguous`` over ``suffix_acronyms``, and +``honorific_tails`` over ``suffix_words``. Entries must appear in the base field too, so add to both and remove from the marker first; anything else raises ``ValueError`` naming the orphans rather than leaving a marker entry that no rule will ever consult. diff --git a/docs/release_log.rst b/docs/release_log.rst index f53936e..3842e48 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -6,6 +6,7 @@ Release Log - Add the Chinese locale pack ``locales.ZH`` — opt-in Han segmentation for unspaced names like ``毛泽东``, with the surname vocabulary it needs. A pack rather than a default because a Chinese surname list corrupts the Japanese names written in the same characters (``高橋一郎`` would split ``高`` + ``橋一郎``); Japanese data goes through ``locales.JA`` and its segmenter instead. It sets no name order, since native-script Han already reads family-first without it (#271) - Add ``Lexicon.surnames``, ``Policy.script_orders``, ``Policy.segment_scripts``, the ``Script`` enum and the ``DEFAULT_SCRIPT_ORDERS`` constant to the public API. This is the first behavior nameparser keys on the script a name is written in; it is allowed only where the script itself settles a convention, never as a proxy for guessing the language (#271) + - Add ``Lexicon.honorific_tails``, the vocabulary the glued-honorific peel matches: entries that may be split off the END of a name token, matched longest-first. Deliberately a separate, narrower set than the spaced honorific vocabulary — a glued tail has no token boundary to lean on — and every entry is also a ``suffix_words`` entry, since the peeled piece is claimed by ordinary suffix classification. Extend it like any other vocabulary field: ``Lexicon.default().add(honorific_tails={"..."})`` (#308) - Add ``AmbiguityKind.SEGMENTATION``, reported when a surname split had a vocabulary-supported alternative: ``"남궁민수"`` is 남궁 + 민수 by the compound surname but 남 + 궁민수 by the single-syllable one, and longest-match had to pick. A name with only one possible split decided nothing and reports nothing (#271) - Add the Japanese locale pack ``locales.JA`` and the segmenter factory ``locales.ja_segmenter()``, which together divide an unspaced Japanese name: ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` reads ``山田太郎`` as family ``山田``, given ``太郎``. The two halves are separate because no surname list can do this job — family and given names draw on the same kanji and the reading, not the spelling, decides most divisions — so the pack activates the stage and a third-party divider performs it. ``ja_segmenter()`` wraps `namedivider-python `_, installed with the new ``nameparser[ja]`` extra; the core stays dependency-free, and ``ja_segmenter(gbdt=True)`` selects namedivider's more accurate gradient-boosted model, which downloads its data on first use. With both packs registered, ``locales.available()`` is now ``('ja', 'ru', 'tr_az', 'zh')`` (closes #272) - Add ``Segmentation`` and the ``Segmenter`` type alias to the public API, plus the keyword-only ``Parser(segmenter=...)`` hook they describe: any callable from a token's text to a ``Segmentation`` (the interior offsets to cut at, and a confidence) or ``None`` to decline. It is consulted only for scripts listed in ``Policy.segment_scripts``, and only where the surname vocabulary declined first, so ``parser_for(locales.ZH, locales.JA, segmenter=...)`` composes — a listed Chinese surname wins, the segmenter takes the rest. Read that composition with the zh pack's own warning still attached: vocabulary-first means a Japanese kanji name opening on a listed Chinese surname never reaches the segmenter, so ``高橋一郎`` still splits ``高`` + ``橋一郎`` under the stack exactly as it does under ``locales.ZH`` alone. The two packs are alternatives, one per corpus; stack them only for genuinely mixed data that accepts that trade (#272) @@ -25,7 +26,8 @@ Release Log - Fix names containing 〆 (U+3006, the shime mark that opens Japanese surnames like 〆木 and 〆谷) parsing given-first: the script classifier now counts 〆 as Han, extending the 々 entry the table already carries, and going a step further than it — 々 is Script=Han and merely outside the ideograph blocks, while 〆 is Script=Common — so these names take the East Asian family-first reading like any other wholly-Han name. ``Policy(script_orders={})`` restores the positional reading for these names exactly as for other wholly-Han names (#303) - Fix the katakana middle dot ``・`` (U+30FB, and its halfwidth twin U+FF65) being read as part of a name rather than as the divider it is: it now separates tokens exactly as a space does. A transcribed foreign name therefore divides into its parts and, being wholly katakana, keeps its source order — ``"マイケル・ジャクソン"`` gives given ``マイケル``, family ``ジャクソン``, where 1.x left the whole string in ``first`` — while a kanji pair written the same way takes the family-first rule (``"高橋・一郎"`` → family ``高橋``). Native Japanese names never contain this character and no other script's names use it, so the separation is unconditional, which also means it is **not** covered by the two policy opt-outs. Rendering follows, the way the Korean split's does: the dot comes back as a space, so ``str(HumanName("マイケル・ジャクソン"))`` is now ``"マイケル ジャクソン"``, and that reaches delimited content too — the nickname in ``"山田 太郎 (マイケル・ジャクソン)"`` renders ``"マイケル ジャクソン"``. The Chinese interpunct ``·`` is not an unconditional separator like these two: U+00B7 is also the Catalan punt volat and appears inside legitimate names (``Gal·la``), so it divides only between classified-script characters — the transcription treatment it marks is #298's (#272) - Fix 间隔号-divided transcriptions parsing as one unsplit token: U+00B7 — the interpunct Chinese text divides a transcribed foreign name with, ``威廉·莎士比亚`` for William Shakespeare — is now a token separator between characters of a classified script, and a name it divides keeps its source order and is never segmented: the dot is the transcription marker, playing the role pure katakana plays in the kana license. It divides ONLY between classified-script characters, so the Catalan punt volat interior to names like ``Gal·la`` is untouched. The Japanese nakaguro is deliberately not a transcription marker — ``高橋・一郎`` is roster formatting, 姓・名, and keeps its family-first reading — so a transcription typed with the wrong dot (``威廉・莎士比亚``) reads by the convention of the codepoint it was typed with; like the spaced form, only the Chinese dot rescues it. Rendering stays space-joined (``str(HumanName("威廉·莎士比亚"))`` is ``"威廉 莎士比亚"``); a custom ``string_format`` such as ``"{first}·{last}"`` reinstates the dot (#298) - - Fix spaced CJK postnominal honorifics parsing as name parts: 씨, 박사, 선생님, 교수님, 군, 양 (Korean — standardly written as their own token), 先生, 女士, 小姐, 博士, 教授 (Chinese, with 先生/博士/教授 shared with Japanese), and 様, 氏 (Japanese) now route to ``suffix``, so ``王小明 先生`` reads family ``王小明`` where the family-first default had confidently made 先生 the given name. Whole-token matching — which also reaches a glued surname+honorific token, since segmentation splits off the surname first (``김씨`` reads family 김, suffix 씨; ``王先生`` the same under ``locales.ZH``); a glued honorific after a given name (``김민준씨``) and glued kana (``山田太郎様``) stay out of reach, tracked as #308 (closes #307) + - Fix spaced CJK postnominal honorifics parsing as name parts: 씨, 박사, 선생님, 교수님, 군, 양 (Korean — standardly written as their own token), 先生, 女士, 小姐, 博士, 教授 (Chinese, with 先生/博士/教授 shared with Japanese), and 様, 氏 (Japanese) now route to ``suffix``, so ``王小明 先生`` reads family ``王小明`` where the family-first default had confidently made 先生 the given name. Whole-token matching, which also reaches a glued surname+honorific token, since segmentation splits off the surname first (``김씨`` reads family 김, suffix 씨). The glued forms whole-token matching cannot reach are handled by the peel described below (closes #307) + - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Six entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``). One honorific peels, never a stack: ``김민준박사님`` keeps its glued 박사. A configured segmenter benefits twice over: it is handed the name without the honorific, and a recognized honorific beside an undivided name no longer counts as a boundary its writer drew, so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads both ``山田太郎様`` and the spaced ``山田太郎 様`` as family 山田, given 太郎, suffix 様. That last part cuts both ways and is worth knowing before you upgrade: an honorific no longer exempts a name from the pack's division at all, so a family name written alone with one — ``田中 さん``, ``佐藤 氏`` — now divides the way bare ``田中`` already did (family 田, given 中). Declining the pack, or the segmenter, is how a caller opts out of that presumption. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) - Fix NFD-decomposed input missing the East Asian defaults entirely: script classification now normalizes to NFC before deciding, so a Korean or Japanese name typed on macOS — where decomposed text is routine — gets the same order rule as its composed twin, which it silently did not before. Segmentation MATCHING deliberately stays raw, so an unspaced NFD hangul name is ordered correctly but not split, rather than being split in the wrong place. One gotcha worth stating plainly: parse output preserves the encoding it was given, so for NFD input ``name.family == "김"`` is ``False`` even though it is the same name — compare NFC-normalized text when comparing across encodings (#272) - Fix the Ukrainian conjunction ``й`` not joining the pieces around it: it is the euphonic alternate of ``і``, the two chosen by the surrounding vowel and consonant rather than by meaning (``"Олесь і Олена"`` but ``"Марія й Петро"``), so real Ukrainian data carries both spellings and shipping only ``і`` recognized just one of them. ``"Олесь й Олена Коваленки"`` now gives given ``"Олесь й Олена"`` where the ``й`` previously landed in ``middle``. Same treatment as the ``и``/``і`` entries added in 2.0.0, single-letter carve-out included: the conjunction joins only once the name has enough pieces, and a punctuated initial still wins, so ``"Й. Сліпий"`` is unaffected. Raised in a comment on #267 diff --git a/docs/usage.rst b/docs/usage.rst index 4e1b4aa..7162e86 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -295,15 +295,31 @@ Chinese dot rescues the source order. And a comma disables the script behaviors entirely, on the reasoning ``name_order`` already follows: whoever wrote the comma has already said where the family name ends. -Honorifics and degrees follow a CJK name, and the spaced forms are -recognized as suffixes — ``王小明 先生`` reads family 王小明 with 先生 -in ``suffix``, and Korean's standardly-spaced 씨 routes the same way. -A glued honorific is reached in exactly one shape — surname + -honorific, where segmentation splits off the surname and the -honorific is what remains: 김씨 reads family 김 with suffix 씨, and -王先生 the same way under the zh pack. A glued honorific after a -given name (김민준씨, 王小明先生) and glued kana (山田太郎様, 田中さん) -stay part of the name. +Honorifics and degrees follow a CJK name, and both the spaced and the +glued forms are recognized as suffixes: ``王小明 先生`` reads family +王小明 with 先生 in ``suffix``, and so do glued ``田中さん``, +``山田太郎様`` and ``김민준씨`` — the honorific is split off the end of +the last name token before anything else reads the name, so the split +and the order rules see the name without it. Everything downstream +then applies as usual, which is why ``김민준씨`` still divides into +family 김 and given 민준, and why a configured Japanese segmenter is +handed 山田太郎 rather than 山田太郎様. The spaced spelling composes +the same way — an honorific is not a boundary its writer drew between +family and given, so ``山田太郎 様`` divides too. + +The glued reading is deliberately narrower than the spaced one. A +spaced honorific sits behind a boundary its writer drew; a glued one +has only itself to go on, so a word peels off the end of a name only +if it could never BE the end of a name. 씨, 님, 박사, 선생님, 교수님, +さん, さま, くん, ちゃん, 様, 先生, 教授, 女士 and 小姐 qualify; 양, +군, 氏, 博士 and 殿 do not, because 김지양 is a given name, 田中博士 +is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety +Japanese surnames end in 殿 (鵜殿, 真殿). Those stay recognized in +their spaced form, where position settles what the glued form leaves +ambiguous. 君 is recognized in neither form, since 王君 is a complete +Chinese name — though its kana spelling くん peels. One honorific +peels, not a stack: ``김민준박사님`` gives up its 님 and keeps its +glued 박사. A division the parser had to choose is reported rather than hidden. When an unspaced name has more than one vocabulary-supported split — diff --git a/tests/v2/cases.py b/tests/v2/cases.py index dd3c018..aa6f448 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -911,17 +911,25 @@ def __post_init__(self) -> None: "so the undivided 王小明 stays one family name"), Case("ko_suffix_matching_is_whole_token", "김지양", {"family": "김", "given": "지양"}, + classification="fix(#271)", notes="지양 ENDS with the honorific 양 but is a given name: " "suffix matching is whole-token, never endswith -- the " "pin the differential rule's anchor mirrors at the " "name-string level. #308 leaves it alone too: 양 is " "excluded from the glued tail set for exactly this " - "name"), + "name. Classified to #271, not parity: 1.4 gave first " + "김지양 and no last, and it is #271's hangul " + "segmentation plus the CJK order flip that produces " + "these two fields"), Case("ko_surname_yang_leads", "양 미선", {"family": "양", "given": "미선"}, + classification="fix(#271)", notes="양 is both a top-tier surname (Yang) and a shipped " "honorific: position decides, and a surname LEADS -- " - "the trailing-only suffix gate never sees it here"), + "the trailing-only suffix gate never sees it here. " + "Classified to #271, not parity: 1.4 gave first 양, " + "last 미선, and it is the CJK order flip that swaps " + "them"), Case("ko_surname_yang_leads_a_segmentable_given", "양 지훈", {"family": "양", "given": "지훈"}, classification="fix(#271)", From 1d46fc6b020da0ba6200c8d89dc1aedbf4b5ec2d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 22:50:25 -0700 Subject: [PATCH 13/36] =?UTF-8?q?Say=20what=20the=20=E5=90=9B=20row=20actu?= =?UTF-8?q?ally=20pins=20(#308)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/v2/cases.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index aa6f448..02128f3 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -824,8 +824,11 @@ def __post_init__(self) -> None: Case("zh_glued_jun_stays", "王君", {"family": "王君"}, classification="fix(#271)", - notes="likewise: 君 is a common Chinese given-name final, so " - "王君 is a complete name, not Mr. Wang"), + notes="likewise unpeeled: 君 is a common Chinese given-name " + "final, so 王君 is a complete name, not Mr. Wang. " + "Unlike its neighbours here, 君 is in NEITHER set -- " + "there is no spaced entry to fall back on either, and " + "only its kana spelling くん peels"), Case("zh_glued_shi_stays", "王氏", {"family": "王氏"}, classification="fix(#271)", From 73fb92fb785616156db2392e4e93e92ebf1fe432 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 23:01:37 -0700 Subject: [PATCH 14/36] Finish the #308 staleness sweep (#308) --- docs/customize.rst | 24 ++++++++++++++++-------- docs/release_log.rst | 2 +- nameparser/_pipeline/_script_segment.py | 13 ++++++++----- tests/v2/cases.py | 15 +++++++++------ 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 7e79593..bd2a92e 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -45,10 +45,14 @@ Removing works the same way, and drops the word from recognition: A few fields mark a subset of another — ``given_name_titles`` over ``titles``, ``particles_ambiguous`` over ``particles``, ``suffix_acronyms_ambiguous`` over ``suffix_acronyms``, and -``honorific_tails`` over ``suffix_words``. Entries must -appear in the base field too, so add to both and remove from the marker -first; anything else raises ``ValueError`` naming the orphans rather -than leaving a marker entry that no rule will ever consult. +``honorific_tails`` over ``suffix_words``. Entries belong in the base +field too, so add to both and remove from the marker first. The last +three enforce that: anything else raises ``ValueError`` naming the +orphans rather than leaving a marker entry that no rule will ever +consult. ``given_name_titles`` is deliberately unchecked — a title run +is matched as one space-joined string, so a legitimate entry like +``"sir and dame"`` is no single word in ``titles`` — and an orphan +there is inert rather than harmful. Turning title detection off ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -353,10 +357,14 @@ deactivates every script at once, which also stops a parser consulting whatever segmenter it was given. The segmenter has an off-switch as well — ``Parser(segmenter=None)``, which is the default; see :ref:`segmenter-contract` for what one is expected to do with text it -does not handle. One Japanese behavior is not a policy field at all: -the katakana middle dot ・ separates tokens the way a space does, -decided in tokenization, so it applies however these two fields are -set. +does not handle. Two behaviors are not policy fields at all, and apply +however these two fields are set. The katakana middle dot ・ separates +tokens the way a space does, decided in tokenization. And a listed CJK +honorific glued to the end of a name token is split off it — ``田中さん`` +reads family ``田中`` with ``さん`` in ``suffix`` — because the tail +vocabulary carries its own license rather than borrowing a script's: +every entry is a word that can never end a name, so there is no +per-script trust question for ``segment_scripts`` to answer. .. note:: diff --git a/docs/release_log.rst b/docs/release_log.rst index 3842e48..a5d43a2 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -6,7 +6,7 @@ Release Log - Add the Chinese locale pack ``locales.ZH`` — opt-in Han segmentation for unspaced names like ``毛泽东``, with the surname vocabulary it needs. A pack rather than a default because a Chinese surname list corrupts the Japanese names written in the same characters (``高橋一郎`` would split ``高`` + ``橋一郎``); Japanese data goes through ``locales.JA`` and its segmenter instead. It sets no name order, since native-script Han already reads family-first without it (#271) - Add ``Lexicon.surnames``, ``Policy.script_orders``, ``Policy.segment_scripts``, the ``Script`` enum and the ``DEFAULT_SCRIPT_ORDERS`` constant to the public API. This is the first behavior nameparser keys on the script a name is written in; it is allowed only where the script itself settles a convention, never as a proxy for guessing the language (#271) - - Add ``Lexicon.honorific_tails``, the vocabulary the glued-honorific peel matches: entries that may be split off the END of a name token, matched longest-first. Deliberately a separate, narrower set than the spaced honorific vocabulary — a glued tail has no token boundary to lean on — and every entry is also a ``suffix_words`` entry, since the peeled piece is claimed by ordinary suffix classification. Extend it like any other vocabulary field: ``Lexicon.default().add(honorific_tails={"..."})`` (#308) + - Add ``Lexicon.honorific_tails``, the vocabulary the glued-honorific peel matches: entries that may be split off the END of a name token, matched longest-first. Deliberately a separate, narrower set than the spaced honorific vocabulary — a glued tail has no token boundary to lean on — and every entry is also a ``suffix_words`` entry, since the peeled piece is claimed by ordinary suffix classification. That subset rule is enforced, so extend both fields in the one call — ``Lexicon.default().add(suffix_words={"ちゃま"}, honorific_tails={"ちゃま"})``; adding to ``honorific_tails`` alone raises ``ValueError`` naming the orphan (#308) - Add ``AmbiguityKind.SEGMENTATION``, reported when a surname split had a vocabulary-supported alternative: ``"남궁민수"`` is 남궁 + 민수 by the compound surname but 남 + 궁민수 by the single-syllable one, and longest-match had to pick. A name with only one possible split decided nothing and reports nothing (#271) - Add the Japanese locale pack ``locales.JA`` and the segmenter factory ``locales.ja_segmenter()``, which together divide an unspaced Japanese name: ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` reads ``山田太郎`` as family ``山田``, given ``太郎``. The two halves are separate because no surname list can do this job — family and given names draw on the same kanji and the reading, not the spelling, decides most divisions — so the pack activates the stage and a third-party divider performs it. ``ja_segmenter()`` wraps `namedivider-python `_, installed with the new ``nameparser[ja]`` extra; the core stays dependency-free, and ``ja_segmenter(gbdt=True)`` selects namedivider's more accurate gradient-boosted model, which downloads its data on first use. With both packs registered, ``locales.available()`` is now ``('ja', 'ru', 'tr_az', 'zh')`` (closes #272) - Add ``Segmentation`` and the ``Segmenter`` type alias to the public API, plus the keyword-only ``Parser(segmenter=...)`` hook they describe: any callable from a token's text to a ``Segmentation`` (the interior offsets to cut at, and a confidence) or ``None`` to decline. It is consulted only for scripts listed in ``Policy.segment_scripts``, and only where the surname vocabulary declined first, so ``parser_for(locales.ZH, locales.JA, segmenter=...)`` composes — a listed Chinese surname wins, the segmenter takes the rest. Read that composition with the zh pack's own warning still attached: vocabulary-first means a Japanese kanji name opening on a listed Chinese surname never reaches the segmenter, so ``高橋一郎`` still splits ``高`` + ``橋一郎`` under the stack exactly as it does under ``locales.ZH`` alone. The two packs are alternatives, one per corpus; stack them only for genuinely mixed data that accepts that trade (#272) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index c518263..23d55fe 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -1,12 +1,15 @@ -"""Stage: script_segment (#271, #272). +"""Stage: script_segment (#271, #272, #308). Consumes: tokens, segments, structure, interpunct_offsets, segmenter. -Produces: tokens (the first activated-script token of the name -segment split into n+1 sub-slices), segments (index runs remapped past -the insertions), ambiguities (indices likewise remapped, plus a +Produces: tokens, by two independent splits into sub-slices -- a +listed honorific peeled off the END of the name part's last +non-post-nominal token, and the first activated-script token of the +name segment split into n+1 -- segments (index runs remapped past the +insertions), ambiguities (indices likewise remapped, plus a SEGMENTATION report when more than one split was vocabulary-supported, or when a segmenter's answer scored under the confidence floor). -Reads: Policy.segment_scripts, Lexicon.surnames, ParseState.segmenter. +Reads: Policy.segment_scripts, Lexicon.surnames, +Lexicon.honorific_tails, ParseState.segmenter. Unspaced CJK names give tokenize no separator to find, so this stage inserts the missing token boundary by vocabulary: the first token diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 02128f3..69e777c 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -853,12 +853,15 @@ def __post_init__(self) -> None: Case("ko_honorific_glued_via_segmentation", "김씨", {"family": "김", "suffix": "씨"}, classification="fix(#307)", - notes="glued hangul is reached ONLY in the surname+honorific " - "shape: default segmentation splits off the surname, " - "and the honorific is what remains -- a partial " - "delivery of #308 that falls out of stage order. A " - "glued honorific after a GIVEN name (김민준씨, the row " - "below) stays out of reach"), + notes="the one glued shape that was already reachable before " + "#308, which is why this row stays fix(#307) where its " + "neighbours are fix(#308): stage order alone delivered " + "it, since segmentation split 김 off the front and the " + "honorific was whatever remained. The peel now takes 씨 " + "off the END before segmentation is consulted, so the " + "route changed and these fields did not. The glued " + "shapes that needed the peel to be reached at all are " + "the rows around it (김민준씨 below)"), Case("ko_honorific_after_comma", "김민준, 씨", {"family": "김민준", "suffix": "씨"}, classification="fix(#307)", From fdd35273be8f24b308607a423942c73eeb7c7908 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 23:03:30 -0700 Subject: [PATCH 15/36] Scope the doctrine paragraph's count to what it counts (#308) --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 268e6e2..363ca42 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,7 +75,7 @@ logging.getLogger('HumanName').setLevel(logging.DEBUG) The library has two layers: `nameparser/config/` (data) and `nameparser/parser.py` (logic). -**Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271/#272): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Six defaults fall out of it. Wholly-Han, wholly-Hangul and kana-licensed names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Hiragana licenses Japanese (#272) — a name whose characters stay inside Han∪kana while carrying at least one kana cannot be Chinese (the kana rules it out) and is not a transcription (foreign names are transcribed in katakana ALONE, マイケル has no kanji), so 高橋みなみ and 山田 エミ read family-first too; mechanically they resolve to the HIRAGANA entry, the license's carrier key. PURE katakana is excluded and keeps the positional default: マイケル・ジャクソン is a transcribed foreign name in its source order. And the 间隔号 U+00B7 (#298) is the transcription marker for scripts that HAVE no transcription script: a name it divides (威廉·莎士比亚 — flanked by classified characters on both sides, so Catalan's Gal·la is untouched) keeps its source order and never segments — the orthography names the convention, exactly as pure katakana does, with the divider carrying the signal instead of the script. And a listed CJK honorific glued to the END of a name token is split off it (#308) — 田中さん is 田中 plus さん — on the same orthography-settles-it test, narrowed for the glued position: an entry peels only where it can never end a name, so 씨/님/さん/様/先生 peel while 양/군/氏/博士/殿 stay spaced-only (김지양 and 田中博士 are names, and ~90 Japanese surnames end in 殿) and 君 is in NEITHER set (王君 is a complete Chinese name), though its kana spelling くん peels. Like the nakaguro's tokenize-level separation described next, it is reached by neither policy opt-out — but for its own reason: the vocabulary carries the license itself rather than borrowing the script's, so `segment_scripts` has nothing to say about it. The nakaguro belongs to the same doctrine but is decided a layer down: U+30FB and its halfwidth twin U+FF65 separate tokens like whitespace, unconditionally and in tokenize, so neither policy opt-out (`script_orders={}`, `segment_scripts=()`) reaches it — the codepoints are CJK-only and appear in no other script's names, which is what licenses a tokenize-level rule where U+00B7 (also the Catalan punt volat, interior to Gal·la) needs the flanked-by-classified-script guard `_tokenize_region` gives it (#298). Han segmentation stays OPT-IN (`locales.ZH` for Chinese, `locales.JA` for Japanese) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎; no surname list divides a kanji name at all, so `locales.JA` activates the stage and a pluggable `Parser(segmenter=...)` does the dividing. Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. +**Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271/#272): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Five defaults fall out of it, plus a sixth that applies the same not-a-guess standard to specific WORDS rather than to a script (#308's honorific peel, below). Wholly-Han, wholly-Hangul and kana-licensed names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Hiragana licenses Japanese (#272) — a name whose characters stay inside Han∪kana while carrying at least one kana cannot be Chinese (the kana rules it out) and is not a transcription (foreign names are transcribed in katakana ALONE, マイケル has no kanji), so 高橋みなみ and 山田 エミ read family-first too; mechanically they resolve to the HIRAGANA entry, the license's carrier key. PURE katakana is excluded and keeps the positional default: マイケル・ジャクソン is a transcribed foreign name in its source order. And the 间隔号 U+00B7 (#298) is the transcription marker for scripts that HAVE no transcription script: a name it divides (威廉·莎士比亚 — flanked by classified characters on both sides, so Catalan's Gal·la is untouched) keeps its source order and never segments — the orthography names the convention, exactly as pure katakana does, with the divider carrying the signal instead of the script. And a listed CJK honorific glued to the END of a name token is split off it (#308) — 田中さん is 田中 plus さん — on the same orthography-settles-it test, narrowed for the glued position: an entry peels only where it can never end a name, so 씨/님/さん/様/先生 peel while 양/군/氏/博士/殿 stay spaced-only (김지양 and 田中博士 are names, and ~90 Japanese surnames end in 殿) and 君 is in NEITHER set (王君 is a complete Chinese name), though its kana spelling くん peels. Like the nakaguro's tokenize-level separation described next, it is reached by neither policy opt-out — but for its own reason: the vocabulary carries the license itself rather than borrowing the script's, so `segment_scripts` has nothing to say about it. The nakaguro belongs to the same doctrine but is decided a layer down: U+30FB and its halfwidth twin U+FF65 separate tokens like whitespace, unconditionally and in tokenize, so neither policy opt-out (`script_orders={}`, `segment_scripts=()`) reaches it — the codepoints are CJK-only and appear in no other script's names, which is what licenses a tokenize-level rule where U+00B7 (also the Catalan punt volat, interior to Gal·la) needs the flanked-by-classified-script guard `_tokenize_region` gives it (#298). Han segmentation stays OPT-IN (`locales.ZH` for Chinese, `locales.JA` for Japanese) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎; no surname list divides a kanji name at all, so `locales.JA` activates the stage and a pluggable `Parser(segmenter=...)` does the dividing. Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. ### Configuration layer (`nameparser/config/`) From 227426d5ed1a06cf2c02e022b9d2e63c057c68e6 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 23:19:19 -0700 Subject: [PATCH 16/36] Note the lone-honorific fix and the peel's off-switch (#308) --- docs/customize.rst | 7 ++++++- docs/release_log.rst | 2 +- nameparser/_pipeline/_script_segment.py | 18 +++++++++++++----- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index bd2a92e..5e4b4cb 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -364,7 +364,12 @@ honorific glued to the end of a name token is split off it — ``田中さん`` reads family ``田中`` with ``さん`` in ``suffix`` — because the tail vocabulary carries its own license rather than borrowing a script's: every entry is a word that can never end a name, so there is no -per-script trust question for ``segment_scripts`` to answer. +per-script trust question for ``segment_scripts`` to answer. That +vocabulary is also the peel's off-switch — +``Lexicon.default().remove(honorific_tails={"さん"})`` leaves +``田中さん`` unsplit while the spaced ``田中 さん`` still reads ``さん`` +as a suffix. Dropping a word from a marker field alone orphans +nothing, so that one needs no matching ``suffix_words`` edit. .. note:: diff --git a/docs/release_log.rst b/docs/release_log.rst index a5d43a2..a9dcc71 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -27,7 +27,7 @@ Release Log - Fix the katakana middle dot ``・`` (U+30FB, and its halfwidth twin U+FF65) being read as part of a name rather than as the divider it is: it now separates tokens exactly as a space does. A transcribed foreign name therefore divides into its parts and, being wholly katakana, keeps its source order — ``"マイケル・ジャクソン"`` gives given ``マイケル``, family ``ジャクソン``, where 1.x left the whole string in ``first`` — while a kanji pair written the same way takes the family-first rule (``"高橋・一郎"`` → family ``高橋``). Native Japanese names never contain this character and no other script's names use it, so the separation is unconditional, which also means it is **not** covered by the two policy opt-outs. Rendering follows, the way the Korean split's does: the dot comes back as a space, so ``str(HumanName("マイケル・ジャクソン"))`` is now ``"マイケル ジャクソン"``, and that reaches delimited content too — the nickname in ``"山田 太郎 (マイケル・ジャクソン)"`` renders ``"マイケル ジャクソン"``. The Chinese interpunct ``·`` is not an unconditional separator like these two: U+00B7 is also the Catalan punt volat and appears inside legitimate names (``Gal·la``), so it divides only between classified-script characters — the transcription treatment it marks is #298's (#272) - Fix 间隔号-divided transcriptions parsing as one unsplit token: U+00B7 — the interpunct Chinese text divides a transcribed foreign name with, ``威廉·莎士比亚`` for William Shakespeare — is now a token separator between characters of a classified script, and a name it divides keeps its source order and is never segmented: the dot is the transcription marker, playing the role pure katakana plays in the kana license. It divides ONLY between classified-script characters, so the Catalan punt volat interior to names like ``Gal·la`` is untouched. The Japanese nakaguro is deliberately not a transcription marker — ``高橋・一郎`` is roster formatting, 姓・名, and keeps its family-first reading — so a transcription typed with the wrong dot (``威廉・莎士比亚``) reads by the convention of the codepoint it was typed with; like the spaced form, only the Chinese dot rescues it. Rendering stays space-joined (``str(HumanName("威廉·莎士比亚"))`` is ``"威廉 莎士比亚"``); a custom ``string_format`` such as ``"{first}·{last}"`` reinstates the dot (#298) - Fix spaced CJK postnominal honorifics parsing as name parts: 씨, 박사, 선생님, 교수님, 군, 양 (Korean — standardly written as their own token), 先生, 女士, 小姐, 博士, 教授 (Chinese, with 先生/博士/教授 shared with Japanese), and 様, 氏 (Japanese) now route to ``suffix``, so ``王小明 先生`` reads family ``王小明`` where the family-first default had confidently made 先生 the given name. Whole-token matching, which also reaches a glued surname+honorific token, since segmentation splits off the surname first (``김씨`` reads family 김, suffix 씨). The glued forms whole-token matching cannot reach are handled by the peel described below (closes #307) - - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Six entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``). One honorific peels, never a stack: ``김민준박사님`` keeps its glued 박사. A configured segmenter benefits twice over: it is handed the name without the honorific, and a recognized honorific beside an undivided name no longer counts as a boundary its writer drew, so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads both ``山田太郎様`` and the spaced ``山田太郎 様`` as family 山田, given 太郎, suffix 様. That last part cuts both ways and is worth knowing before you upgrade: an honorific no longer exempts a name from the pack's division at all, so a family name written alone with one — ``田中 さん``, ``佐藤 氏`` — now divides the way bare ``田中`` already did (family 田, given 中). Declining the pack, or the segmenter, is how a caller opts out of that presumption. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) + - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Six entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``). One honorific peels, never a stack: ``김민준박사님`` keeps its glued 박사. A token that is nothing but an honorific is no longer taken apart either — ``선생님`` and ``박사`` now stay whole where hangul segmentation had split them 선 + 생님 and 박 + 사, since 선 and 박 are listed surnames — which is **default-on** in its own right, hangul segmentation being a default. A configured segmenter benefits twice over: it is handed the name without the honorific, and a recognized honorific beside an undivided name no longer counts as a boundary its writer drew, so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads both ``山田太郎様`` and the spaced ``山田太郎 様`` as family 山田, given 太郎, suffix 様. That last part cuts both ways and is worth knowing before you upgrade: an honorific no longer exempts a name from the pack's division at all, so a family name written alone with one — ``田中 さん``, ``佐藤 氏`` — now divides the way bare ``田中`` already did (family 田, given 中). Declining the pack, or the segmenter, is how a caller opts out of that presumption. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) - Fix NFD-decomposed input missing the East Asian defaults entirely: script classification now normalizes to NFC before deciding, so a Korean or Japanese name typed on macOS — where decomposed text is routine — gets the same order rule as its composed twin, which it silently did not before. Segmentation MATCHING deliberately stays raw, so an unspaced NFD hangul name is ordered correctly but not split, rather than being split in the wrong place. One gotcha worth stating plainly: parse output preserves the encoding it was given, so for NFD input ``name.family == "김"`` is ``False`` even though it is the same name — compare NFC-normalized text when comparing across encodings (#272) - Fix the Ukrainian conjunction ``й`` not joining the pieces around it: it is the euphonic alternate of ``і``, the two chosen by the surrounding vowel and consonant rather than by meaning (``"Олесь і Олена"`` but ``"Марія й Петро"``), so real Ukrainian data carries both spellings and shipping only ``і`` recognized just one of them. ``"Олесь й Олена Коваленки"`` now gives given ``"Олесь й Олена"`` where the ``й`` previously landed in ``middle``. Same treatment as the ``и``/``і`` entries added in 2.0.0, single-letter carve-out included: the conjunction joins only once the name has enough pieces, and a punctuated initial still wins, so ``"Й. Сліпий"`` is unaffected. Raised in a comment on #267 diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 23d55fe..c74841b 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -223,11 +223,19 @@ def _longest_entry(entries: frozenset[str]) -> int: def _is_post_nominal(state: ParseState, i: int) -> bool: - """Whether token `i` is vocabulary the parse will read as a - post-nominal. Three of this stage's decisions ask it and none of - them is about script: an honorific is neither a surname SITE, nor - the token a glued honorific hangs off, nor a boundary its writer - drew between family and given (#308). + """Whether token `i` is post-nominal VOCABULARY. Three of this + stage's decisions ask it and none of them is about script: an + honorific is neither a surname SITE, nor the token a glued + honorific hangs off, nor a boundary its writer drew between family + and given (#308). + + It answers a VOCABULARY question, not a positional one, and the + three callers do not spend the answer the same way. In TRAILING + position vocabulary and position agree, so the peel site and the + neighbour test step over a True and go on scanning. In LEADING + position they disagree -- 양 is the family name there, whatever + the suffix set says -- so the surname site reads a True as an + ANSWER and declines, rather than as a token to step past. Suffixes only, deliberately. The neighbour rule's prose covers "a Latin title or suffix", but a Latin title needs no test here -- From 063fd1a9378edaedd07ebf69594e3d234f3d8258 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 03:16:19 -0700 Subject: [PATCH 17/36] Exempt only the tail the peel manufactured (#308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The neighbour precondition exempted any recognized post-nominal, which reached well past the case it was written for: it broke the SPACED forms, whose writers drew an actual boundary. Under the JA pack "佐藤 氏" divided 佐 + 藤, "田中 様" 田 + 中, "鈴木 先生" 鈴 + 木, "中村 教授" 中 + 村 -- while glued 佐藤氏, which never peels (氏 is deliberately not a tail), stayed correct. That inverts the branch's own doctrine: the spaced position is a token boundary its writer drew. Narrow the carve-out to the tail the peel MANUFACTURED. The peel tags the token it cuts off and the neighbour test reads that tag instead of asking the vocabulary -- which cannot separate the two spellings, since they put the same word in the same place and only the provenance differs. A glued honorific means the writer drew no boundary anywhere; a spaced one means they drew one and wrote the name beside it as a unit. The motivating fix is untouched: glued 山田太郎様 still reaches the segmenter as 山田太郎, and 田中さん still divides under the pack. _is_post_nominal loses a caller and says so. --- nameparser/_pipeline/_script_segment.py | 95 +++++++++++++++--------- tests/v2/pipeline/test_script_segment.py | 48 ++++++------ tests/v2/test_locales.py | 41 +++++++--- 3 files changed, 119 insertions(+), 65 deletions(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index c74841b..0459514 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -48,10 +48,15 @@ has no twin of: a segmenter answers where an UNDIVIDED name divides, so it is consulted only when the gated token is the name part's ONLY script-written one -- "山田 太郎" was divided by its writer and must -not have its family divided again (a title or post-nominal draws no -such boundary, whatever script it is in). The neighbour test reads -effective_script -- merely non-None -- with a post-nominal excluded by -vocabulary rather than by script (#308). A segmenter's own exceptions +not have its family divided again (a Latin title or suffix draws no +such boundary either, and effective_script gates those out before the +test is reached). The neighbour test reads effective_script -- merely +non-None -- and exempts exactly one token: the tail the peel above +MANUFACTURED (#308), which is a boundary nobody drew. A SPACED +honorific is not exempt, and the distinction is provenance rather +than vocabulary: glued 山田太郎様 was written undivided, while whoever +wrote "佐藤 氏" drew that boundary and wrote 佐藤 as a unit beside it, +so 佐藤 stays whole. A segmenter's own exceptions PROPAGATE -- the single declared exception to parse totality (locales spec section 4): a user-supplied callable's error is a user-code error, not a content error. @@ -113,6 +118,16 @@ from nameparser._pipeline._vocab import effective_script, is_suffix_strict from nameparser._types import AmbiguityKind, Segmentation, Span +#: Marks the tail token the peel below MANUFACTURED, so the segmenter's +#: neighbour test can tell it from a token somebody wrote. Namespaced, +#: therefore unstable provenance rather than API (_types.STABLE_TAGS is +#: the whole stable set, and FOLDED_TAG is the precedent for a +#: structural marker carrying this prefix); it is vocabulary-derived +#: besides, since honorific_tails is what licensed the split. Emitter +#: and reader are both in this module, so unlike FOLDED_TAG it needs no +#: home in _types. +_PEELED_TAG = "vocab:peeled-honorific" + #: Segmenter answers scoring below this attach a SEGMENTATION report #: (amendment 2026-07-29 section 3). Kept at the drafted 0.9 after #: measuring namedivider 0.4.1 over 112 names (#272 Task 5), which @@ -223,27 +238,24 @@ def _longest_entry(entries: frozenset[str]) -> int: def _is_post_nominal(state: ParseState, i: int) -> bool: - """Whether token `i` is post-nominal VOCABULARY. Three of this - stage's decisions ask it and none of them is about script: an - honorific is neither a surname SITE, nor the token a glued - honorific hangs off, nor a boundary its writer drew between family - and given (#308). - - It answers a VOCABULARY question, not a positional one, and the - three callers do not spend the answer the same way. In TRAILING - position vocabulary and position agree, so the peel site and the - neighbour test step over a True and go on scanning. In LEADING - position they disagree -- 양 is the family name there, whatever - the suffix set says -- so the surname site reads a True as an - ANSWER and declines, rather than as a token to step past. - - Suffixes only, deliberately. The neighbour rule's prose covers "a - Latin title or suffix", but a Latin title needs no test here -- - effective_script gates it out first. Should a CJK entry ever join - titles, the surname site and the neighbour test would both want it - excluded while the peel site must not: a title never trails, so - widening the peel's scan-back would move it off the real last name - token. Split the predicate then, not before.""" + """Whether token `i` is post-nominal VOCABULARY. Two of this + stage's decisions ask it and neither is about script: an honorific + is neither a surname SITE nor the token a glued honorific hangs off + (#308). + + It answers a VOCABULARY question, not a positional one, and the two + callers do not spend the answer the same way. In TRAILING position + vocabulary and position agree, so the peel site steps over a True + and goes on scanning. In LEADING position they disagree -- 양 is + the family name there, whatever the suffix set says -- so the + surname site reads a True as an ANSWER and declines, rather than as + a token to step past. + + Suffixes only, deliberately. Should a CJK entry ever join titles, + the surname site would want it excluded while the peel site must + not: a title never trails, so widening the peel's scan-back would + move it off the real last name token. Split the predicate then, not + before.""" return is_suffix_strict(state.tokens[i].text, state.lexicon) @@ -305,7 +317,18 @@ def _peel_honorific_tail(state: ParseState) -> ParseState: cap = min(_longest_entry(tails), len(text) - 1) for length in range(cap, 0, -1): if text[-length:] in tails: - return _split(state, i, (len(text) - length,), None) + state = _split(state, i, (len(text) - length,), None) + # Tag the tail this stage just MANUFACTURED. The segmenter's + # neighbour test below needs to tell it from a token + # somebody wrote, and no vocabulary question can: the two + # spellings put the same word in the same place, and only + # the provenance differs. + tail = state.tokens[i + 1] + tokens = (state.tokens[:i + 1] + + (dataclasses.replace( + tail, tags=tail.tags | {_PEELED_TAG}),) + + state.tokens[i + 2:]) + return dataclasses.replace(state, tokens=tokens) return state @@ -418,16 +441,18 @@ def script_segment(state: ParseState) -> ParseState: # Jr." still reaches the segmenter. Vocabulary keeps its own rule # -- a listed surname is a certainty about that exact string, # whoever else stands beside it. - # The Latin carve-out is about being VOCABULARY, not about being - # Latin, and it was written in terms of script only because every - # suffix WAS Latin when it was written. #307 shipped CJK honorifics - # and #308's peel manufactures one, so the test asks the - # vocabulary: 様 beside 山田太郎 -- glued or spaced -- says nothing - # about where the kanji name divides, and reading it as the - # writer's own boundary left the commonest addressed form - # undivided. + # The one neighbour that does not count is the one this stage + # MANUFACTURED (#308): a glued 山田太郎様 has no writer-drawn + # boundary anywhere, so the 様 the peel just cut off cannot be read + # as one -- the precondition must see what the writer wrote, which + # was a single undivided token. A SPACED 様 is the opposite case + # and does count: its writer drew that boundary and chose to write + # 山田太郎 as a unit, so "佐藤 氏" leaves 佐藤 whole rather than + # dividing it into 佐 + 藤. Provenance, not vocabulary: the two + # spellings put the same word in the same place, and asking the + # suffix set instead cannot separate them. if any(j != i and effective_script(state.tokens[j].text) is not None - and not _is_post_nominal(state, j) + and _PEELED_TAG not in state.tokens[j].tags for j in state.segments[0]): return state # No try/except around the call: the module docstring's totality diff --git a/tests/v2/pipeline/test_script_segment.py b/tests/v2/pipeline/test_script_segment.py index 558e06d..a249d21 100644 --- a/tests/v2/pipeline/test_script_segment.py +++ b/tests/v2/pipeline/test_script_segment.py @@ -408,30 +408,36 @@ def test_a_neighbour_in_an_UNACTIVATED_script_still_blocks_the_consult() -> None assert out.ambiguities == (), name -def test_a_recognized_suffix_neighbour_does_not_block_the_consult() -> None: - # The carve-out the docstring already makes for a Latin title or - # suffix, asked of the VOCABULARY instead of the script: an - # honorific is not a boundary its writer drew between family and - # given, whatever script it is written in. Both spellings reach - # the segmenter with the NAME alone -- the glued one after the - # #308 peel manufactured the neighbour, the spaced one because its - # writer wrote it that way. - for name in ("山田太郎様", "山田太郎 様"): - asked, seg = _capture() - out = _run(name, policy=_JA, lexicon=_LEX_TAILS, segmenter=seg) - assert asked == ["山田太郎"], name - assert _texts(out) == ["山田", "太郎", "様"], name - - -def test_an_honorific_does_not_excuse_a_real_boundary() -> None: - # The honorific stops counting, and nothing else does: 太郎 is - # still a boundary its writer drew, so the name is already - # divided and the segmenter must not be asked. Without this the - # rule reads as "a post-nominal anywhere disables the neighbour +def test_only_a_manufactured_tail_does_not_block_the_consult() -> None: + # The exemption is PROVENANCE, not vocabulary. A tail the peel + # manufactured is a boundary nobody drew -- glued 山田太郎様 was + # written as one token -- so the segmenter is asked, and asked + # about the name alone. The SPACED spelling of the same honorific + # is the opposite case: its writer drew that boundary and wrote + # 山田太郎 as a unit beside it, so the consult is declined and the + # unit stays whole. Asking the suffix VOCABULARY instead cannot + # tell these apart -- same word, same place -- and answering both + # the first way divided 佐藤 氏 into 佐 + 藤. + asked, seg = _capture() + out = _run("山田太郎様", policy=_JA, lexicon=_LEX_TAILS, segmenter=seg) + assert asked == ["山田太郎"] + assert _texts(out) == ["山田", "太郎", "様"] + + asked, seg = _capture() + out = _run("山田太郎 様", policy=_JA, lexicon=_LEX_TAILS, segmenter=seg) + assert asked == [] + assert _texts(out) == ["山田太郎", "様"] + + +def test_a_manufactured_tail_does_not_excuse_a_real_boundary() -> None: + # The peeled tail stops counting, and nothing else does: 太郎 is + # still a boundary its writer drew, so the name is already divided + # and the segmenter must not be asked. Without the narrowing the + # rule reads as "an exempt token anywhere disables the neighbour # test", which passes every other test in this file and divides # 山田 into 山 + 田. asked, seg = _capture() - out = _run("山田 太郎 様", policy=_JA, lexicon=_LEX_TAILS, + out = _run("山田 太郎様", policy=_JA, lexicon=_LEX_TAILS, segmenter=seg) assert asked == [] assert _texts(out) == ["山田", "太郎", "様"] diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index a25e356..9538890 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -444,36 +444,59 @@ def test_ja_leaves_spaced_names_to_the_default_order() -> None: @_needs_ja -def test_ja_divides_a_name_carrying_an_honorific() -> None: +def test_ja_divides_a_glued_name_carrying_an_honorific() -> None: # #308 end to end, with the real divider: the honorific peels off # first, so namedivider is handed 山田太郎 rather than 山田太郎様 # -- which it would have divided somewhere wrong, answering for - # any string it is given. The spaced spelling composes the same - # way: a recognized honorific is not a boundary its writer drew. + # any string it is given. GLUED only: the writer of 山田太郎様 drew + # no boundary anywhere, so the tail the peel manufactured is none + # either and the consult goes ahead. p = _PACKED["ja"] - for name in ("山田太郎様", "山田太郎 様"): - n = p.parse(name) - assert (n.family, n.given, n.suffix) == ("山田", "太郎", "様"), name + n = p.parse("山田太郎様") + assert (n.family, n.given, n.suffix) == ("山田", "太郎", "様") # the kana-licensed composite too, whose division is rule-based n = p.parse("高橋みなみ様") assert (n.family, n.given, n.suffix) == ("高橋", "みなみ", "様") + # and the spaced spelling is left alone, honorific and all: its + # writer wrote 山田太郎 as a unit, which the pack respects the way + # it respects "山田 太郎" + n = p.parse("山田太郎 様") + assert (n.family, n.given, n.suffix) == ("山田太郎", "", "様") + + +@_needs_ja +def test_ja_does_not_divide_a_spaced_surname_under_an_honorific() -> None: + # The writer drew this boundary: 佐藤 is a unit, and the + # honorific beside it says nothing about where 佐藤 divides. + # Only a tail this stage MANUFACTURED is exempt from the + # neighbour test -- a glued honorific means no boundary was + # drawn anywhere, a spaced one means one was. + p = _PACKED["ja"] + for name, family in (("佐藤 氏", "佐藤"), ("田中 様", "田中"), + ("鈴木 先生", "鈴木")): + n = p.parse(name) + assert (n.family, n.given) == (family, ""), name @_needs_ja -def test_ja_divides_a_two_character_name_under_an_honorific() -> None: +def test_ja_divides_a_two_character_name_under_a_glued_honorific() -> None: # The consequence of the rule above on the commonest addressed # form, pinned so it is a recorded decision rather than a side - # effect: with the honorific no longer blocking the consult, a + # effect: with the peeled tail not blocking the consult, a # two-character remainder reaches namedivider, which divides it # one character each way BY RULE (score 1.0, so no SEGMENTATION # report). That is the same presumption locales/ja.py already # documents and test_ja_reports_statistical_divisions... pins for # 原恵 -- 田中さん is simply the shortest route to it. A caller # who wants 田中 kept whole is asking not to have unspaced names - # divided, which is what declining the pack means. + # divided, which is what declining the pack means -- or can write + # the honorific spaced, which the test above pins. n = _PACKED["ja"].parse("田中さん") assert (n.family, n.given, n.suffix) == ("田", "中", "さん") assert n.ambiguities == () # rule-based, so nothing to report + # the spaced twin, for the contrast in one place + n = _PACKED["ja"].parse("田中 さん") + assert (n.family, n.given, n.suffix) == ("田中", "", "さん") @_needs_ja From 931ce0e6d56c4fd8e85ad1f27e8579cd4708368c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 03:19:43 -0700 Subject: [PATCH 18/36] =?UTF-8?q?Ship=20=EB=B0=95=EC=82=AC=EB=8B=98,=20the?= =?UTF-8?q?=20missing=20third=20-=EB=8B=98=20honorific=20(#308)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 선생님, 교수님 and 박사님 are the three standard -님 professional honorifics; the first two shipped in both sets and the third in neither, which broke both of its spellings. Glued, 김민준박사님 peeled 님 alone and stranded 박사 in the given name; spaced, 김민준 박사님 was cut by the peel -- the token is not a whole-token suffix word -- and came back as two suffixes for one honorific. Both now give suffix 박사님. The case row that used 김민준박사님 to pin the one-peel/no-recursion rule can no longer demonstrate it, and no shipped entry pair has the shape that could: it needs a tail whose remainder ends in a DIFFERENT tail, the two not forming a listed entry themselves. The pin stays at stage level, where a synthetic lexicon can build that shape, and the row is repointed at the behavior it now shows. --- docs/release_log.rst | 2 +- docs/usage.rst | 21 +++++++++--------- nameparser/config/suffixes.py | 18 ++++++++++------ tests/v2/cases.py | 27 ++++++++++++++++++------ tests/v2/pipeline/test_script_segment.py | 9 +++++++- tools/differential/corpus_cjk.jsonl | 1 + tools/differential/expected_changes.toml | 2 +- 7 files changed, 54 insertions(+), 26 deletions(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index a9dcc71..72f980e 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -27,7 +27,7 @@ Release Log - Fix the katakana middle dot ``・`` (U+30FB, and its halfwidth twin U+FF65) being read as part of a name rather than as the divider it is: it now separates tokens exactly as a space does. A transcribed foreign name therefore divides into its parts and, being wholly katakana, keeps its source order — ``"マイケル・ジャクソン"`` gives given ``マイケル``, family ``ジャクソン``, where 1.x left the whole string in ``first`` — while a kanji pair written the same way takes the family-first rule (``"高橋・一郎"`` → family ``高橋``). Native Japanese names never contain this character and no other script's names use it, so the separation is unconditional, which also means it is **not** covered by the two policy opt-outs. Rendering follows, the way the Korean split's does: the dot comes back as a space, so ``str(HumanName("マイケル・ジャクソン"))`` is now ``"マイケル ジャクソン"``, and that reaches delimited content too — the nickname in ``"山田 太郎 (マイケル・ジャクソン)"`` renders ``"マイケル ジャクソン"``. The Chinese interpunct ``·`` is not an unconditional separator like these two: U+00B7 is also the Catalan punt volat and appears inside legitimate names (``Gal·la``), so it divides only between classified-script characters — the transcription treatment it marks is #298's (#272) - Fix 间隔号-divided transcriptions parsing as one unsplit token: U+00B7 — the interpunct Chinese text divides a transcribed foreign name with, ``威廉·莎士比亚`` for William Shakespeare — is now a token separator between characters of a classified script, and a name it divides keeps its source order and is never segmented: the dot is the transcription marker, playing the role pure katakana plays in the kana license. It divides ONLY between classified-script characters, so the Catalan punt volat interior to names like ``Gal·la`` is untouched. The Japanese nakaguro is deliberately not a transcription marker — ``高橋・一郎`` is roster formatting, 姓・名, and keeps its family-first reading — so a transcription typed with the wrong dot (``威廉・莎士比亚``) reads by the convention of the codepoint it was typed with; like the spaced form, only the Chinese dot rescues it. Rendering stays space-joined (``str(HumanName("威廉·莎士比亚"))`` is ``"威廉 莎士比亚"``); a custom ``string_format`` such as ``"{first}·{last}"`` reinstates the dot (#298) - Fix spaced CJK postnominal honorifics parsing as name parts: 씨, 박사, 선생님, 교수님, 군, 양 (Korean — standardly written as their own token), 先生, 女士, 小姐, 博士, 教授 (Chinese, with 先生/博士/教授 shared with Japanese), and 様, 氏 (Japanese) now route to ``suffix``, so ``王小明 先生`` reads family ``王小明`` where the family-first default had confidently made 先生 the given name. Whole-token matching, which also reaches a glued surname+honorific token, since segmentation splits off the surname first (``김씨`` reads family 김, suffix 씨). The glued forms whole-token matching cannot reach are handled by the peel described below (closes #307) - - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Six entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``). One honorific peels, never a stack: ``김민준박사님`` keeps its glued 박사. A token that is nothing but an honorific is no longer taken apart either — ``선생님`` and ``박사`` now stay whole where hangul segmentation had split them 선 + 생님 and 박 + 사, since 선 and 박 are listed surnames — which is **default-on** in its own right, hangul segmentation being a default. A configured segmenter benefits twice over: it is handed the name without the honorific, and a recognized honorific beside an undivided name no longer counts as a boundary its writer drew, so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads both ``山田太郎様`` and the spaced ``山田太郎 様`` as family 山田, given 太郎, suffix 様. That last part cuts both ways and is worth knowing before you upgrade: an honorific no longer exempts a name from the pack's division at all, so a family name written alone with one — ``田中 さん``, ``佐藤 氏`` — now divides the way bare ``田中`` already did (family 田, given 中). Declining the pack, or the segmenter, is how a caller opts out of that presumption. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) + - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 박사님, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Seven entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님, 박사님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``, ``김민준 박사님``). 박사님 closes a gap in the shipped set rather than opening new ground: 선생님 and 교수님 shipped in 2.1 without it, so ``김민준박사님`` stranded 박사 in the given name and the spaced ``김민준 박사님`` came back as two suffixes for one honorific. Exactly one honorific peels off a token, and every entry is a whole honorific rather than a part of one. A token that is nothing but an honorific is no longer taken apart either — ``선생님`` and ``박사`` now stay whole where hangul segmentation had split them 선 + 생님 and 박 + 사, since 선 and 박 are listed surnames — which is **default-on** in its own right, hangul segmentation being a default. A configured segmenter benefits twice over: it is handed the name without the honorific, and a recognized honorific beside an undivided name no longer counts as a boundary its writer drew, so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads both ``山田太郎様`` and the spaced ``山田太郎 様`` as family 山田, given 太郎, suffix 様. That last part cuts both ways and is worth knowing before you upgrade: an honorific no longer exempts a name from the pack's division at all, so a family name written alone with one — ``田中 さん``, ``佐藤 氏`` — now divides the way bare ``田中`` already did (family 田, given 中). Declining the pack, or the segmenter, is how a caller opts out of that presumption. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) - Fix NFD-decomposed input missing the East Asian defaults entirely: script classification now normalizes to NFC before deciding, so a Korean or Japanese name typed on macOS — where decomposed text is routine — gets the same order rule as its composed twin, which it silently did not before. Segmentation MATCHING deliberately stays raw, so an unspaced NFD hangul name is ordered correctly but not split, rather than being split in the wrong place. One gotcha worth stating plainly: parse output preserves the encoding it was given, so for NFD input ``name.family == "김"`` is ``False`` even though it is the same name — compare NFC-normalized text when comparing across encodings (#272) - Fix the Ukrainian conjunction ``й`` not joining the pieces around it: it is the euphonic alternate of ``і``, the two chosen by the surrounding vowel and consonant rather than by meaning (``"Олесь і Олена"`` but ``"Марія й Петро"``), so real Ukrainian data carries both spellings and shipping only ``і`` recognized just one of them. ``"Олесь й Олена Коваленки"`` now gives given ``"Олесь й Олена"`` where the ``й`` previously landed in ``middle``. Same treatment as the ``и``/``і`` entries added in 2.0.0, single-letter carve-out included: the conjunction joins only once the name has enough pieces, and a punctuated initial still wins, so ``"Й. Сліпий"`` is unaffected. Raised in a comment on #267 diff --git a/docs/usage.rst b/docs/usage.rst index 7162e86..3347290 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -310,16 +310,17 @@ family and given, so ``山田太郎 様`` divides too. The glued reading is deliberately narrower than the spaced one. A spaced honorific sits behind a boundary its writer drew; a glued one has only itself to go on, so a word peels off the end of a name only -if it could never BE the end of a name. 씨, 님, 박사, 선생님, 교수님, -さん, さま, くん, ちゃん, 様, 先生, 教授, 女士 and 小姐 qualify; 양, -군, 氏, 博士 and 殿 do not, because 김지양 is a given name, 田中博士 -is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety -Japanese surnames end in 殿 (鵜殿, 真殿). Those stay recognized in -their spaced form, where position settles what the glued form leaves -ambiguous. 君 is recognized in neither form, since 王君 is a complete -Chinese name — though its kana spelling くん peels. One honorific -peels, not a stack: ``김민준박사님`` gives up its 님 and keeps its -glued 박사. +if it could never BE the end of a name. 씨, 님, 박사, 박사님, 선생님, +교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士 and 小姐 +qualify; 양, 군, 氏, 博士 and 殿 do not, because 김지양 is a given +name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some +ninety Japanese surnames end in 殿 (鵜殿, 真殿). Those stay recognized +in their spaced form, where position settles what the glued form +leaves ambiguous. 君 is recognized in neither form, since 王君 is a +complete Chinese name — though its kana spelling くん peels. Exactly +one honorific peels off a token, and the entries are whole +honorifics rather than parts: ``김민준박사님`` gives up 박사님 entire, +not 님 with 박사 left behind. A division the parser had to choose is reported rather than hidden. When an unspaced name has more than one vocabulary-supported split — diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index 37810b3..83608b9 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -68,10 +68,15 @@ # Sino-Korean given name. # Bare hangul 선생/교수 are deliberately absent (only the -님 # honorific forms ship): the bare forms read as common nouns as - # readily as address terms. Further candidates (여사, 太太) wait on - # the same case-by-case argument. + # readily as address terms. 박사 is the exception among the three, + # shipping bare as well as with -님, because it names a degree + # rather than a role -- but all three -님 forms ship together + # (선생님, 교수님, 박사님 are the standard professional honorifics, + # and shipping two of the three was an oversight). Further + # candidates (여사, 太太) wait on the same case-by-case argument. '씨', # ko Mr./Ms. -- standardly spaced in Korean orthography '박사', # ko doctorate holder ("Dr.") + '박사님', # ko the same, honorific form '선생님', # ko teacher/respected elder '교수님', # ko professor (honorific form) '군', # ko young man ("Master") @@ -114,10 +119,11 @@ '様', '先生', '教授', '女士', '小姐', # hangul -- the -님 compounds too: longest-first peels 선생님 off # 김선생님 whole, rather than leaving 김선생 to segment into a - # family 김 and a given 선생. 박사 is safe glued where its Han twin - # 博士 is not: that collision is Japanese (博士 = ひろし) and the - # hangul spelling carries none of it. - '씨', '님', '선생님', '교수님', '박사', + # family 김 and a given 선생, and 박사님 off 김민준박사님 rather + # than stranding 박사 in the given name. 박사 is safe glued where + # its Han twin 博士 is not: that collision is Japanese (博士 = + # ひろし) and the hangul spelling carries none of it. + '씨', '님', '선생님', '교수님', '박사', '박사님', } """ diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 69e777c..c6b4739 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -786,14 +786,27 @@ def __post_init__(self) -> None: "peeled 선생님 would otherwise be split into 선 + 생님 " "-- the stage dissecting the honorific it had just " "manufactured"), - Case("ko_glued_stack_peels_once", "김민준박사님", - {"family": "김", "given": "민준박사", "suffix": "님"}, + Case("ko_honorific_glued_doctor", "김민준박사님", + {"family": "김", "given": "민준", "suffix": "박사님"}, classification="fix(#308)", - notes="one peel, no recursion -- and the remainder 김민준박사 " - "ends in a listed tail of its own, so this is the " - "shape that would recurse if anything did. 님 comes " - "off, the glued 박사 stays in the given name. Accepted " - "and pinned rather than chased"), + notes="박사님 is one honorific, not 박사 plus 님, and ships " + "as one entry: 선생님, 교수님 and 박사님 are the three " + "standard -님 professional honorifics and the first two " + "shipped without it, which stranded 박사 in the given " + "name here and rendered the spaced 김민준 박사님 as two " + "suffixes. This row USED to pin the one-peel rule " + "(named ko_glued_stack_peels_once) on the reading that " + "left 박사 behind; no shipped vocabulary now has that " + "shape, so the pin lives at stage level with a " + "synthetic lexicon -- test_script_segment.py's " + "test_one_peel_never_a_stack"), + Case("ko_honorific_glued_doctor_spaced", "김민준 박사님", + {"family": "김", "given": "민준", "suffix": "박사님"}, + classification="fix(#308)", + notes="the spaced twin, and the second half of the same gap: " + "without a 박사님 entry the peel cut this token too -- " + "it is not a whole-token suffix word, so 박사 + 님 came " + "back as two suffixes for one honorific"), Case("ko_glued_tail_alone_never_peels", "씨", {"family": "씨"}, classification="fix(#271)", diff --git a/tests/v2/pipeline/test_script_segment.py b/tests/v2/pipeline/test_script_segment.py index a249d21..fb210ff 100644 --- a/tests/v2/pipeline/test_script_segment.py +++ b/tests/v2/pipeline/test_script_segment.py @@ -488,7 +488,14 @@ def test_longest_tail_wins() -> None: def test_one_peel_never_a_stack() -> None: # The remainder here ENDS in another listed tail (박사) and is # still not peeled again: one peel, then the stage moves on to - # segmentation, which splits the surname off what is left + # segmentation, which splits the surname off what is left. + # The SOLE home of this pin, and it needs the synthetic lexicon: + # _TAILS deliberately omits 박사님, which the shipped vocabulary + # carries, so 김민준박사님 gives up the whole honorific there (the + # ko_honorific_glued_doctor case row). No shipped entry pair has + # the shape this needs -- a tail whose remainder ends in a + # DIFFERENT tail, the two not forming a listed entry themselves -- + # so no realistic name can carry the pin end to end. assert _texts(_run("김민준박사님", policy=_HANGUL, lexicon=_LEX_TAILS)) == ["김", "민준박사", "님"] diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index 96274f4..e62254c 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -50,6 +50,7 @@ "김민준 님" "김민준 박사" "김민준 박사 씨" +"김민준 박사님" "김민준 씨" "김민준, 씨" "김민준님" diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index f6df8c9..01bcfc7 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -208,7 +208,7 @@ issue = "fix(cjk-honorific-suffix) postnominal honorifics recognized, compoundin # 2026-07-30 run, not assumed: #308's peel put a dozen glued names # in the corpus (Latin+glued like Andersonさん included) and every # one classified there -- none carried a middle-field diff. -name_regex = "(?:^| )(?:씨|박사|선생님|교수님|군|양|님|先生|女士|小姐|博士|教授|様|氏|殿|さん|さま|くん|ちゃん)$" +name_regex = "(?:^| )(?:씨|박사|박사님|선생님|교수님|군|양|님|先生|女士|小姐|博士|教授|様|氏|殿|さん|さま|くん|ちゃん)$" fields = ["first", "middle", "last", "suffix"] [[change]] From cd626cb8d942e307be30dae6342a77805927f70c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 03:21:59 -0700 Subject: [PATCH 19/36] Correct six claims the code makes about itself (#308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing here changes behavior; each is a statement a later reader would have built on. - The ASCII bail justified itself purely in terms of scripts, which was true when only the surname half existed. It now also short-circuits the peel, which has no script gate, so a configured ASCII tail never fires and an accent on an unrelated token switches it on. Both stated at the bail and at the public field, matching rather than changed: correct for the CJK vocabulary that ships, and changing it needs its own vetting pass. - The peel's maiden paragraph documented the direction that fires and not the one that does not: "김민준씨 née 박" still gives given 민준씨, and "김민준씨 née 박씨" routes the maiden's honorific while leaving the person's own. Stated as a limit, with why chasing it here is not possible (classify has not run). - _longest_entry's cache is keyed by the frozenset VALUE, so lexicons sharing KOREAN_SURNAMES collapse to one entry; the "times the two fields" multiplication overstated it. - _SUBSET_FIELDS' header generalized "each marker NARROWS its base", which honorific_tails inverts -- it widens where a suffix word may match. Its base is also narrower than what classifies the peeled piece (suffix_as_written ORs in the non-ambiguous acronyms); that is a trade, recorded so nobody fixes it. - GLUED_HONORIFICS' exclusion list was headed "these ship spaced only" while its last bullet named two entries that ship in neither set. Split the two kinds. - The kana-vs-kanji vetting note claimed hiragana spells no name character, true only within its implicit "in Chinese" -- this project's own case table pins the hiragana given name 高橋みなみ. --- nameparser/_lexicon.py | 30 +++++++++++++++++++----- nameparser/_pipeline/_script_segment.py | 31 +++++++++++++++++++------ nameparser/config/suffixes.py | 19 +++++++++------ 3 files changed, 60 insertions(+), 20 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 67a5e3a..09fac6b 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -27,11 +27,15 @@ "honorific_tails", ) -#: (marker, base, why) triples. Each marker narrows how entries of its -#: base vocabulary are read and carries no vocabulary of its own, so an -#: entry outside the base is a configuration mistake -- but the mistake -#: differs per pair, and the reason is recorded here rather than -#: generalized, because an orphan is NOT simply inert: +#: (marker, base, why) triples. Each marker QUALIFIES how entries of +#: its base vocabulary are read and carries no vocabulary of its own, +#: so an entry outside the base is a configuration mistake -- but the +#: mistake differs per pair, and the reason is recorded here rather +#: than generalized, because an orphan is NOT simply inert. Nor is the +#: qualification one-directional: the first two NARROW their base (an +#: entry is read as vocabulary in fewer places), while honorific_tails +#: WIDENS it, granting a suffix word the glued position on top of the +#: whole-token match every suffix word already gets. #: #: * particles_ambiguous: _assign keys on the tag alone, so an orphan #: makes the parse emit a spurious particle-or-given ambiguity. @@ -41,6 +45,15 @@ #: * honorific_tails: script_segment peels the tail into its own token #: before classify ever runs, so an orphan splits the name and leaves #: the fragment stranded inside it -- worse than not peeling at all. +#: Its base is deliberately NARROWER than what actually claims the +#: peeled piece: suffix_as_written ORs suffix_words with the +#: non-ambiguous suffix_acronyms, so a tail listed only as an acronym +#: would classify fine yet is rejected here. Accepted, and a decision +#: rather than an oversight -- the three-term predicate is easy to +#: get wrong in the dangerous direction (an ambiguous acronym admitted +#: as a tail would peel a period-gated word off a real name), and +#: nothing needs the acronym half: the shipped tails are CJK +#: honorifics, which are words. #: #: given_name_titles is deliberately NOT here and has no check of its #: own -- see the note in __post_init__ for why every attempt at one @@ -350,7 +363,12 @@ class Lexicon: #: ` (unlike :attr:`surnames` #: above): 田中さん peels under the default policy, where HAN is in #: no activation set, because a tail entry carries its own license - #: to fire. Full default list: + #: to fire. Entries are matched against the RAW token text, and + #: only within a name containing a non-ASCII character, so an ASCII + #: or mixed-case entry is at best conditionally active -- a ``"Jr"`` + #: entry is stored ``"jr"`` and matches only lowercase text. The + #: field is effectively CJK-scoped in 2.1, which is what the shipped + #: vocabulary is. Full default list: #: :data:`~nameparser.config.suffixes.GLUED_HONORIFICS`. honorific_tails: frozenset[str] = frozenset() #: Lowercase word -> exact-cased replacement used by capitalized() diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 0459514..e035bc5 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -227,9 +227,12 @@ def _longest_entry(entries: frozenset[str]) -> int: cannot carry a cached_property of its own). The frozenset is hashable and a process holds only a handful of distinct vocabularies -- the default one, plus one per constructed pack - parser, times the two FIELDS that call this (surnames and - honorific_tails) -- so maxsize=16 bounds pathological many-lexicon - churn without ever evicting in normal use. + parser -- so maxsize=16 bounds pathological many-lexicon churn + without ever evicting in normal use. Keyed by the frozenset VALUE, + not by (lexicon, field): the two callers pass surnames and + honorific_tails, but every lexicon that leaves KOREAN_SURNAMES + alone shares one entry, so the count is distinct SETS rather than + lexicons times fields. Callers must pass a NON-EMPTY vocabulary: max() of an empty set raises, and both call sites sit under a match guard that cannot @@ -287,9 +290,18 @@ def _peel_honorific_tail(state: ParseState) -> ParseState: here (extract_delimited has masked only bracketed content), so "김민준 née 박씨" peels 씨 off the MAIDEN name 박씨 and hands it to the person's suffix list -- "née Ms. Park". Intended rather than - incidental: the honorific is the reader's regardless of which of - her names it was glued to, and a name-part-final honorific is - exactly what this peels. + incidental in that direction: the honorific is the reader's + regardless of which of her names it was glued to, and a + name-part-final honorific is exactly what this peels. + + The other direction is a LIMIT, stated rather than fixed: a maiden + clause pushes the site off the person's own name, so "김민준씨 née + 박" does NOT peel and gives given "민준씨" -- the original bug, + intact behind a marker. "김민준씨 née 박씨" shows both at once, two + identical honorifics of which only the maiden's is routed. Chasing + the marker into the site scan is scope creep for an uncommon input, + and it could not be done here anyway: classify has not run, so the + marker tokens carry no tag this stage could read. Longest-first. ONE peel, never recursive: 김민준박사님 gives up its 님 and keeps its glued 박사, though 박사 is itself a listed @@ -336,7 +348,12 @@ def script_segment(state: ParseState) -> ParseState: if state.original.isascii(): # spans index the original exactly (the anti-#100 invariant), # so an ASCII original has only ASCII tokens: nothing here is - # in any script's ranges + # in any script's ranges. It also short-circuits the PEEL, + # which has no script gate of its own -- so a caller-configured + # ASCII tail never fires, and one non-ASCII character anywhere + # in the name switches it on. Correct for the CJK vocabulary + # that ships, stated because honorific_tails is public: see + # that field's own note. return state if not state.segments: return state diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index 83608b9..2b3dfd0 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -110,8 +110,12 @@ # never end one belong here. # kana -- name-final never, in any of the four, and the kana/kanji # split is itself a vetting result: くん ships where 君 cannot, - # since 王君 is a complete Chinese name while hiragana spells no - # name character at all. + # since 王君 is a complete Chinese name while the hiragana spelling + # is unavailable to Chinese at all. Scoped to Chinese on purpose -- + # hiragana of course spells Japanese GIVEN names (高橋みなみ is one, + # pinned in the case table), which is why the four entries above + # are vetted one at a time as never name-FINAL rather than waved + # through as kana. 'さん', 'さま', 'くん', 'ちゃん', # Han -- two-character honorifics with no name-final reading in # either language, plus 様. 殿 is deliberately absent though it @@ -130,7 +134,7 @@ The subset of :data:`SUFFIX_NOT_ACRONYMS` a name token may end WITH, peeled off as its own token before segmentation (#308). Deliberately harsher than the spaced set, because a glued tail has no writer-drawn token boundary to -lean on -- these entries ship spaced only: +lean on -- these entries are recognized in the SPACED position only: * 양, 군 -- 김지양 and 김지군 are given names ending in these syllables, and 양 is a top-tier surname besides. @@ -140,11 +144,12 @@ (Madono) with four-figure populations, so peeling it would cut a real family name in two. Spaced 殿 is safe for the reason 양/군 are: a 殿-surnamed person's name LEADS, and the suffix gate is trailing-only. -* bare 선생, 교수 -- read as common nouns; only the -님 forms ship at all. -君 is in NEITHER set: 王君 is a complete Chinese name (君 is a common -given-name final), so the honorific reading never gets the benefit of the -doubt -- while its kana spelling くん ships glued, above. +Two more are in NEITHER set, so neither spelling is recognized. 君: 王君 is +a complete Chinese name (君 is a common given-name final), so the honorific +reading never gets the benefit of the doubt -- while its kana spelling くん +ships glued, above. Bare 선생 and 교수: they read as common nouns as readily +as address terms, and only their -님 forms ship. """ SUFFIX_ACRONYMS_AMBIGUOUS = { From a527bd6424a9602134a308b8ae18cdcf66c4fae5 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 03:23:05 -0700 Subject: [PATCH 20/36] Pin the v1 shim's honorific_tails translation (#308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit honorific_tails appeared nowhere in tests/v2/test_config_shim.py, and the snapshot-equality test passes VACUOUSLY here -- with the default config the GLUED_HONORIFICS ∩ suffix_words intersection is a no-op, which is exactly the failure mode AGENTS.md warns about. Pin the case the translation decides: deleting a CJK honorific from Constants.suffix_not_acronyms, the only v1 knob that reaches the field, turns the peel off rather than raising the subset error or leaving a tail that peels into a suffix field no longer recognizing it. AGENTS.md's roster of shim transformations said four and named four; this branch added the fifth. Same-commit rule applied one commit late. --- AGENTS.md | 2 +- tests/v2/test_config_shim.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 363ca42..69121f1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -131,7 +131,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Parser owns config-dependent conveniences**: `Parser.matches`/`Parser.capitalized`/`Parser.revise` exist because the `ParsedName` equivalents fall back to DEFAULT config for str/omitted arguments (documented loudly in both docstrings). `revise` harvests tokens from a full sub-parse of each replacement value (tags kept minus `FOLDED_TAG`, roles forced, ambiguities discarded); the merge tail is shared with `replace()` via `ParsedName._with_field_tokens`. `Parser.capitalized` delegates through `name.capitalized(self.lexicon)` specifically so `_parser` never imports `_render` — keep it that way. - **Per-word vocabulary fields warn on multi-word entries** (`_normset`/`_normpairs` via `_warn_dead_entry`, UserWarning, never a raise — see the given_name_titles Gotcha for why raising is wrong). `given_name_titles` is the one multi-word-matched field and is exempt; `_edit` passes `warn=False` (add() warns once via the new instance's `__post_init__`; remove() stores nothing). The default vocabulary and every locale pack must stay warning-free (`test_default_lexicon_builds_warning_free`, `test_pack_vocabulary_entries_are_single_words`). - **Invariants guard harm, not no-ops**: add a constructor check when violating it produces a *wrong parse*, not when it produces *nothing*. A false positive costs a working configuration; a true positive on an inert condition costs the user nothing, so that trade is never worth taking. `suffix_acronyms_ambiguous ∩ suffix_words` is guarded because the overlap loses a family name; `given_name_titles` is not, because an unreachable entry is simply never consulted (see Gotchas). Before adding one, construct the config it forbids and check what actually breaks. -- **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Four exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), and `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`). When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. +- **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Five exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`), and `honorific_tails = GLUED_HONORIFICS ∩ suffix_words` (#308 behavior with no v1 manager of its own, so the one v1 knob that reaches it is deleting the suffix word — which turns the peel off, `test_snapshot_removing_a_honorific_word_turns_the_peel_off`). When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). `PolicyPatch`'s repr shows only set (non-UNSET) fields; `_order_repr` must never raise even on an unvalidated patch's garbage `name_order` (PolicyPatch defers validation to apply time); the sweep test in `tests/v2/test_reprs.py` pins that no config repr leaks the UNSET sentinel. - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. **Document the positive direction of a partial property**: "a non-empty `ambiguities` is a signal to act on" is checkable, while "an empty one means no fork occurred" is a universal negative needing exhaustive verification -- that claim was written twice and falsified twice, at sites the author had not audited. - **The segmenter contract**: the optional `Parser(segmenter=...)` hook is parse-totality's ONE exception (locales spec section 4). Everything inside that exception is a bug in USER CODE, never a fact about the name, so it is surfaced rather than absorbed: the segmenter's own exceptions propagate, and the two protocol violations the stage can detect for itself — an answer of the wrong type, and one cutting at or past the end of the token it was handed — raise `TypeError`/`ValueError` from `_script_segment` for the same reason. The line to hold when adding a check there: a protocol violation by the segmenter's AUTHOR raises, while an adapter's defense against its own third-party library (`locales/ja.py`'s repertoire, length, reconstruction and score guards) declines with `None`, because what those catch is a fact about the content. diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index 194fc6e..be7d466 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -476,6 +476,26 @@ def test_snapshot_keeps_a_bound_never_given_prefix_parseable() -> None: assert (name.first, name.last) == ("dos Santos", "Silva") +def test_snapshot_removing_a_honorific_word_turns_the_peel_off() -> None: + # The deciding case for honorific_tails, which has no v1 manager of + # its own: the snapshot intersects GLUED_HONORIFICS with the WORD + # set, so deleting 씨 from suffix_not_acronyms -- the only v1 knob + # that reaches it -- must make the tail stop mattering rather than + # raise the subset error or leave 씨 peeling into a suffix field + # that no longer recognizes it. With the default config the + # intersection is a no-op, so the equality test above pins nothing + # here. + c = Constants() + assert HumanName("김민준씨", constants=c).suffix == "씨" # baseline + c.suffix_not_acronyms.remove("씨") + lexicon, _, _ = c._snapshot() # must not raise + assert "씨" not in lexicon.honorific_tails + # the peel is off: the glued honorific goes back into the name, + # which is 2.0's answer for this input + name = HumanName("김민준씨", constants=c) + assert (name.first, name.last, name.suffix) == ("민준씨", "김", "") + + def test_snapshot_field_translation() -> None: c = Constants() lexicon, policy, defaults = c._snapshot() From 288fef291e38912911676946c50c36e5e096149c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 03:25:20 -0700 Subject: [PATCH 21/36] Say what the peel actually does, in prose (#308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two overclaims in the user docs and four stale notes in the tests. usage.rst said the honorific is split off "before anything else reads the name". Comma structure and interpunct detection both run first and both gate the peel: "田中さん, 太郎" keeps the whole 田中さん as the family name, pinned by test_family_comma_skips_the_peel and documented nowhere. Said now, next to the comma rule it belongs to. Both usage.rst and the release log promised that the spaced spelling "composes the same way" and warned that a spaced family-name-only address would start dividing. The previous commit that narrowed the carve-out makes both false: spaced forms are unaffected, and the release log's ja-pack paragraph now says which spelling divides and which does not, and offers the spaced spelling as the opt-out it now is. The case-table notes: the lone-씨 row pointed at the length cap when the peel-site scan is what holds it, ja_sama_spaced still forward- pointed to work that has landed, the lone-さん row named its twin by position and got the wrong row, and test_lexicon's rationale credited an ambiguous subtraction that happens only in the v1 shim. --- docs/release_log.rst | 2 +- docs/usage.rst | 27 +++++++++++++++++++-------- tests/v2/cases.py | 25 ++++++++++++++++--------- tests/v2/test_lexicon.py | 6 ++++-- 4 files changed, 40 insertions(+), 20 deletions(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index 72f980e..40bf81c 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -27,7 +27,7 @@ Release Log - Fix the katakana middle dot ``・`` (U+30FB, and its halfwidth twin U+FF65) being read as part of a name rather than as the divider it is: it now separates tokens exactly as a space does. A transcribed foreign name therefore divides into its parts and, being wholly katakana, keeps its source order — ``"マイケル・ジャクソン"`` gives given ``マイケル``, family ``ジャクソン``, where 1.x left the whole string in ``first`` — while a kanji pair written the same way takes the family-first rule (``"高橋・一郎"`` → family ``高橋``). Native Japanese names never contain this character and no other script's names use it, so the separation is unconditional, which also means it is **not** covered by the two policy opt-outs. Rendering follows, the way the Korean split's does: the dot comes back as a space, so ``str(HumanName("マイケル・ジャクソン"))`` is now ``"マイケル ジャクソン"``, and that reaches delimited content too — the nickname in ``"山田 太郎 (マイケル・ジャクソン)"`` renders ``"マイケル ジャクソン"``. The Chinese interpunct ``·`` is not an unconditional separator like these two: U+00B7 is also the Catalan punt volat and appears inside legitimate names (``Gal·la``), so it divides only between classified-script characters — the transcription treatment it marks is #298's (#272) - Fix 间隔号-divided transcriptions parsing as one unsplit token: U+00B7 — the interpunct Chinese text divides a transcribed foreign name with, ``威廉·莎士比亚`` for William Shakespeare — is now a token separator between characters of a classified script, and a name it divides keeps its source order and is never segmented: the dot is the transcription marker, playing the role pure katakana plays in the kana license. It divides ONLY between classified-script characters, so the Catalan punt volat interior to names like ``Gal·la`` is untouched. The Japanese nakaguro is deliberately not a transcription marker — ``高橋・一郎`` is roster formatting, 姓・名, and keeps its family-first reading — so a transcription typed with the wrong dot (``威廉・莎士比亚``) reads by the convention of the codepoint it was typed with; like the spaced form, only the Chinese dot rescues it. Rendering stays space-joined (``str(HumanName("威廉·莎士比亚"))`` is ``"威廉 莎士比亚"``); a custom ``string_format`` such as ``"{first}·{last}"`` reinstates the dot (#298) - Fix spaced CJK postnominal honorifics parsing as name parts: 씨, 박사, 선생님, 교수님, 군, 양 (Korean — standardly written as their own token), 先生, 女士, 小姐, 博士, 教授 (Chinese, with 先生/博士/教授 shared with Japanese), and 様, 氏 (Japanese) now route to ``suffix``, so ``王小明 先生`` reads family ``王小明`` where the family-first default had confidently made 先生 the given name. Whole-token matching, which also reaches a glued surname+honorific token, since segmentation splits off the surname first (``김씨`` reads family 김, suffix 씨). The glued forms whole-token matching cannot reach are handled by the peel described below (closes #307) - - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 박사님, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Seven entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님, 박사님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``, ``김민준 박사님``). 박사님 closes a gap in the shipped set rather than opening new ground: 선생님 and 교수님 shipped in 2.1 without it, so ``김민준박사님`` stranded 박사 in the given name and the spaced ``김민준 박사님`` came back as two suffixes for one honorific. Exactly one honorific peels off a token, and every entry is a whole honorific rather than a part of one. A token that is nothing but an honorific is no longer taken apart either — ``선생님`` and ``박사`` now stay whole where hangul segmentation had split them 선 + 생님 and 박 + 사, since 선 and 박 are listed surnames — which is **default-on** in its own right, hangul segmentation being a default. A configured segmenter benefits twice over: it is handed the name without the honorific, and a recognized honorific beside an undivided name no longer counts as a boundary its writer drew, so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads both ``山田太郎様`` and the spaced ``山田太郎 様`` as family 山田, given 太郎, suffix 様. That last part cuts both ways and is worth knowing before you upgrade: an honorific no longer exempts a name from the pack's division at all, so a family name written alone with one — ``田中 さん``, ``佐藤 氏`` — now divides the way bare ``田中`` already did (family 田, given 中). Declining the pack, or the segmenter, is how a caller opts out of that presumption. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) + - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 박사님, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Seven entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님, 박사님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``, ``김민준 박사님``). 박사님 closes a gap in the shipped set rather than opening new ground: 선생님 and 교수님 shipped in 2.1 without it, so ``김민준박사님`` stranded 박사 in the given name and the spaced ``김민준 박사님`` came back as two suffixes for one honorific. Exactly one honorific peels off a token, and every entry is a whole honorific rather than a part of one. A token that is nothing but an honorific is no longer taken apart either — ``선생님`` and ``박사`` now stay whole where hangul segmentation had split them 선 + 생님 and 박 + 사, since 선 and 박 are listed surnames — which is **default-on** in its own right, hangul segmentation being a default. A configured segmenter benefits twice over: it is handed the name without the honorific, and the honorific the peel just cut off does not then look to it like a boundary the writer drew — so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads ``山田太郎様`` as family 山田, given 太郎, suffix 様. Worth knowing before you upgrade: that exemption is what makes a GLUED honorific stop protecting a name from division, so a family name written alone with one — ``田中さん`` — now divides the way bare ``田中`` already did (family 田, given 中, suffix さん). It is exactly and only the peeled tail that is exempt. A SPACED honorific is a boundary its writer did draw, and the name beside it is left as written: ``田中 さん`` and ``佐藤 氏`` keep family 田中 and 佐藤 under the pack — their division is untouched by this release, whichever field the honorific itself lands in. Writing the honorific spaced is therefore an opt-out in its own right, alongside declining the pack or the segmenter. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) - Fix NFD-decomposed input missing the East Asian defaults entirely: script classification now normalizes to NFC before deciding, so a Korean or Japanese name typed on macOS — where decomposed text is routine — gets the same order rule as its composed twin, which it silently did not before. Segmentation MATCHING deliberately stays raw, so an unspaced NFD hangul name is ordered correctly but not split, rather than being split in the wrong place. One gotcha worth stating plainly: parse output preserves the encoding it was given, so for NFD input ``name.family == "김"`` is ``False`` even though it is the same name — compare NFC-normalized text when comparing across encodings (#272) - Fix the Ukrainian conjunction ``й`` not joining the pieces around it: it is the euphonic alternate of ``і``, the two chosen by the surrounding vowel and consonant rather than by meaning (``"Олесь і Олена"`` but ``"Марія й Петро"``), so real Ukrainian data carries both spellings and shipping only ``і`` recognized just one of them. ``"Олесь й Олена Коваленки"`` now gives given ``"Олесь й Олена"`` where the ``й`` previously landed in ``middle``. Same treatment as the ``и``/``і`` entries added in 2.0.0, single-letter carve-out included: the conjunction joins only once the name has enough pieces, and a punctuated initial still wins, so ``"Й. Сліпий"`` is unaffected. Raised in a comment on #267 diff --git a/docs/usage.rst b/docs/usage.rst index 3347290..028929e 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -298,14 +298,25 @@ wrote the comma has already said where the family name ends. Honorifics and degrees follow a CJK name, and both the spaced and the glued forms are recognized as suffixes: ``王小明 先生`` reads family 王小明 with 先生 in ``suffix``, and so do glued ``田中さん``, -``山田太郎様`` and ``김민준씨`` — the honorific is split off the end of -the last name token before anything else reads the name, so the split -and the order rules see the name without it. Everything downstream -then applies as usual, which is why ``김민준씨`` still divides into -family 김 and given 민준, and why a configured Japanese segmenter is -handed 山田太郎 rather than 山田太郎様. The spaced spelling composes -the same way — an honorific is not a boundary its writer drew between -family and given, so ``山田太郎 様`` divides too. +``山田太郎様`` and ``김민준씨``. The honorific is split off the end of +the last name token before the name is split or ordered, so those +rules see the name without it — which is why ``김민준씨`` still divides +into family 김 and given 민준, and why a configured Japanese segmenter +is handed 山田太郎 rather than 山田太郎様. The comma rule above comes +first, though, and covers this too: in ``田中さん, 太郎`` the writer +has already said where the family name ends, so nothing is peeled and +the family name is the whole ``田中さん``. A 间隔号-divided +transcription opts out for its own reason, the same way. + +Splitting a name is where the two spellings part company. A spaced +honorific stands beside a name its writer chose to write as one unit, +and that unit is left alone: under the Japanese pack ``佐藤 氏`` keeps +family 佐藤, where bare ``佐藤`` would have been divided 佐 + 藤. A +glued honorific says nothing of the kind — its writer drew no boundary +anywhere — so ``田中さん`` divides the way bare ``田中`` does, into +family 田 and given 中, with さん in ``suffix``. Writing the honorific +spaced is therefore also how you ask for a family name to be kept +whole, short of declining the pack. The glued reading is deliberately narrower than the spaced one. A spaced honorific sits behind a boundary its writer drew; a glued one diff --git a/tests/v2/cases.py b/tests/v2/cases.py index c6b4739..c2d0218 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -730,9 +730,10 @@ def __post_init__(self) -> None: Case("ja_sama_spaced", "田中 太郎 様", {"family": "田中", "given": "太郎", "suffix": "様"}, classification="fix(#307)", - notes="the spaced 様 of forms and databases; the glued " - "mail-addressing form 山田太郎様 is #308's mechanism, " - "out of reach of whole-token matching"), + notes="the spaced 様 of forms and databases, which whole-token " + "matching reaches on its own; the glued " + "mail-addressing form is ja_sama_glued below, reached " + "by #308's peel instead"), Case("ja_san_spaced", "田中 さん", {"family": "田中", "suffix": "さん"}, classification="fix(#308)", @@ -810,10 +811,12 @@ def __post_init__(self) -> None: Case("ko_glued_tail_alone_never_peels", "씨", {"family": "씨"}, classification="fix(#271)", - notes="the empty-remainder guard: a token that IS a tail has " - "nothing to peel off. Classified to #271 because that " - "is what moves the lone token from first to family; " - "the row exists for the guard"), + notes="a token that IS a tail is not a peel site at all -- " + "every tail is a suffix word, so the site scan steps " + "past it and finds nothing else. NOT the length cap, " + "which is never reached here. Classified to #271 " + "because that is what moves the lone token from first " + "to family; the row exists for the guard"), Case("ko_honorific_token_alone_stays_whole", "선생님", {"family": "선생님"}, classification="fix(#308)", @@ -825,8 +828,12 @@ def __post_init__(self) -> None: Case("ja_glued_tail_alone_never_peels", "さん", {"family": "さん"}, classification="fix(#272)", - notes="the kana twin of the row above; the field placement " - "is the kana-licensed order default, not the peel"), + notes="the kana twin of ko_glued_tail_alone_never_peels, and " + "of that row only: a lone さん carries none of " + "ko_honorific_token_alone_stays_whole's risk, since no " + "surname vocabulary is written in kana. The field " + "placement is the kana-licensed order default, not the " + "peel"), Case("ja_glued_degree_stays", "田中博士", {"family": "田中博士"}, classification="fix(#271)", diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index f07144b..b7840aa 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -602,7 +602,9 @@ def test_default_lexicon_honorific_tails_are_all_suffix_words() -> None: # Not redundant with suffixes.py's GLUED_HONORIFICS <= SUFFIX_NOT_ # ACRONYMS assert: that one relates the raw constants at import # time (and is gone under python -O); this one relates the - # normalized, post-ambiguous-subtraction Lexicon fields, and - # survives -O. Structural check, not a content pin. + # normalized fields of a constructed Lexicon, and survives -O. + # (No ambiguous subtraction here -- that lives in the v1 shim's + # _build_snapshot, which this never touches.) Structural check, + # not a content pin. d = Lexicon.default() assert d.honorific_tails <= d.suffix_words From acbfefc3c318fe419164cc08a1390fdcf7f8cd2c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 03:30:34 -0700 Subject: [PATCH 22/36] Pin the peel's segment choice and its two boundaries (#308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A mutation pass found one live survivor on a fully-covered line: replacing the peel's `state.segments[0]` with `range(len(state.tokens))` -- the natural simplification, since under NO_COMMA the two runs are usually identical -- changes real output and fails nothing. Extracted delimited content is still in the token stream at this stage with only its role set, so "김민준씨 (Jimmy)" would hand the scan-back Jimmy as its site: no post-nominal to step over, no tail to peel, and the peel lost entirely. The comment above the segment lookup made it worse by saying extracted content is unreachable and that no input can produce it -- true of the surname half it was written for, actively misleading for the peel, where the choice is what keeps Jimmy out. Case row plus both comments. Two boundaries a reader could otherwise meet by surprise: - "田中さん, PhD" keeps the whole 田中さん as the family name while "田中さん PhD" gives family 田中 and suffix "さん, PhD". A one-word name part before a comma is FAMILY_COMMA, which opts the stage out by doctrine -- the one spelling disagreement #308 does not remove, where the 김민준씨 Jr. pair got two rows for removing exactly that. A case row, so it is a recorded decision. - The off-switch docs/customize.rst promises was verified and untested: removing a tail leaves the glued name unsplit while the spaced spelling still routes the honorific. Pinned at parse level, against a peel-is-on baseline so it cannot pass vacuously. And a clause recording why _is_post_nominal is strict rather than lenient: the initial veto makes "田中さん V." decline where "田中さん II" peels, which agrees with what classify does with "V." downstream. Swapping the predicate fails no test; the clause is the record instead. --- nameparser/_pipeline/_script_segment.py | 27 +++++++++++++++++++++++-- tests/v2/cases.py | 26 ++++++++++++++++++++++++ tests/v2/test_lexicon.py | 20 ++++++++++++++++++ tools/differential/corpus_cjk.jsonl | 2 ++ 4 files changed, 73 insertions(+), 2 deletions(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index e035bc5..8b0baa4 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -254,6 +254,15 @@ def _is_post_nominal(state: ParseState, i: int) -> bool: surname site reads a True as an ANSWER and declines, rather than as a token to step past. + STRICT, not lenient: the initial veto applies, so "V." is not a + post-nominal here though bare "v" is a suffix word. That agrees + with what classify does with the same token downstream -- "V." is + a middle initial -- and the difference is reachable under the + default lexicon: "田中さん V." stops its scan-back at the initial + and does not peel, while "田中さん II" steps over II and does. + Swapping in is_suffix_lenient fails no test; this clause is the + record instead. + Suffixes only, deliberately. Should a CJK entry ever join titles, the surname site would want it excluded while the peel site must not: a title never trails, so widening the peel's scan-back would @@ -285,6 +294,13 @@ def _peel_honorific_tail(state: ParseState) -> ParseState: that returns None where the outright index would raise removes an unpinned reliance on the FAMILY_COMMA gate landing first. + WHICH run is scanned is a separate decision, and the one a reader + is likeliest to undo: segments[0], never state.tokens. The two + agree under NO_COMMA except where extract_delimited has already + claimed a token, which is still in the stream here with only its + role set -- so "김민준씨 (Jimmy)" is the input that tells them + apart. See the note above the segment lookup in script_segment. + That last token is the last of the NAME PART, which under NO_COMMA reaches into a maiden clause: maiden tokens are still main-stream here (extract_delimited has masked only bracketed content), so @@ -386,8 +402,15 @@ def script_segment(state: ParseState) -> ParseState: # segments[0] is the NAME part under both remaining structures # (everything, under NO_COMMA); later segments are suffixes. Its # members are main-stream token indices by construction, so - # extracted nickname/maiden content is unreachable from here -- - # no input can produce it, so no test pins it. + # extracted nickname/maiden content is unreachable from here. + # For the surname site that is merely tidy -- the first + # script-written token is the same either way. For the PEEL above + # it is load-bearing, and iterating state.tokens instead is a + # silent regression rather than a refactor: extracted content is + # still IN the token stream at this stage (only its role is set), + # so "김민준씨 (Jimmy)" would give the scan-back Jimmy as its site, + # which is no post-nominal to step over and ends in no tail, and + # the peel would be lost. Pinned by the case row of that name. i = next((i for i in state.segments[0] if effective_script(state.tokens[i].text) in scripts), None) # A post-nominal in the surname's own position is not a site to diff --git a/tests/v2/cases.py b/tests/v2/cases.py index c2d0218..25d0a8a 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -914,6 +914,32 @@ def __post_init__(self) -> None: "ko_honorific_glued_given_trailing_suffix, whose " "comma-less spelling of the same name reaches the same " "answer by the scan-back instead"), + Case("ko_honorific_glued_given_nickname", "김민준씨 (Jimmy)", + {"family": "김", "given": "민준", "suffix": "씨", + "nickname": "Jimmy"}, + classification="fix(#308)", + notes="the other half of indexing the NAME part: extracted " + "content is still in the token stream at this stage but " + "NOT in segments[0], so the scan-back never reaches " + "Jimmy. Scanning the tokens instead would take Jimmy as " + "the site -- it is no post-nominal -- and lose the peel " + "entirely, with 씨 back in the given name. Nothing else " + "pins that choice: under NO_COMMA the two are otherwise " + "the same run"), + Case("ko_honorific_glued_family_comma", "田中さん, PhD", + {"family": "田中さん", "title": "PhD"}, + classification="fix(#271)", + notes="the comma doctrine outranks the peel, and this is " + "where a reader meets that boundary: a one-word name " + "part before a comma is FAMILY_COMMA, which opts the " + "whole stage out, so 田中さん stays whole where the " + "comma-less 田中さん PhD gives family 田中 and suffix " + "'さん, PhD'. Deliberate -- whoever wrote the comma " + "already said where the family name ends -- and the " + "one spelling disagreement #308 does not remove, unlike " + "the 김민준씨 Jr. pair above. Classified to #271, not " + "#308: the fields here are the CJK order flip's, and " + "the peel never fires on this input"), Case("zh_honorific_glued_surname", "王先生", {"family": "王", "suffix": "先生"}, locale="zh", diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index b7840aa..fd43aa9 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -608,3 +608,23 @@ def test_default_lexicon_honorific_tails_are_all_suffix_words() -> None: # not a content pin. d = Lexicon.default() assert d.honorific_tails <= d.suffix_words + + +def test_removing_a_honorific_tail_is_the_peels_off_switch() -> None: + # The off-switch docs/customize.rst documents, at parse level: + # dropping a tail leaves the glued name unsplit while the SPACED + # spelling still routes the honorific, since suffix_words is a + # separate field and only the peel consults honorific_tails. + # honorific_tails is the one marker field a caller can empty + # without a matching base edit -- an orphan is what raises, and a + # removal cannot make one. + lex = Lexicon.default().remove(honorific_tails={"さん"}) + p = Parser(lexicon=lex) + glued = p.parse("田中さん") + assert (glued.family, glued.suffix) == ("田中さん", "") + spaced = p.parse("田中 さん") + assert (spaced.family, spaced.suffix) == ("田中", "さん") + # the baseline the removal is against, so the test cannot pass by + # the peel being broken outright + on = Parser().parse("田中さん") + assert (on.family, on.suffix) == ("田中", "さん") diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index e62254c..dd5f6d8 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -37,6 +37,7 @@ "田中 殿" "田中『ハナ』花子" "田中さん" +"田中さん, PhD" "田中博士" "諸葛亮" "阿明" @@ -56,6 +57,7 @@ "김민준님" "김민준박사님" "김민준씨" +"김민준씨 (Jimmy)" "김민준씨 Jr." "김선생님" "김씨" From 3e5dad6db8b8079ddd88782d4b8b69c6ae4f6d94 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 03:34:26 -0700 Subject: [PATCH 23/36] Reach the peel from a drawn lexicon (#308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The property layer draws honorific_tails and repairs the draw, but _names_using only manufactured `surname + 민준`, so no generated token ever ENDED in a drawn tail and the peel never fired under a drawn lexicon. Instrumented before and after: 14 fires across test_properties.py, all 14 under Lexicon.default() and 0 under a drawn one; deriving the glued token from the draw moves that to 2. Both are ASCII tails on a non-Latin stem ('민준de', '민준and'), which is the one shape that reaches an ASCII tail at all -- the stage bails on a wholly ASCII original. Same failure mode the pool's own comment records having measured and fixed for surnames. The comment now covers both derivations, and no longer states a per-run fire count for the surname half as fact: the same instrumentation shows zero surname-half splits under drawn lexicons in this run, so the honest note is to instrument before concluding either half is exercised. --- tests/v2/test_properties.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index 6622abf..118a26e 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -311,24 +311,38 @@ def _names_using(draw: st.DrawFn, lexicon: Lexicon) -> str: """ vocab = sorted({w for name in _SET_FIELDS for w in getattr(lexicon, name)}) - # script_segment (#271) is the one stage a space-joined name can - # never reach: it splits an UNSPACED token whose PREFIX is a drawn - # surname, so every drawn surname is also offered concatenated with - # a given name. Waiting instead for a drawn surname and a matching - # literal to coincide left the stage unexercised on all 250 - # examples (measured); deriving the token from the draw itself - # reaches it on a handful. A Latin surname makes a mixed-script - # token the stage correctly declines -- useful input in its own - # right. + # script_segment (#271, #308) holds the two halves a space-joined + # name can never reach, and BOTH need their token derived from the + # draw: waiting for a drawn entry and a matching literal to + # coincide leaves the stage unexercised. The surname half splits an + # unspaced token whose PREFIX is a drawn surname; the peel splits a + # listed tail off the END of one. The peel's line was added after a + # mutation pass asked the same question of it and instrumentation + # answered: 14 fires across this file, every one under the DEFAULT + # lexicon and none under a drawn one, because no generated token + # ended in a drawn tail. Deriving it moves that off zero. + # A Latin entry is useful input in its own right: it makes a + # mixed-script token the surname half correctly declines, and for + # the peel it is the one shape that reaches an ASCII tail at all -- + # the stage bails on a wholly-ASCII original, so the non-Latin stem + # is what admits '민준jr'. Both fires observed under drawn lexicons + # are of exactly that shape. + # Neither derivation guarantees a fire on any given run: the piece + # is one of ~30 in the pool, so it competes with the bare vocabulary + # words, and a bare drawn surname standing earlier in the name takes + # the surname site before the unspaced token can. Instrument before + # concluding either half is exercised -- the counts above are what + # that costs to find out. # sorted for the same reason `vocab` above is: frozenset iteration # order is not stable across runs, and an unsorted pool shifts # every index sampled_from draws -- which would defeat # derandomize=True on the whole strategy, not just this slice. unspaced = sorted(w + "민준" for w in lexicon.surnames) + glued = sorted("민준" + w for w in lexicon.honorific_tails) # plain names and structure characters are always available, so the # pool is never empty even for an empty lexicon pieces = st.sampled_from( - vocab + unspaced + ["John", "Smith", "Q.", ",", "(", "'"]) + vocab + unspaced + glued + ["John", "Smith", "Q.", ",", "(", "'"]) return " ".join(draw(st.lists(pieces, min_size=1, max_size=8))) From b31f7590f6d61f55d20097fff1807195692ca70b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 11:04:01 -0700 Subject: [PATCH 24/36] Replace the peel docstring's self-refuting no-peel-twice example (#308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The clause read "ONE peel, never recursive: 김민준박사님 gives up its 님 and keeps its glued 박사". Shipping 박사님 falsified it in the same branch: longest-first now reaches the whole honorific, and that input measures {'given': '민준', 'family': '김', 'suffix': '박사님'}. Every other home of the claim was updated when 박사님 landed -- the case row renamed and re-expected, the stage test moved to the synthetic _TAILS that deliberately omits 박사님, both docs. This docstring was the survivor, and it is the one a maintainer reads while editing the peel. State the rule with an honest witness instead: no shipped pair has the shape the pin needs, so the synthetic lexicon in test_one_peel_never_a_stack is the only one left. --- nameparser/_pipeline/_script_segment.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 8b0baa4..1652691 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -319,11 +319,17 @@ def _peel_honorific_tail(state: ParseState) -> ParseState: and it could not be done here anyway: classify has not run, so the marker tokens carry no tag this stage could read. - Longest-first. ONE peel, never recursive: 김민준박사님 gives up - its 님 and keeps its glued 박사, though 박사 is itself a listed - tail, which is accepted rather than chased. No script precondition - on the remainder either, since the tail alone is the license: - Andersonさん peels. + Longest-first, and ONE peel: a remainder that itself ends in a + listed tail is accepted rather than chased. No SHIPPED input + witnesses that any more -- 박사님 was the last one, and adding it + is what removed the case: 김민준박사님 now gives up 박사님 entire, + since longest-first reaches the whole honorific. The pin needs a + tail whose remainder ends in a DIFFERENT tail, the two not + themselves a listed entry, and no pair in the shipped vocabulary + has that shape; the stage test test_one_peel_never_a_stack carries + it on a synthetic lexicon, which is now its only witness. No + script precondition on the remainder either, since the tail alone + is the license: Andersonさん peels. Emits no ambiguity, unlike the surname fork below, though longest-first does CHOOSE here too -- 김선생님 gives 선생님 where From dde0c6055b9eec03f4a0e9d407d3407d02742f5f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 11:04:31 -0700 Subject: [PATCH 25/36] Reclassify the FAMILY-comma honorific row to comma-family (#308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The row for "田中さん, PhD" was classified fix(#271) with the note "the fields here are the CJK order flip's". The observation behind it was right -- the only deviation from 1.4.0 is first -> family -- but the attribution was untested, and it is wrong. Measured, 1.4.0 against HEAD: Smith, PhD first Smith -> family Smith Smith, Dr. first Smith -> family Smith 田中さん, PhD first 田中さん -> family 田中さん "Smith, PhD" is pure Latin, so #271's script-conditional rules cannot be touching it, and it makes the identical move. The behavior is FAMILY_COMMA's and predates #271; the table already has the slug for it, used by family_comma_lone_title. Also rename ko_ -> ja_: the input is Han plus hiragana, and the table prefixes rows by their input's script. --- tests/v2/cases.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 25d0a8a..c78493f 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -926,9 +926,9 @@ def __post_init__(self) -> None: "entirely, with 씨 back in the given name. Nothing else " "pins that choice: under NO_COMMA the two are otherwise " "the same run"), - Case("ko_honorific_glued_family_comma", "田中さん, PhD", + Case("ja_honorific_glued_family_comma", "田中さん, PhD", {"family": "田中さん", "title": "PhD"}, - classification="fix(#271)", + classification="fix(comma-family)", notes="the comma doctrine outranks the peel, and this is " "where a reader meets that boundary: a one-word name " "part before a comma is FAMILY_COMMA, which opts the " @@ -937,9 +937,13 @@ def __post_init__(self) -> None: "'さん, PhD'. Deliberate -- whoever wrote the comma " "already said where the family name ends -- and the " "one spelling disagreement #308 does not remove, unlike " - "the 김민준씨 Jr. pair above. Classified to #271, not " - "#308: the fields here are the CJK order flip's, and " - "the peel never fires on this input"), + "the 김민준씨 Jr. pair above. Classified to " + "comma-family, not #308 or #271: the peel never fires " + "on this input, and the only deviation from 1.4.0 is " + "first -> family, which the pure-Latin 'Smith, PhD' " + "makes identically (1.4.0 first Smith, here family " + "Smith) -- so no script-conditional rule can be " + "reaching it. Same move as family_comma_lone_title"), Case("zh_honorific_glued_surname", "王先生", {"family": "王", "suffix": "先生"}, locale="zh", From d6cd806cef0080674b80f162ba6c9781f6695dca Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 11:06:05 -0700 Subject: [PATCH 26/36] Justify the segmenter's spaced-honorific rule by its cost (#308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The narrowing was explained in four places as "its writer drew that boundary and chose to write 山田太郎 as a unit". That is false for exactly the input it describes: in "山田太郎 様" the unit the writer drew is the whole name, honorific and all. The neighbour precondition exists to detect a boundary WITHIN the name ("山田 太郎"), and a trailing honorific is not one in either spelling. What the code actually relies on is weaker: by position a spaced honorific is indistinguishable from a spaced name element, so the precondition conservatively counts it. State that, and the measured trade that makes it worth taking -- exempting spaced honorifics would divide 佐藤 氏, 田中 様, 鈴木 先生 and 中村 教授 (all four divide bare under the JA pack) to win the single division 山田太郎 様. Fixed in the module docstring, the inline comment above the precondition, docs/usage.rst and the release log's #308 bullet. --- docs/release_log.rst | 2 +- docs/usage.rst | 16 +++++++++---- nameparser/_pipeline/_script_segment.py | 32 ++++++++++++++++++------- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index 40bf81c..5eb8b61 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -27,7 +27,7 @@ Release Log - Fix the katakana middle dot ``・`` (U+30FB, and its halfwidth twin U+FF65) being read as part of a name rather than as the divider it is: it now separates tokens exactly as a space does. A transcribed foreign name therefore divides into its parts and, being wholly katakana, keeps its source order — ``"マイケル・ジャクソン"`` gives given ``マイケル``, family ``ジャクソン``, where 1.x left the whole string in ``first`` — while a kanji pair written the same way takes the family-first rule (``"高橋・一郎"`` → family ``高橋``). Native Japanese names never contain this character and no other script's names use it, so the separation is unconditional, which also means it is **not** covered by the two policy opt-outs. Rendering follows, the way the Korean split's does: the dot comes back as a space, so ``str(HumanName("マイケル・ジャクソン"))`` is now ``"マイケル ジャクソン"``, and that reaches delimited content too — the nickname in ``"山田 太郎 (マイケル・ジャクソン)"`` renders ``"マイケル ジャクソン"``. The Chinese interpunct ``·`` is not an unconditional separator like these two: U+00B7 is also the Catalan punt volat and appears inside legitimate names (``Gal·la``), so it divides only between classified-script characters — the transcription treatment it marks is #298's (#272) - Fix 间隔号-divided transcriptions parsing as one unsplit token: U+00B7 — the interpunct Chinese text divides a transcribed foreign name with, ``威廉·莎士比亚`` for William Shakespeare — is now a token separator between characters of a classified script, and a name it divides keeps its source order and is never segmented: the dot is the transcription marker, playing the role pure katakana plays in the kana license. It divides ONLY between classified-script characters, so the Catalan punt volat interior to names like ``Gal·la`` is untouched. The Japanese nakaguro is deliberately not a transcription marker — ``高橋・一郎`` is roster formatting, 姓・名, and keeps its family-first reading — so a transcription typed with the wrong dot (``威廉・莎士比亚``) reads by the convention of the codepoint it was typed with; like the spaced form, only the Chinese dot rescues it. Rendering stays space-joined (``str(HumanName("威廉·莎士比亚"))`` is ``"威廉 莎士比亚"``); a custom ``string_format`` such as ``"{first}·{last}"`` reinstates the dot (#298) - Fix spaced CJK postnominal honorifics parsing as name parts: 씨, 박사, 선생님, 교수님, 군, 양 (Korean — standardly written as their own token), 先生, 女士, 小姐, 博士, 教授 (Chinese, with 先生/博士/教授 shared with Japanese), and 様, 氏 (Japanese) now route to ``suffix``, so ``王小明 先生`` reads family ``王小明`` where the family-first default had confidently made 先生 the given name. Whole-token matching, which also reaches a glued surname+honorific token, since segmentation splits off the surname first (``김씨`` reads family 김, suffix 씨). The glued forms whole-token matching cannot reach are handled by the peel described below (closes #307) - - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 박사님, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Seven entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님, 박사님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``, ``김민준 박사님``). 박사님 closes a gap in the shipped set rather than opening new ground: 선생님 and 교수님 shipped in 2.1 without it, so ``김민준박사님`` stranded 박사 in the given name and the spaced ``김민준 박사님`` came back as two suffixes for one honorific. Exactly one honorific peels off a token, and every entry is a whole honorific rather than a part of one. A token that is nothing but an honorific is no longer taken apart either — ``선생님`` and ``박사`` now stay whole where hangul segmentation had split them 선 + 생님 and 박 + 사, since 선 and 박 are listed surnames — which is **default-on** in its own right, hangul segmentation being a default. A configured segmenter benefits twice over: it is handed the name without the honorific, and the honorific the peel just cut off does not then look to it like a boundary the writer drew — so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads ``山田太郎様`` as family 山田, given 太郎, suffix 様. Worth knowing before you upgrade: that exemption is what makes a GLUED honorific stop protecting a name from division, so a family name written alone with one — ``田中さん`` — now divides the way bare ``田中`` already did (family 田, given 中, suffix さん). It is exactly and only the peeled tail that is exempt. A SPACED honorific is a boundary its writer did draw, and the name beside it is left as written: ``田中 さん`` and ``佐藤 氏`` keep family 田中 and 佐藤 under the pack — their division is untouched by this release, whichever field the honorific itself lands in. Writing the honorific spaced is therefore an opt-out in its own right, alongside declining the pack or the segmenter. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) + - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 박사님, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Seven entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님, 박사님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``, ``김민준 박사님``). 박사님 closes a gap in the shipped set rather than opening new ground: 선생님 and 교수님 shipped in 2.1 without it, so ``김민준박사님`` stranded 박사 in the given name and the spaced ``김민준 박사님`` came back as two suffixes for one honorific. Exactly one honorific peels off a token, and every entry is a whole honorific rather than a part of one. A token that is nothing but an honorific is no longer taken apart either — ``선생님`` and ``박사`` now stay whole where hangul segmentation had split them 선 + 생님 and 박 + 사, since 선 and 박 are listed surnames — which is **default-on** in its own right, hangul segmentation being a default. A configured segmenter benefits twice over: it is handed the name without the honorific, and the honorific the peel just cut off does not then look to it like a boundary the writer drew — so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads ``山田太郎様`` as family 山田, given 太郎, suffix 様. Worth knowing before you upgrade: that exemption is what makes a GLUED honorific stop protecting a name from division, so a family name written alone with one — ``田中さん`` — now divides the way bare ``田中`` already did (family 田, given 中, suffix さん). It is exactly and only the peeled tail that is exempt. A SPACED honorific is a token boundary its writer typed, and anything standing beside a name calls the segmenter off, so the name is left as written: ``田中 さん`` and ``佐藤 氏`` keep family 田中 and 佐藤 under the pack — the division the pack gives them without this change, whichever field the honorific itself lands in. That is the conservative reading rather than a claim about intent: a spaced honorific cannot be told apart from a spaced given name by position, and counting it as one keeps four real surnames whole (``佐藤 氏``, ``田中 様``, ``鈴木 先生``, ``中村 教授``) at the price of the one division it then declines to make (``山田太郎 様``). Writing the honorific spaced is therefore an opt-out in its own right, alongside declining the pack or the segmenter. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) - Fix NFD-decomposed input missing the East Asian defaults entirely: script classification now normalizes to NFC before deciding, so a Korean or Japanese name typed on macOS — where decomposed text is routine — gets the same order rule as its composed twin, which it silently did not before. Segmentation MATCHING deliberately stays raw, so an unspaced NFD hangul name is ordered correctly but not split, rather than being split in the wrong place. One gotcha worth stating plainly: parse output preserves the encoding it was given, so for NFD input ``name.family == "김"`` is ``False`` even though it is the same name — compare NFC-normalized text when comparing across encodings (#272) - Fix the Ukrainian conjunction ``й`` not joining the pieces around it: it is the euphonic alternate of ``і``, the two chosen by the surrounding vowel and consonant rather than by meaning (``"Олесь і Олена"`` but ``"Марія й Петро"``), so real Ukrainian data carries both spellings and shipping only ``і`` recognized just one of them. ``"Олесь й Олена Коваленки"`` now gives given ``"Олесь й Олена"`` where the ``й`` previously landed in ``middle``. Same treatment as the ``и``/``і`` entries added in 2.0.0, single-letter carve-out included: the conjunction joins only once the name has enough pieces, and a punctuated initial still wins, so ``"Й. Сліпий"`` is unaffected. Raised in a comment on #267 diff --git a/docs/usage.rst b/docs/usage.rst index 028929e..2e32519 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -308,11 +308,17 @@ has already said where the family name ends, so nothing is peeled and the family name is the whole ``田中さん``. A 间隔号-divided transcription opts out for its own reason, the same way. -Splitting a name is where the two spellings part company. A spaced -honorific stands beside a name its writer chose to write as one unit, -and that unit is left alone: under the Japanese pack ``佐藤 氏`` keeps -family 佐藤, where bare ``佐藤`` would have been divided 佐 + 藤. A -glued honorific says nothing of the kind — its writer drew no boundary +Where a segmenter divides the name, the two spellings part company. A +spaced honorific is a token boundary the writer typed, and the +segmenter is asked only where an undivided name divides — so anything +standing beside the name calls it off, honorific or not: under the +Japanese pack ``佐藤 氏`` keeps family 佐藤, where bare ``佐藤`` would +have been divided 佐 + 藤. That is a conservative reading rather than +a principled one; a spaced honorific cannot be told apart from a +spaced given name by position, and treating it as one keeps four real +surnames whole (``佐藤 氏``, ``田中 様``, ``鈴木 先生``, ``中村 教授``) +at the price of one division it declines to make (``山田太郎 様``). A +glued honorific carries no boundary at all — its writer drew none anywhere — so ``田中さん`` divides the way bare ``田中`` does, into family 田 and given 中, with さん in ``suffix``. Writing the honorific spaced is therefore also how you ask for a family name to be kept diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 1652691..f9f8b40 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -54,9 +54,15 @@ non-None -- and exempts exactly one token: the tail the peel above MANUFACTURED (#308), which is a boundary nobody drew. A SPACED honorific is not exempt, and the distinction is provenance rather -than vocabulary: glued 山田太郎様 was written undivided, while whoever -wrote "佐藤 氏" drew that boundary and wrote 佐藤 as a unit beside it, -so 佐藤 stays whole. A segmenter's own exceptions +than vocabulary: glued 山田太郎様 was written undivided, while "佐藤 +氏" carries a boundary its writer typed. Not that the writer thereby +declared 佐藤 a unit -- in "山田太郎 様" the unit they drew is the +whole name -- but that at this position a spaced honorific cannot be +told apart from a spaced name ELEMENT, so the precondition counts it +rather than guess. The trade that buys is measured: counting spaced +honorifics keeps four real surnames whole (佐藤 氏, 田中 様, 鈴木 +先生, 中村 教授, all four divided bare under the JA pack) and costs +the one division 山田太郎 様. A segmenter's own exceptions PROPAGATE -- the single declared exception to parse totality (locales spec section 4): a user-supplied callable's error is a user-code error, not a content error. @@ -491,12 +497,20 @@ def script_segment(state: ParseState) -> ParseState: # MANUFACTURED (#308): a glued 山田太郎様 has no writer-drawn # boundary anywhere, so the 様 the peel just cut off cannot be read # as one -- the precondition must see what the writer wrote, which - # was a single undivided token. A SPACED 様 is the opposite case - # and does count: its writer drew that boundary and chose to write - # 山田太郎 as a unit, so "佐藤 氏" leaves 佐藤 whole rather than - # dividing it into 佐 + 藤. Provenance, not vocabulary: the two - # spellings put the same word in the same place, and asking the - # suffix set instead cannot separate them. + # was a single undivided token. A SPACED 様 does count, and the + # reason is weaker than "its writer drew that boundary and chose + # to write 山田太郎 as a unit": in "山田太郎 様" the unit the writer + # drew is the WHOLE NAME, honorific and all, so that story is false + # for the very input it describes. What the code relies on is that + # by POSITION a spaced honorific is indistinguishable from a spaced + # name element, so the test conservatively counts it. Measured, the + # trade is worth it: counting them keeps 佐藤 氏, 田中 様, 鈴木 先生 + # and 中村 教授 whole -- all four divide bare under the JA pack (佐 + # + 藤, 田 + 中, 鈴 + 木, 中 + 村) -- and costs the single division + # 山田太郎 様. Four real surnames against one. + # Provenance, not vocabulary: the two spellings put the same word + # in the same place, and asking the suffix set instead cannot + # separate them. if any(j != i and effective_script(state.tokens[j].text) is not None and _PEELED_TAG not in state.tokens[j].tags for j in state.segments[0]): From 7819a3d7c98732e72faab979dda1d219bfbb8a31 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 11:07:00 -0700 Subject: [PATCH 27/36] Scope the spaced-honorific advice to the segmenter path (#308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit usage.rst promised that "writing the honorific spaced is therefore also how you ask for a family name to be kept whole, short of declining the pack", and the release log carried the same closing sentence. Both generalize a fact about the SEGMENTER path to a paragraph a Korean- or Chinese-data reader also reads. Measured, the vocabulary path does not part company at all: 김민준 씨 -> family 김, given 민준, suffix 씨 김민준씨 identical 王小明 先生 -> family 王, given 小明, suffix 先生 王小明先生 identical (zh) Spacing buys nothing there, and "short of declining the pack" is not even available for hangul, whose segmentation is a default. Narrow both, and pin the boundary with a test, since the claim is falsifiable prose. --- docs/release_log.rst | 2 +- docs/usage.rst | 10 +++++++++- tests/v2/test_locales.py | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index 5eb8b61..c6991e9 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -27,7 +27,7 @@ Release Log - Fix the katakana middle dot ``・`` (U+30FB, and its halfwidth twin U+FF65) being read as part of a name rather than as the divider it is: it now separates tokens exactly as a space does. A transcribed foreign name therefore divides into its parts and, being wholly katakana, keeps its source order — ``"マイケル・ジャクソン"`` gives given ``マイケル``, family ``ジャクソン``, where 1.x left the whole string in ``first`` — while a kanji pair written the same way takes the family-first rule (``"高橋・一郎"`` → family ``高橋``). Native Japanese names never contain this character and no other script's names use it, so the separation is unconditional, which also means it is **not** covered by the two policy opt-outs. Rendering follows, the way the Korean split's does: the dot comes back as a space, so ``str(HumanName("マイケル・ジャクソン"))`` is now ``"マイケル ジャクソン"``, and that reaches delimited content too — the nickname in ``"山田 太郎 (マイケル・ジャクソン)"`` renders ``"マイケル ジャクソン"``. The Chinese interpunct ``·`` is not an unconditional separator like these two: U+00B7 is also the Catalan punt volat and appears inside legitimate names (``Gal·la``), so it divides only between classified-script characters — the transcription treatment it marks is #298's (#272) - Fix 间隔号-divided transcriptions parsing as one unsplit token: U+00B7 — the interpunct Chinese text divides a transcribed foreign name with, ``威廉·莎士比亚`` for William Shakespeare — is now a token separator between characters of a classified script, and a name it divides keeps its source order and is never segmented: the dot is the transcription marker, playing the role pure katakana plays in the kana license. It divides ONLY between classified-script characters, so the Catalan punt volat interior to names like ``Gal·la`` is untouched. The Japanese nakaguro is deliberately not a transcription marker — ``高橋・一郎`` is roster formatting, 姓・名, and keeps its family-first reading — so a transcription typed with the wrong dot (``威廉・莎士比亚``) reads by the convention of the codepoint it was typed with; like the spaced form, only the Chinese dot rescues it. Rendering stays space-joined (``str(HumanName("威廉·莎士比亚"))`` is ``"威廉 莎士比亚"``); a custom ``string_format`` such as ``"{first}·{last}"`` reinstates the dot (#298) - Fix spaced CJK postnominal honorifics parsing as name parts: 씨, 박사, 선생님, 교수님, 군, 양 (Korean — standardly written as their own token), 先生, 女士, 小姐, 博士, 教授 (Chinese, with 先生/博士/教授 shared with Japanese), and 様, 氏 (Japanese) now route to ``suffix``, so ``王小明 先生`` reads family ``王小明`` where the family-first default had confidently made 先生 the given name. Whole-token matching, which also reaches a glued surname+honorific token, since segmentation splits off the surname first (``김씨`` reads family 김, suffix 씨). The glued forms whole-token matching cannot reach are handled by the peel described below (closes #307) - - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 박사님, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Seven entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님, 박사님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``, ``김민준 박사님``). 박사님 closes a gap in the shipped set rather than opening new ground: 선생님 and 교수님 shipped in 2.1 without it, so ``김민준박사님`` stranded 박사 in the given name and the spaced ``김민준 박사님`` came back as two suffixes for one honorific. Exactly one honorific peels off a token, and every entry is a whole honorific rather than a part of one. A token that is nothing but an honorific is no longer taken apart either — ``선생님`` and ``박사`` now stay whole where hangul segmentation had split them 선 + 생님 and 박 + 사, since 선 and 박 are listed surnames — which is **default-on** in its own right, hangul segmentation being a default. A configured segmenter benefits twice over: it is handed the name without the honorific, and the honorific the peel just cut off does not then look to it like a boundary the writer drew — so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads ``山田太郎様`` as family 山田, given 太郎, suffix 様. Worth knowing before you upgrade: that exemption is what makes a GLUED honorific stop protecting a name from division, so a family name written alone with one — ``田中さん`` — now divides the way bare ``田中`` already did (family 田, given 中, suffix さん). It is exactly and only the peeled tail that is exempt. A SPACED honorific is a token boundary its writer typed, and anything standing beside a name calls the segmenter off, so the name is left as written: ``田中 さん`` and ``佐藤 氏`` keep family 田中 and 佐藤 under the pack — the division the pack gives them without this change, whichever field the honorific itself lands in. That is the conservative reading rather than a claim about intent: a spaced honorific cannot be told apart from a spaced given name by position, and counting it as one keeps four real surnames whole (``佐藤 氏``, ``田中 様``, ``鈴木 先生``, ``中村 教授``) at the price of the one division it then declines to make (``山田太郎 様``). Writing the honorific spaced is therefore an opt-out in its own right, alongside declining the pack or the segmenter. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) + - Fix glued CJK honorifics parsing as part of the name: an honorific written against the name — ``田中さん``, ``山田太郎様``, ``김민준씨``, ``김민준님``, ``王小明先生`` — is now split off the end of the last name token and routed to ``suffix``, where it had been swallowed by the name (the whole of ``田中さん`` was the family name; ``김민준씨`` gave given 민준씨). The peeled name then goes through the ordinary machinery, so the Korean split still happens (``김민준씨`` → family 김, given 민준, suffix 씨) and the ``田中さん`` case stops misreading as a kana-licensed composite. Only entries that can never END a name peel — 씨, 님, 박사, 박사님, 선생님, 교수님, さん, さま, くん, ちゃん, 様, 先生, 教授, 女士, 小姐 — while 양, 군, 氏, 博士 and 殿 are recognized in their spaced form only, because 김지양 is a given name, 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka, and some ninety Japanese surnames end in 殿 (鵜殿, 真殿); 君 is recognized in neither form, since 王君 is a complete Chinese name, though its kana spelling くん peels. Seven entries are new vocabulary in this change (さん, さま, くん, ちゃん, 殿, 님, 박사님), so their spaced forms route to ``suffix`` too (``田中 さん``, ``田中 殿``, ``김민준 님``, ``김민준 박사님``). 박사님 closes a gap in the shipped set rather than opening new ground: 선생님 and 교수님 shipped in 2.1 without it, so ``김민준박사님`` stranded 박사 in the given name and the spaced ``김민준 박사님`` came back as two suffixes for one honorific. Exactly one honorific peels off a token, and every entry is a whole honorific rather than a part of one. A token that is nothing but an honorific is no longer taken apart either — ``선생님`` and ``박사`` now stay whole where hangul segmentation had split them 선 + 생님 and 박 + 사, since 선 and 박 are listed surnames — which is **default-on** in its own right, hangul segmentation being a default. A configured segmenter benefits twice over: it is handed the name without the honorific, and the honorific the peel just cut off does not then look to it like a boundary the writer drew — so ``parser_for(locales.JA, segmenter=ja_segmenter())`` reads ``山田太郎様`` as family 山田, given 太郎, suffix 様. Worth knowing before you upgrade: that exemption is what makes a GLUED honorific stop protecting a name from division, so a family name written alone with one — ``田中さん`` — now divides the way bare ``田中`` already did (family 田, given 中, suffix さん). It is exactly and only the peeled tail that is exempt. A SPACED honorific is a token boundary its writer typed, and anything standing beside a name calls the segmenter off, so the name is left as written: ``田中 さん`` and ``佐藤 氏`` keep family 田中 and 佐藤 under the pack — the division the pack gives them without this change, whichever field the honorific itself lands in. That is the conservative reading rather than a claim about intent: a spaced honorific cannot be told apart from a spaced given name by position, and counting it as one keeps four real surnames whole (``佐藤 氏``, ``田中 様``, ``鈴木 先生``, ``中村 教授``) at the price of the one division it then declines to make (``山田太郎 様``). Writing the honorific spaced is therefore an opt-out in its own right on the SEGMENTER path, alongside declining the pack or the segmenter. It is no lever where the VOCABULARY divides the name, the two spellings agreeing exactly there — ``김민준 씨`` and ``김민준씨`` both give family 김, given 민준, suffix 씨, as do ``王小明 先生`` and ``王小明先生`` under the Chinese pack — and Korean data has no pack to decline either, hangul segmentation being on by default. **Default-on: changes parse output for glued CJK honorific forms**, through ``HumanName`` as well as the 2.0 API (closes #308) - Fix NFD-decomposed input missing the East Asian defaults entirely: script classification now normalizes to NFC before deciding, so a Korean or Japanese name typed on macOS — where decomposed text is routine — gets the same order rule as its composed twin, which it silently did not before. Segmentation MATCHING deliberately stays raw, so an unspaced NFD hangul name is ordered correctly but not split, rather than being split in the wrong place. One gotcha worth stating plainly: parse output preserves the encoding it was given, so for NFD input ``name.family == "김"`` is ``False`` even though it is the same name — compare NFC-normalized text when comparing across encodings (#272) - Fix the Ukrainian conjunction ``й`` not joining the pieces around it: it is the euphonic alternate of ``і``, the two chosen by the surrounding vowel and consonant rather than by meaning (``"Олесь і Олена"`` but ``"Марія й Петро"``), so real Ukrainian data carries both spellings and shipping only ``і`` recognized just one of them. ``"Олесь й Олена Коваленки"`` now gives given ``"Олесь й Олена"`` where the ``й`` previously landed in ``middle``. Same treatment as the ``и``/``і`` entries added in 2.0.0, single-letter carve-out included: the conjunction joins only once the name has enough pieces, and a punctuated initial still wins, so ``"Й. Сліпий"`` is unaffected. Raised in a comment on #267 diff --git a/docs/usage.rst b/docs/usage.rst index 2e32519..601a7fe 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -322,7 +322,15 @@ glued honorific carries no boundary at all — its writer drew none anywhere — so ``田中さん`` divides the way bare ``田中`` does, into family 田 and given 中, with さん in ``suffix``. Writing the honorific spaced is therefore also how you ask for a family name to be kept -whole, short of declining the pack. +whole on this path, short of declining the pack. + +Where the VOCABULARY divides the name the two spellings never part +company, because the peel hands the same remainder to the same surname +match either way: ``김민준 씨`` and ``김민준씨`` both give family 김, +given 민준 and suffix 씨, and under the Chinese pack ``王小明 先生`` +and ``王小明先生`` both give family 王, given 小明 and suffix 先生. +Spacing the honorific is no lever there, and for Korean data there is +no pack to decline either — hangul segmentation is on by default. The glued reading is deliberately narrower than the spaced one. A spaced honorific sits behind a boundary its writer drew; a glued one diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 9538890..e46fc09 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -195,6 +195,25 @@ def test_zh_composes_with_korean_defaults() -> None: assert (n.family, n.given) == ("김", "민준") +def test_honorific_spellings_agree_where_the_vocabulary_divides() -> None: + # usage.rst says a SPACED honorific is how you keep a family name + # whole. That holds only where a SEGMENTER divides it -- the ja + # tests below pin that half -- because the segmenter's precondition + # counts any neighbour and a spaced honorific is one. The + # VOCABULARY has no such precondition: the peel (#308) hands it the + # same remainder whichever way the honorific was written, so the + # two spellings agree exactly. Pinned in both scripts because the + # reader's escape hatch differs -- the zh pack can be declined, + # hangul segmentation is a DEFAULT and cannot, so a Korean-data + # reader has no lever at all here. + ko = _DEFAULT_PARSER.parse("김민준 씨") + assert (ko.family, ko.given, ko.suffix) == ("김", "민준", "씨") + assert _DEFAULT_PARSER.parse("김민준씨").as_dict() == ko.as_dict() + zh = _PACKED["zh"].parse("王小明 先生") + assert (zh.family, zh.given, zh.suffix) == ("王", "小明", "先生") + assert _PACKED["zh"].parse("王小明先生").as_dict() == zh.as_dict() + + def test_ja_pack_contents() -> None: assert locales.JA.code == "ja" # segmentation activation ONLY: no vocabulary (no list settles a From 2189401a3446a614fee8b7e443ee1583c0bfd4d0 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 11:09:21 -0700 Subject: [PATCH 28/36] Pin _is_post_nominal's strict suffix test with the input it names (#308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Swapping is_suffix_lenient into _is_post_nominal still failed zero tests. The previous round closed that with a docstring clause -- but the clause NAMES the discriminating input under the default lexicon, and when a comment can name the input, the input belongs in a test. Measured, default lexicon: 田中さん V. strict given 田中さん, family 'V.' (no peel) 田中さん V. lenient given 田中, middle さん, family 'V.' 田中さん II strict family 田中, suffix 'さん, II' 田中さん II lenient identical The V. row is 1.4.0 parity (first 田中さん / last 'V.'), so it lands as a parity row; the II row is a #308 deviation and is classified as one. Together they are the pin: only the strict/lenient choice separates them, and a future "unify the two suffix predicates" refactor would have changed a user-visible parse silently. corpus_cjk.jsonl regenerated (it derives from the case table). --- nameparser/_pipeline/_script_segment.py | 9 ++++++--- tests/v2/cases.py | 23 +++++++++++++++++++++++ tools/differential/corpus_cjk.jsonl | 2 ++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index f9f8b40..3625c72 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -265,9 +265,12 @@ def _is_post_nominal(state: ParseState, i: int) -> bool: with what classify does with the same token downstream -- "V." is a middle initial -- and the difference is reachable under the default lexicon: "田中さん V." stops its scan-back at the initial - and does not peel, while "田中さん II" steps over II and does. - Swapping in is_suffix_lenient fails no test; this clause is the - record instead. + and does not peel, while "田中さん II" steps over II and does. That + pair is the case table's ja_honorific_glued_before_an_initial and + ja_honorific_glued_before_a_roman_suffix, added because the clause + could NAME the discriminating input while swapping in + is_suffix_lenient still failed no test -- the shape a "unify the + two suffix predicates" refactor would have walked straight past. Suffixes only, deliberately. Should a CJK entry ever join titles, the surname site would want it excluded while the peel site must diff --git a/tests/v2/cases.py b/tests/v2/cases.py index c78493f..c7e3263 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -750,6 +750,29 @@ def __post_init__(self) -> None: "hiragana, so the kana license read the whole string " "as a Japanese name. The peel runs before the license " "is consulted, so it now sees 田中 alone"), + Case("ja_honorific_glued_before_a_roman_suffix", "田中さん II", + {"family": "田中", "suffix": "さん, II"}, + classification="fix(#308)", + notes="an unrelated trailing suffix does not hide the peel " + "site: the scan-back steps over II and peels さん off " + "the token behind it. Half of the pair that pins " + "_is_post_nominal's use of is_suffix_STRICT -- the " + "other half is the row below, and swapping in " + "is_suffix_lenient changes that one and not this one"), + Case("ja_honorific_glued_before_an_initial", "田中さん V.", + {"given": "田中さん", "family": "V."}, + notes="the strict/lenient discriminator, and the reason " + "_is_post_nominal reads is_suffix_strict. Bare 'v' is " + "a suffix word, but 'V.' is an initial and the strict " + "test vetoes it -- so the scan-back stops HERE rather " + "than stepping over it, finds no tail on 'V.', and " + "does not peel. Under is_suffix_lenient it would step " + "over and give given 田中, middle さん, family 'V.'. " + "Parity besides: 1.4.0 read this first 田中さん / last " + "'V.', which is these fields under the 2.0 names. " + "Classification agrees with what classify does with " + "the same token downstream -- 'V.' is a middle " + "initial, not a post-nominal"), Case("ja_sama_glued", "山田太郎様", {"family": "山田太郎", "suffix": "様"}, classification="fix(#308)", diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index dd5f6d8..cc0fa12 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -37,6 +37,8 @@ "田中 殿" "田中『ハナ』花子" "田中さん" +"田中さん II" +"田中さん V." "田中さん, PhD" "田中博士" "諸葛亮" From b8a9252b405ee4b9f8961109dee2f15e44f809cb Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 11:10:05 -0700 Subject: [PATCH 29/36] Give the spaced-honorific test its missing control (#308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_ja_does_not_divide_a_spaced_surname_under_an_honorific asserted that 佐藤 氏 keeps family 佐藤, but nothing in it showed the pack WOULD divide 佐藤 otherwise -- it passed vacuously for any reason the segmenter went unconsulted, and the contrast is what the doc sentence it supports rests on. Assert the bare division too (佐 + 藤, 田 + 中, 鈴 + 木, 中 + 村), and add 中村 教授 so the four cases the prose now cites as the measured price of this rule are the four the test runs. The comment's "the writer drew this boundary: 佐藤 is a unit" goes with them, for the reason the previous commit gives. --- tests/v2/test_locales.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index e46fc09..0b6049f 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -485,16 +485,27 @@ def test_ja_divides_a_glued_name_carrying_an_honorific() -> None: @_needs_ja def test_ja_does_not_divide_a_spaced_surname_under_an_honorific() -> None: - # The writer drew this boundary: 佐藤 is a unit, and the - # honorific beside it says nothing about where 佐藤 divides. - # Only a tail this stage MANUFACTURED is exempt from the - # neighbour test -- a glued honorific means no boundary was - # drawn anywhere, a spaced one means one was. + # A spaced honorific is a neighbour, and the segmenter is asked + # only where an UNDIVIDED name divides, so it is never consulted + # here. Only a tail this stage MANUFACTURED is exempt -- a glued + # honorific means no boundary was typed anywhere, a spaced one + # means one was, and by position a spaced honorific cannot be told + # apart from a spaced name element besides. + # The bare control is what makes any of that load-bearing: without + # it the assertions pass vacuously, for any reason at all that the + # segmenter went unconsulted. Each family name here DOES divide + # standing alone, so the honorific is what stopped it -- and these + # four are the measured price of counting spaced honorifics, paid + # to decline the one division 山田太郎 様 (the test above). p = _PACKED["ja"] - for name, family in (("佐藤 氏", "佐藤"), ("田中 様", "田中"), - ("鈴木 先生", "鈴木")): + for name, family, divided in (("佐藤 氏", "佐藤", ("佐", "藤")), + ("田中 様", "田中", ("田", "中")), + ("鈴木 先生", "鈴木", ("鈴", "木")), + ("中村 教授", "中村", ("中", "村"))): n = p.parse(name) assert (n.family, n.given) == (family, ""), name + bare = p.parse(family) + assert (bare.family, bare.given) == divided, family @_needs_ja From e6a6b698d5c9ca9989ebef22ff1fbe4efdf292ec Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 11:11:58 -0700 Subject: [PATCH 30/36] Correct five accuracy nits around the peel (#308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _script_segment's module header Reads: line omitted the suffix vocabulary the stage now reads through _is_post_nominal -> _vocab.is_suffix_strict; _segment.py sets the convention for naming it. Produces: likewise omitted the peel's _PEELED_TAG. - The same docstring's gate list ("gated by the FAMILY comma and by 间隔号 but NOT by segment_scripts") read as exhaustive and omitted the ASCII bail -- the gate that actually decides a caller-configured LATIN tail. The bail's own comment had it right; the header had not caught up. - config/suffixes.py said "Two more are in NEITHER set" and then named three strings (君, 선생, 교수). The list above it counts entries. - AGENTS.md's shim-transformation roster was one short: maiden_delimiters minus nickname_delimiters carries the same v1-reachability comment and is pinned by test_snapshot_overlap_keeps_v1_nickname_precedence. Verified: with parens in both v1 buckets, 1.4.0 and HEAD both parse nickname, while Policy alone gives maiden. It sits on the Policy half of _snapshot(), which is why a Lexicon-only sweep missed it. - test_properties.py's "neither derivation guarantees a fire" reads as "sometimes fires". Measured over the 250 examples: peel 2, surname half 0. Give the number and the structural reason -- a non-hangul surname makes `w + "민준"` a mixed-script token whose effective_script is None, so it is never an activated site; the one hangul token drawn ('남궁민준') landed in an example whose policy had HANGUL out of segment_scripts. --- AGENTS.md | 2 +- nameparser/_pipeline/_script_segment.py | 29 +++++++++++++++---------- nameparser/config/suffixes.py | 2 +- tests/v2/test_properties.py | 12 +++++++++- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 69121f1..1b21c4d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -131,7 +131,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Parser owns config-dependent conveniences**: `Parser.matches`/`Parser.capitalized`/`Parser.revise` exist because the `ParsedName` equivalents fall back to DEFAULT config for str/omitted arguments (documented loudly in both docstrings). `revise` harvests tokens from a full sub-parse of each replacement value (tags kept minus `FOLDED_TAG`, roles forced, ambiguities discarded); the merge tail is shared with `replace()` via `ParsedName._with_field_tokens`. `Parser.capitalized` delegates through `name.capitalized(self.lexicon)` specifically so `_parser` never imports `_render` — keep it that way. - **Per-word vocabulary fields warn on multi-word entries** (`_normset`/`_normpairs` via `_warn_dead_entry`, UserWarning, never a raise — see the given_name_titles Gotcha for why raising is wrong). `given_name_titles` is the one multi-word-matched field and is exempt; `_edit` passes `warn=False` (add() warns once via the new instance's `__post_init__`; remove() stores nothing). The default vocabulary and every locale pack must stay warning-free (`test_default_lexicon_builds_warning_free`, `test_pack_vocabulary_entries_are_single_words`). - **Invariants guard harm, not no-ops**: add a constructor check when violating it produces a *wrong parse*, not when it produces *nothing*. A false positive costs a working configuration; a true positive on an inert condition costs the user nothing, so that trade is never worth taking. `suffix_acronyms_ambiguous ∩ suffix_words` is guarded because the overlap loses a family name; `given_name_titles` is not, because an unreachable entry is simply never consulted (see Gotchas). Before adding one, construct the config it forbids and check what actually breaks. -- **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Five exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`), and `honorific_tails = GLUED_HONORIFICS ∩ suffix_words` (#308 behavior with no v1 manager of its own, so the one v1 knob that reaches it is deleting the suffix word — which turns the peel off, `test_snapshot_removing_a_honorific_word_turns_the_peel_off`). When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. +- **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Six exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`), `honorific_tails = GLUED_HONORIFICS ∩ suffix_words` (#308 behavior with no v1 manager of its own, so the one v1 knob that reaches it is deleting the suffix word — which turns the peel off, `test_snapshot_removing_a_honorific_word_turns_the_peel_off`), and `maiden_delimiters − nickname_delimiters` on the POLICY half of the same method (v1 precedence: a pair in both v1 buckets parses as a nickname, while `Policy` resolves the overlap the other way, so the subtraction is what keeps the facade at v1 behavior, `test_snapshot_overlap_keeps_v1_nickname_precedence`). Note that last one is on the `Policy`, not the `Lexicon` — the roster is per-`_snapshot()`, not per-vocabulary-field, so a sweep that only reads the `Lexicon(...)` call misses it. When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). `PolicyPatch`'s repr shows only set (non-UNSET) fields; `_order_repr` must never raise even on an unvalidated patch's garbage `name_order` (PolicyPatch defers validation to apply time); the sweep test in `tests/v2/test_reprs.py` pins that no config repr leaks the UNSET sentinel. - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. **Document the positive direction of a partial property**: "a non-empty `ambiguities` is a signal to act on" is checkable, while "an empty one means no fork occurred" is a universal negative needing exhaustive verification -- that claim was written twice and falsified twice, at sites the author had not audited. - **The segmenter contract**: the optional `Parser(segmenter=...)` hook is parse-totality's ONE exception (locales spec section 4). Everything inside that exception is a bug in USER CODE, never a fact about the name, so it is surfaced rather than absorbed: the segmenter's own exceptions propagate, and the two protocol violations the stage can detect for itself — an answer of the wrong type, and one cutting at or past the end of the token it was handed — raise `TypeError`/`ValueError` from `_script_segment` for the same reason. The line to hold when adding a check there: a protocol violation by the segmenter's AUTHOR raises, while an adapter's defense against its own third-party library (`locales/ja.py`'s repertoire, length, reconstruction and score guards) declines with `None`, because what those catch is a fact about the content. diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 3625c72..6e4f7aa 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -3,13 +3,16 @@ Consumes: tokens, segments, structure, interpunct_offsets, segmenter. Produces: tokens, by two independent splits into sub-slices -- a listed honorific peeled off the END of the name part's last -non-post-nominal token, and the first activated-script token of the -name segment split into n+1 -- segments (index runs remapped past the -insertions), ambiguities (indices likewise remapped, plus a -SEGMENTATION report when more than one split was vocabulary-supported, -or when a segmenter's answer scored under the confidence floor). +non-post-nominal token (whose tail token also carries this module's +_PEELED_TAG), and the first activated-script token of the name segment +split into n+1 -- segments (index runs remapped past the insertions), +ambiguities (indices likewise remapped, plus a SEGMENTATION report +when more than one split was vocabulary-supported, or when a +segmenter's answer scored under the confidence floor). Reads: Policy.segment_scripts, Lexicon.surnames, -Lexicon.honorific_tails, ParseState.segmenter. +Lexicon.honorific_tails, ParseState.segmenter, and Lexicon suffix +vocabulary (via _vocab.is_suffix_strict) -- both the peel's scan-back +and the surname site ask whether a token is a post-nominal. Unspaced CJK names give tokenize no separator to find, so this stage inserts the missing token boundary by vocabulary: the first token @@ -25,11 +28,15 @@ peeled off as its own token -- 田中さん -> 田中 + さん -- so that suffix classification can claim it and the surname match or segmenter consult below sees the name rather than the name plus an honorific. It -is gated by the FAMILY comma and by 间隔号 but NOT by segment_scripts: -the vocabulary of tails is licensed by the entries themselves, each of -which can never end a name, so no per-script trust question arises. A -suffix comma does not gate it -- "Dr 김민준씨, Jr." peels within its -name part like any other. +is gated by the stage's own ASCII bail, by the FAMILY comma and by +间隔号, but NOT by segment_scripts: the vocabulary of tails is licensed +by the entries themselves, each of which can never end a name, so no +per-script trust question arises. The ASCII bail is the gate a caller +adding a LATIN tail meets -- it sits above everything here, so such a +tail fires only on a name carrying at least one non-ASCII character +(see the bail's own comment, and honorific_tails' field note). A +suffix comma gates nothing -- "Dr 김민준씨, Jr." peels within its name +part like any other. Where the VOCABULARY declines -- no prefix matched -- an optional Parser(segmenter=...) gets the token (#272 amendment 2026-07-29). diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index 2b3dfd0..ff5e4c4 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -145,7 +145,7 @@ family name in two. Spaced 殿 is safe for the reason 양/군 are: a 殿-surnamed person's name LEADS, and the suffix gate is trailing-only. -Two more are in NEITHER set, so neither spelling is recognized. 君: 王君 is +Three more are in NEITHER set, so neither spelling is recognized. 君: 王君 is a complete Chinese name (君 is a common given-name final), so the honorific reading never gets the benefit of the doubt -- while its kana spelling くん ships glued, above. Bare 선생 and 교수: they read as common nouns as readily diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index 118a26e..5c85caa 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -332,7 +332,17 @@ def _names_using(draw: st.DrawFn, lexicon: Lexicon) -> str: # words, and a bare drawn surname standing earlier in the name takes # the surname site before the unspaced token can. Instrument before # concluding either half is exercised -- the counts above are what - # that costs to find out. + # that costs to find out, and the two halves are NOT in the same + # state. Measured over this test's 250 examples: the peel fires + # twice, the surname half ZERO times -- currently inert. Structural, + # not luck: `w + "민준"` on a non-hangul surname is a MIXED-script + # token, whose effective_script is None, so it can never be an + # activated surname site at all; only a drawn HANGUL surname makes + # one, and across the run exactly one such token was sampled + # ('남궁민준'), into an example whose drawn policy had HANGUL out of + # segment_scripts. Deriving the token was still the right move -- + # it took the half off structurally-unreachable -- but a fix worth + # having would derive it in the script the policy activated. # sorted for the same reason `vocab` above is: frozenset iteration # order is not stable across runs, and an unsorted pool shifts # every index sampled_from draws -- which would defeat From 41588b4b608a61b0eebaaa1b656a14e3898b0dbe Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 12:09:11 -0700 Subject: [PATCH 31/36] Let _split tag the tail it just built (#308) The peel called _split and then rebuilt the token tuple slice-wise to set one tag on the piece _split had just created, re-deriving where its own tail landed after handing the index arithmetic off -- an idiom no other stage uses. _split now takes an optional tail_tag and applies it to the last piece, which is the tail by construction for the one caller that passes one. The surname path passes nothing and the default leaves it unchanged. Co-Authored-By: Claude Opus 5 --- nameparser/_pipeline/_script_segment.py | 35 ++++++++++++++----------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 6e4f7aa..55b1d29 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -190,9 +190,10 @@ def _pieces(text: str, splits: tuple[int, ...]) -> tuple[str, ...]: def _split(state: ParseState, i: int, splits: tuple[int, ...], - detail: str | None) -> ParseState: + detail: str | None, tail_tag: str | None = None) -> ParseState: """Cut token `i` at every offset in `splits`, recording `detail` as - a SEGMENTATION report when there is one. + a SEGMENTATION report when there is one, and adding `tail_tag` to + the LAST piece when there is one of those. The offsets arrive non-empty, ascending and interior whatever chose them. From a segmenter: non-empty is the caller's own check, @@ -203,7 +204,13 @@ def _split(state: ParseState, i: int, splits: tuple[int, ...], The ONE split path: the vocabulary hit is the single-offset case and the segmenter's answer the general one, so neither can drift - from the other's index arithmetic.""" + from the other's index arithmetic. `tail_tag` is part of keeping it + one path: the piece is tagged HERE, where it is built, rather than + by the caller, which would have to re-derive where its own tail + landed after handing the index arithmetic off. Only the peel passes + one, and the piece it wants is the last by construction -- the + honorific it cut off the end; the surname path passes nothing and + the default leaves it exactly as it was.""" token = state.tokens[i] base = token.span.start parts: list[WorkToken] = [] @@ -213,6 +220,9 @@ def _split(state: ParseState, i: int, splits: tuple[int, ...], parts.append(dataclasses.replace( token, text=piece, span=Span(base + start, base + end))) start = end + if tail_tag is not None: + parts[-1] = dataclasses.replace( + parts[-1], tags=parts[-1].tags | {tail_tag}) added = len(splits) tokens = state.tokens[:i] + tuple(parts) + state.tokens[i + 1:] # Every index the earlier stages recorded is now stale past the @@ -367,18 +377,13 @@ def _peel_honorific_tail(state: ParseState) -> ParseState: cap = min(_longest_entry(tails), len(text) - 1) for length in range(cap, 0, -1): if text[-length:] in tails: - state = _split(state, i, (len(text) - length,), None) - # Tag the tail this stage just MANUFACTURED. The segmenter's - # neighbour test below needs to tell it from a token - # somebody wrote, and no vocabulary question can: the two - # spellings put the same word in the same place, and only - # the provenance differs. - tail = state.tokens[i + 1] - tokens = (state.tokens[:i + 1] - + (dataclasses.replace( - tail, tags=tail.tags | {_PEELED_TAG}),) - + state.tokens[i + 2:]) - return dataclasses.replace(state, tokens=tokens) + # The tail carries a tag because this stage MANUFACTURED + # it. The segmenter's neighbour test below needs to tell it + # from a token somebody wrote, and no vocabulary question + # can: the two spellings put the same word in the same + # place, and only the provenance differs. + return _split(state, i, (len(text) - length,), None, + tail_tag=_PEELED_TAG) return state From 2c5edcc5e67978231d6cf0dca22b8403de0dcb70 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 12:10:10 -0700 Subject: [PATCH 32/36] Derive the peel test lexicon from the stage's own (#308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _LEX_TAILS re-declared _LEX's six-surname frozenset and its "jr" suffix word verbatim three lines below it, so an edit to _LEX -- whose comment explains why 欧/欧阳 and 남/남궁 are there -- would silently miss the peel tests that exercise the same surname half. Building it with add() produces an identical lexicon and keeps one declaration. Co-Authored-By: Claude Opus 5 --- tests/v2/pipeline/test_script_segment.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/v2/pipeline/test_script_segment.py b/tests/v2/pipeline/test_script_segment.py index fb210ff..b12d281 100644 --- a/tests/v2/pipeline/test_script_segment.py +++ b/tests/v2/pipeline/test_script_segment.py @@ -31,10 +31,7 @@ # segmenter tests below reads that membership, so a stage lexicon that # omitted it would pin a shape no real configuration can have. _TAILS = frozenset({"씨", "님", "선생님", "박사", "さん", "先生", "様"}) -_LEX_TAILS = Lexicon( - surnames=frozenset({"毛", "欧", "欧阳", "김", "남", "남궁"}), - suffix_words=frozenset({"jr"}) | _TAILS, - honorific_tails=_TAILS) +_LEX_TAILS = _LEX.add(suffix_words=_TAILS, honorific_tails=_TAILS) def _run(text: str, policy: Policy = _HAN, lexicon: Lexicon = _LEX, From 6ae335147c4aedc35cc03783dea70c0b464d5749 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 12:10:10 -0700 Subject: [PATCH 33/36] Drop a test whose assertion cannot fail (#308) test_default_lexicon_honorific_tails_are_all_suffix_words justified itself as closing a python -O gap in config/suffixes.py's import-time assert, but Lexicon.__post_init__ enforces the same invariant with a raise, which -O does not strip: Lexicon.default() can never construct a violating instance, so the assert line is unreachable as a failure and the constructor on the line above would raise first. Neither sibling subset field has such a test either; the shared coverage is the parametrized test_subset_error_names_the_fix. The useful half of the justification -- that the runtime guard, unlike the config assert, survives -O -- moves to the honorific_tails row in _SUBSET_FIELDS. Co-Authored-By: Claude Opus 5 --- nameparser/_lexicon.py | 7 +++++++ tests/v2/test_lexicon.py | 12 ------------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 09fac6b..b61ff32 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -54,6 +54,13 @@ #: as a tail would peel a period-gated word off a real name), and #: nothing needs the acronym half: the shipped tails are CJK #: honorifics, which are words. +#: The same relation is asserted a second time in config/suffixes.py, +#: over the raw GLUED_HONORIFICS/SUFFIX_NOT_ACRONYMS constants at +#: import. The two are not redundant in the way they look: that one +#: is an `assert`, stripped under `python -O`, while the check here +#: raises unconditionally -- so under -O this is what still holds the +#: SHIPPED vocabulary to the invariant, as it is the only thing that +#: ever held a caller's own. #: #: given_name_titles is deliberately NOT here and has no check of its #: own -- see the note in __post_init__ for why every attempt at one diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index fd43aa9..08f6aee 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -598,18 +598,6 @@ def test_honorific_tails_is_a_vocab_field() -> None: assert merged.honorific_tails == frozenset({"씨", "様"}) -def test_default_lexicon_honorific_tails_are_all_suffix_words() -> None: - # Not redundant with suffixes.py's GLUED_HONORIFICS <= SUFFIX_NOT_ - # ACRONYMS assert: that one relates the raw constants at import - # time (and is gone under python -O); this one relates the - # normalized fields of a constructed Lexicon, and survives -O. - # (No ambiguous subtraction here -- that lives in the v1 shim's - # _build_snapshot, which this never touches.) Structural check, - # not a content pin. - d = Lexicon.default() - assert d.honorific_tails <= d.suffix_words - - def test_removing_a_honorific_tail_is_the_peels_off_switch() -> None: # The off-switch docs/customize.rst documents, at parse level: # dropping a tail leaves the glued name unsplit while the SPACED From 31712675f5d2fb36ab0fa0c26507af1c2407a794 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 12:11:59 -0700 Subject: [PATCH 34/36] =?UTF-8?q?Pin=20the=20=E6=AE=BF=20exclusion,=20the?= =?UTF-8?q?=20one=20nothing=20held=20(#308)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every other glued-set exclusion has a case row holding it -- 氏 by zh_glued_shi_stays, 博士 by ja_glued_degree_stays, 君 by zh_glued_jun_stays, 양 by ko_suffix_matching_is_whole_token -- while 殿, which has the longest argument behind it, had none: adding it back to GLUED_HONORIFICS passed the entire suite. The discriminating input is the lone surname 鵜殿 (Udono), which a peeled 殿 would report as family 鵜 plus suffix 殿; a two-token 真殿 太郎 puts the site scan on 太郎 and never consults the exclusion. Verified by adding 殿 back: the new row and its facade twin are the only tests that then fail. Classified fix(#271) like its neighbours -- 1.4.0 gives first 鵜殿 and no last, so it is the CJK order flip that moves the field. corpus_cjk.jsonl regenerated; differential stays 0 unexplained. Co-Authored-By: Claude Opus 5 --- tests/v2/cases.py | 15 +++++++++++++++ tools/differential/corpus_cjk.jsonl | 1 + 2 files changed, 16 insertions(+) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index c7e3263..0dab9d2 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -877,6 +877,21 @@ def __post_init__(self) -> None: classification="fix(#271)", notes="likewise: 王氏 is a historical name form ('the Wang " "woman'). The spaced 田中氏 keeps its entry"), + Case("ja_glued_dono_stays", "鵜殿", + {"family": "鵜殿"}, + classification="fix(#271)", + notes="the exclusion with the longest argument behind it and, " + "until this row, the only one nothing held: adding 殿 " + "back to GLUED_HONORIFICS passed the whole suite. 鵜殿 " + "(Udono) is a real surname, one of roughly ninety " + "Japanese surnames ending in 殿 (真殿, 大殿, ...), and a " + "peeled 殿 would give family 鵜 with 殿 in suffix. It " + "has to be the BARE surname: in a two-token 真殿 太郎 " + "the site scan lands on 太郎 and the exclusion is never " + "consulted, so only a lone surname discriminates. " + "Classified to #271 like its neighbours, not parity: " + "1.4 gave first 鵜殿 and no last, and it is the CJK " + "order flip that makes the one token a family name"), Case("ja_dono_spaced", "田中 殿", {"family": "田中", "suffix": "殿"}, classification="fix(#308)", diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index cc0fa12..3306642 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -48,6 +48,7 @@ "高橋みなみ" "高橋・一郎" "高橋一郎" +"鵜殿" "김 민준" "김민준" "김민준 님" From 46fdc3e1c479e0ca14beb9f0cd1206061d41cf20 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 12:13:46 -0700 Subject: [PATCH 35/36] Put a non-ASCII shape in the scaling benchmark (#308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every unit in _SHAPES was pure ASCII, so script_segment returned at its isascii() bail before either the #308 peel or the #271 surname site ran: the file's own docstring warns the absolute tests are structurally blind to a complexity regression, and for this stage the scaling test was blind too. A repeated 씨 walks the peel's site scan-back over the whole run -- every token is a post-nominal -- and measures 4.0-4.3 at base 800, inside the clean column the ASCII shapes already occupy, so _MAX_RATIO is unchanged. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 2 +- tests/v2/test_benchmark.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1b21c4d..7379c99 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -186,7 +186,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **`_normalize` must reach a fixed point** — storage and match-time share the one fold, and `Lexicon.__setstate__` re-validates, so a value that changes on re-normalization changes under its owner. `strip().strip(".")` alone is not idempotent (`'. a .'` → `' a '` → `'a'`). The loop is the fix; keep any new stripping inside it. **Anything built on `_normalize` must converge too** — `_title_key` joins per-word `_normalize` and DROPS words that fold away; keeping the empty slot stored `'lt .'` as `'lt '`, a key match-time can never rebuild (so the entry is silently inert) and `__setstate__` rejects on the next round-trip as "not written by this version". -**Perf regressions are caught by the scaling test, not the absolute-time ones** — `tests/v2/test_benchmark.py::test_parse_cost_grows_no_worse_than_linearly` times a repeated unit at n vs 4n over nine shapes (one per pipeline inner loop) and bounds the ratio; the `_thousand_names` tests use constant-size, delimiter-free input and are structurally blind to a complexity regression. Two rules when touching it: calibrate `_MAX_RATIO` against the WEAKEST quadratic's signal (a mixed quadratic surfaces far below the textbook 16×, so the operating point `_BASE` matters more than the bound), and confirm a planted regression fails it across REPEATED runs — one failure is a coin-flip on a timing test. The nine shapes cover different dimensions (segment count only via `commas`, intra-piece accumulation only via `particles`/`conjunctions`); measure before pruning one. +**Perf regressions are caught by the scaling test, not the absolute-time ones** — `tests/v2/test_benchmark.py::test_parse_cost_grows_no_worse_than_linearly` times a repeated unit at n vs 4n over ten shapes (one per pipeline inner loop) and bounds the ratio; the `_thousand_names` tests use constant-size, delimiter-free input and are structurally blind to a complexity regression. Two rules when touching it: calibrate `_MAX_RATIO` against the WEAKEST quadratic's signal (a mixed quadratic surfaces far below the textbook 16×, so the operating point `_BASE` matters more than the bound), and confirm a planted regression fails it across REPEATED runs — one failure is a coin-flip on a timing test. The ten shapes cover different dimensions (segment count only via `commas`, intra-piece accumulation only via `particles`/`conjunctions`, non-ASCII input only via `honorifics` — the other nine are pure ASCII, so `script_segment` returns at its bail and the CJK stages go unmeasured); measure before pruning one. **Expected-failure tests use `@pytest.mark.xfail`** — the conftest parametrized fixture breaks `@unittest.expectedFailure`; always use `@pytest.mark.xfail` instead. diff --git a/tests/v2/test_benchmark.py b/tests/v2/test_benchmark.py index cd9ed15..50f3d96 100644 --- a/tests/v2/test_benchmark.py +++ b/tests/v2/test_benchmark.py @@ -50,7 +50,7 @@ def test_facade_thousand_names_under_a_second() -> None: # before pruning one that looks redundant: # # dimension covered by -# token count all nine +# token count all ten # piece count unmatched_*, plain_tokens, commas, titles # SEGMENT count commas ONLY -- deleting it leaves every # segment-keyed regression unguarded @@ -58,6 +58,14 @@ def test_facade_thousand_names_under_a_second() -> None: # conjunctions (800) -- the merge() quadratic # masked-span count delimiter_pairs, quote_pairs (0 pieces: # everything is consumed as a delimited run) +# NON-ASCII input honorifics ONLY -- every other unit here is +# pure ASCII, so script_segment returns at +# its isascii() bail and neither the #308 +# peel nor the #271 surname site is measured +# at all. A repeated 씨 is what walks the +# peel's site scan: every token is a +# post-nominal, so the scan-back crosses the +# whole run before declining _SHAPES = { "delimiter_pairs": "(a) ", # extract: matched pairs -> masked spans "quote_pairs": '"a" ', # extract: the open==close path @@ -68,6 +76,7 @@ def test_facade_thousand_names_under_a_second() -> None: "titles": "Dr. ", # group: one long title chain "particles": "van ", # group: the prefix-chain inner loop "conjunctions": "and ", # group: merge() accumulating one piece + "honorifics": "씨 ", # script_segment: the peel's site scan } _BASE = 800 @@ -95,6 +104,9 @@ def test_facade_thousand_names_under_a_second() -> None: # a gap that is not there: at 800 the bound has ~1.4x over the worst # clean run (noise headroom on a shared runner) and ~1.3x under the # weakest quadratic. Re-derive BOTH numbers when adding a shape. +# The tenth shape (honorifics) was measured into the clean column when +# it arrived: 4.0-4.3 at base 800 across repeated runs, inside the range +# the ASCII shapes already occupy, so neither number moved. _MAX_RATIO = 6.0 From 53611b1a9fbf810c62fe7ea48f1b88ddd2b9d2fe Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 31 Jul 2026 12:14:16 -0700 Subject: [PATCH 36/36] Name the escape hatch that actually works (#308) The peel costs a couple of microseconds per non-ASCII parse and is not gated on segment_scripts -- deliberate, and the prose already says so, but a performance-sensitive caller reaches for that field first. Emptying honorific_tails hits the peel's first-line guard and recovers the cost; customize.rst already documents that removal as the BEHAVIORAL off-switch, so one clause there covers the other reason to reach for it. Co-Authored-By: Claude Opus 5 --- docs/customize.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/customize.rst b/docs/customize.rst index 5e4b4cb..ea163cf 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -369,7 +369,11 @@ vocabulary is also the peel's off-switch — ``Lexicon.default().remove(honorific_tails={"さん"})`` leaves ``田中さん`` unsplit while the spaced ``田中 さん`` still reads ``さん`` as a suffix. Dropping a word from a marker field alone orphans -nothing, so that one needs no matching ``suffix_words`` edit. +nothing, so that one needs no matching ``suffix_words`` edit. Emptying +the field is also the way to opt out of what the peel *costs* a +non-ASCII parse — an empty ``honorific_tails`` stops it at its first +guard, whereas ``segment_scripts`` never gated it and so cannot turn it +off. .. note::