Tighten comments and pin review-found gaps (PR #303 review)

Comment corrections:
- _policy.py: scope the 〆 justification to personal names -- the mark
  is a general-purpose Japanese symbol (〆切, the envelope closing
  mark); only its within-names use is surname-only
- _policy.py: "a pack must not import re" -> "a range-declaring pack"
  (marker packs like ru/tr_az import re legitimately, by exposing
  module-level patterns)
- _policy.py: the regex-sync sweep scans the pipeline modules PLUS
  _render, and only private (underscore-prefixed) module-level
  patterns -- say so
- _policy.py: replace the hand-copy PR-history parenthetical with the
  durable constraint (the table lives here because packs must not
  import the pipeline)
- locales/ja.py: reflow two ragged comment wraps (wording unchanged)
- locales/ja.py: acknowledge the blanket 〆 decline's deliberate
  breadth -- mid-token 〆 (山〆太郎) measures 0.375 on the
  kanji_feature path and WOULD have carried a SEGMENTATION report;
  swept in anyway because decline is the safe direction

New pins:
- test_locales.py guard-stack stub test: assert seg("Yamada太郎") is
  None -- kills a mutation dropping whole=True from _wholly_japanese,
  which survived the whole suite
- test_locales.py pack-contents tests: _zh/_ja.DEVIATES("𠮷田") --
  pins the astral extent of the pack predicates, which a BMP-only
  hand-rolled replacement would otherwise survive
- cases.py ja_shime_with_kana_given ("〆木 ひろ") -- the kana license
  composing with the widened Han span, previously untested together
- test_policy.py test_every_script_member_has_a_range_entry -- a
  future Script member without a _SCRIPT_RANGES entry is accepted by
  Policy validation yet never classified; this turns that silent
  no-op into a failure
- test_locales.py test_namedivider_still_miscuts_shime_names -- the
  self-expiring canary behind the adapter's 〆 decline; when
  namedivider learns the mark this fails and the decline gets
  revisited instead of lasting forever

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: derek73

nameparser/_policy.py

@@ -64,8 +64,8 @@ class Script(StrEnum):
 # it -- _pipeline/_vocab.py compiles its per-script patterns from it,
 # and it is importable from the pipeline and the locale packs alike,
 # so the packs' predicates build on it too, through _script_matcher
-# below (the hand-copies this replaced dated from when the table
-# lived in the pipeline, which packs must not import).
+# below (the table lives here rather than in the pipeline because
+# packs must not import the pipeline).
 # HAN: the ideographic iteration mark U+3005 and the shime mark
 # U+3006, the URO plus Extension
 # A, the compatibility block, and the supplementary-plane block
@@ -87,9 +87,11 @@ class Script(StrEnum):
 # U+3006 〆 (the shime mark) extends that singleton to a two-codepoint
 # span on a DIFFERENT justification: unlike 々, 〆 is Script=Common
 # under UAX #24, so this is the table deliberately reaching PAST the
-# Script property, not around a block boundary -- justified because 〆
-# functions solely as a component of Japanese surnames (〆木 Shimeki,
-# 〆谷 Shimetani, 〆野) and appears in no other script's names.
+# Script property, not around a block boundary -- justified because
+# within personal names 〆 appears solely in Japanese surnames (〆木
+# Shimeki, 〆谷 Shimetani, 〆野) -- its other uses (the envelope
+# closing mark, 〆切) never reach a name parser -- and it appears in
+# no other script's names.
 # Leaving it out made 〆木 a mixed-script token: the name reversed and
 # never gated into segmentation.
 # HANGUL: precomposed syllables only -- modern Korean
@@ -153,12 +155,13 @@ def _script_matcher(*scripts: Script,
     holding a module-level re.Pattern as a marker pack needing rotator
     branch coverage, and its registry gate goes further -- a pack that
     so much as IMPORTS re without exposing such a pattern fails
-    "imports re but exposes no module-level pattern" -- so a pack must
-    not import re at all; predicates built here keep the packs
-    invisible to that sweep by construction (and spare _vocab's
-    derived matchers a declaration row in tests/v2/test_regex_sync.py's
-    completeness sweep, which scans the pipeline modules for
-    module-level patterns)."""
+    "imports re but exposes no module-level pattern" -- so a
+    range-declaring pack must not import re at all; predicates built
+    here keep the packs invisible to that sweep by construction (and
+    spare _vocab's derived matchers a declaration row in
+    tests/v2/test_regex_sync.py's completeness sweep, which scans the
+    pipeline modules (plus _render) for private module-level
+    patterns)."""
     if not scripts:
         raise ValueError("_script_matcher needs at least one Script")
     cls = "".join(f"\\U{lo:08x}-\\U{hi:08x}"

nameparser/locales/ja.py

@@ -76,12 +76,11 @@
 #: here even though the pack never ACTIVATES it: a katakana-bearing
 #: token can still be a kana-licensed composite the pack acts on
 #: (山田エミ) -- DEVIATES must declare it, and the adapter must not
-#: decline it. A
-#: pure-katakana token never reaches THE ADAPTER: KATAKANA is in no
-#: activation set, so the stage's own gate stops it before the
-#: segmenter is consulted. DEVIATES still declares one, and must: it
-#: is a predicate over any name at all, called before any gate runs,
-#: and over-declaring is its safe direction by design.
+#: decline it. A pure-katakana token never reaches THE ADAPTER:
+#: KATAKANA is in no activation set, so the stage's own gate stops it
+#: before the segmenter is consulted. DEVIATES still declares one,
+#: and must: it is a predicate over any name at all, called before
+#: any gate runs, and over-declaring is its safe direction by design.
 _JA_SCRIPTS = (Script.HAN, Script.HIRAGANA, Script.KATAKANA)
 
 # Both compiled by _policy's factory from the shared codepoint table,
@@ -129,15 +128,13 @@ def ja_segmenter(*, gbdt: bool = False) -> Segmenter:
     The adapter declines -- returns None, leaving the token whole --
     for text outside the Japanese repertoire, for text bearing the
     shime mark 〆 (namedivider 0.4.x's rule path cuts it in the wrong
-    place), for text too short to divide, for any answer that fails to
-    reconstruct its input, and for
-    any score outside [0, 1] by more than float noise (a divider
-    scoring 87.0 is broken, and its division is worth no more than its
-    score). An
-    answer that puts the whole token on one side comes back as
-    "confidently one token" rather than a decline: the divider read it
-    and found nothing to cut, which is not the same as having no
-    opinion.
+    place), for text too short to divide, for any answer that fails
+    to reconstruct its input, and for any score outside [0, 1] by
+    more than float noise (a divider scoring 87.0 is broken, and its
+    division is worth no more than its score). An answer that puts
+    the whole token on one side comes back as "confidently one token"
+    rather than a decline: the divider read it and found nothing to
+    cut, which is not the same as having no opinion.
     Answers arrive with namedivider's own confidence, which the parsing
     stage compares against its floor to decide whether to report a
     SEGMENTATION ambiguity: namedivider scores a rule-based division
@@ -176,7 +173,12 @@ def segment(text: str) -> Segmentation | None:
         # 〆野), so a wrong family would arrive with no SEGMENTATION
         # report. Decline instead: the family-first ORDER fix for 〆
         # stands regardless, and division can return when a divider
-        # knows the mark.
+        # knows the mark. The blanket check is deliberately broader
+        # than that rule-path failure: mid-token 〆 (山〆太郎) comes
+        # back 山 | 〆太郎 at 0.375 on the kanji_feature path
+        # (measured), a low-confidence answer that WOULD have carried
+        # a SEGMENTATION report -- swept in anyway, because decline is
+        # the safe direction and mid-token 〆 is vanishingly rare.
         if "〆" in text:
             return None
         # namedivider RAISES below two characters ("Name length needs

tests/v2/cases.py

@@ -638,6 +638,11 @@ def __post_init__(self) -> None:
          notes="wholly-Han lone token under the family-first entry: "
                "the whole token lands in family, unsegmented by "
                "default"),
+    Case("ja_shime_with_kana_given", "〆木 ひろ",
+         {"family": "〆木", "given": "ひろ"},
+         classification="fix(shime-mark)",
+         notes="the kana license composes with the widened Han span: "
+               "〆木 reads as kanji beside a kana given name"),
     Case("ja_nakaguro_inside_a_nickname",
          "山田 太郎 (マイケル・ジャクソン)",
          {"family": "山田", "given": "太郎",

tests/v2/test_locales.py

@@ -125,6 +125,7 @@ def test_zh_pack_contents() -> None:
     assert "欧阳" in locales.ZH.lexicon.surnames       # compound
     assert "歐陽" in locales.ZH.lexicon.surnames       # traditional
     assert _zh.DEVIATES("毛泽东")
+    assert _zh.DEVIATES("𠮷田")   # astral Han declared
     assert not _zh.DEVIATES("John Smith")
     assert not _zh.DEVIATES("김민준")   # HAN alone, not a wider union
 
@@ -205,6 +206,7 @@ def test_ja_pack_contents() -> None:
     assert locales.JA.policy.script_orders is UNSET
     assert locales.JA.lexicon == Lexicon.empty()
     assert _ja.DEVIATES("山田太郎")
+    assert _ja.DEVIATES("𠮷田")   # astral Han declared
     # katakana declared (see the pack's repertoire note)
     assert _ja.DEVIATES("マイケル")
     assert _ja.DEVIATES("みなみ")   # hiragana declared
@@ -266,6 +268,8 @@ def test_ja_adapter_guard_stack_against_a_stub(
     assert answer is not None
     assert answer.splits == (2,) and answer.confidence == 0.5
     assert seg("김민준") is None            # outside the repertoire
+    # contains JA chars but is not WHOLLY JA
+    assert seg("Yamada太郎") is None
     assert seg("林") is None                # namedivider raises below 2
 
 
@@ -485,6 +489,18 @@ def test_ja_divides_an_astral_kanji_name() -> None:
     assert [a.kind for a in n.ambiguities] == [AmbiguityKind.SEGMENTATION]
 
 
+@_needs_ja
+def test_namedivider_still_miscuts_shime_names() -> None:
+    # The canary behind the adapter's 〆 decline: namedivider 0.4.x's
+    # rule path cuts every attested 〆 surname at offset 1 with
+    # confidence 1.0. When this fails, namedivider learned the mark --
+    # revisit the decline in ja.py rather than deleting this test.
+    import namedivider
+    result = namedivider.BasicNameDivider().divide_name("〆木太郎")
+    assert (result.family, result.given) == ("〆", "木太郎")
+    assert float(result.score) == 1.0
+
+
 @_needs_ja
 def test_ja_stacked_on_zh_pays_the_zh_packs_documented_cost() -> None:
     # Stacking composes MECHANICALLY (the test above) but not for free,

tests/v2/test_policy.py

@@ -6,7 +6,7 @@
 from nameparser._policy import (
     DEFAULT_SCRIPT_ORDERS, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST,
     GIVEN_FIRST, PatronymicRule, Policy, PolicyPatch, Script, UNSET,
-    _script_matcher, apply_patch,
+    _SCRIPT_RANGES, _script_matcher, apply_patch,
 )
 from nameparser._types import Role
 
@@ -100,6 +100,13 @@ def test_script_matcher_reaches_astral_han() -> None:
     assert _script_matcher(Script.HAN, whole=True)("𠮷田")
 
 
+def test_every_script_member_has_a_range_entry() -> None:
+    # a member added to the enum alone would be accepted by
+    # Policy(segment_scripts=...) yet never classified by
+    # single_script -- a silent no-op until this fails
+    assert set(_SCRIPT_RANGES) == set(Script)
+
+
 def test_patronymic_rules_coerce_and_reject() -> None:
     p = Policy(patronymic_rules=frozenset({"east-slavic"}))  # type: ignore[arg-type]
     assert p.patronymic_rules == frozenset({PatronymicRule.EAST_SLAVIC})