From ef7e116dc27e1c085793614eee36b1756ae33683 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 00:20:40 -0700 Subject: [PATCH 1/2] Give the differential harness a CJK corpus generated from the case table The fix(#271/#272) rule in expected_changes.toml had never classified a name in a real run: corpus.jsonl regenerates from v1's test banks (no reason to test CJK) and build_issues_corpus.py requires an internal space, which unspaced names -- the exact shape the rule explains -- never have. The rule was verified once against a throwaway synthetic corpus during #294 and was otherwise dead weight the harness could not distinguish from coverage. corpus_cjk.jsonl is the third corpus with the third provenance: generated by build_cjk_corpus.py from every distinct case-table text bearing a codepoint the script table classifies, through the same _script_matcher the parser uses. Row context is deliberately ignored -- the corpus carries name STRINGS and the harness parses them with the default facade, so a zh-scoped row's text is simply one more CJK name to diff. The selection self-extends: a case row added for future CJK work enters the corpus at the next regeneration, and test_regex_sync.py pins the checked-in file against the generator's selection so a stale corpus fails the suite. First real run: 685 names, 44 intentional diffs, 0 unexplained, with the CJK rule claiming 23 names. It also immediately surfaced a shape no rule covered -- a name where delimiter recognition (corner brackets, the nakaguro nickname join) and the CJK family-first flip change fields in the SAME diff, which neither single-change rule may claim because each excludes the other's fields on purpose. The new fix(cjk-delimited-nickname) rule takes exactly that union, scoped to the CJK-typographic delimiters so it cannot absorb a Latin delimiter diff; its issue slug avoids the literal #271/#272 substrings the sync test uses to select the one canonical CJK rule. Closes #295 Co-Authored-By: Claude Fable 5 --- tests/v2/test_regex_sync.py | 27 +++++++++++ tools/differential/README.md | 15 ++++-- tools/differential/build_cjk_corpus.py | 62 ++++++++++++++++++++++++ tools/differential/corpus_cjk.jsonl | 31 ++++++++++++ tools/differential/expected_changes.toml | 35 +++++++++---- 5 files changed, 157 insertions(+), 13 deletions(-) create mode 100644 tools/differential/build_cjk_corpus.py create mode 100644 tools/differential/corpus_cjk.jsonl diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index a8d6839..486dd24 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -15,6 +15,8 @@ the last test reaches outside the package altogether, to a TOML file that could not import a Python constant if it wanted to. """ +import importlib.util +import json import re import tomllib from pathlib import Path @@ -232,3 +234,28 @@ def test_differential_cjk_rule_matches_the_script_ranges() -> None: assert declared == expected, ( f"{toml_path.name}'s CJK name_regex declares {sorted(declared)}; " f"_SCRIPT_RANGES' BMP spans are {sorted(expected)}") + + +def test_cjk_corpus_matches_the_case_table() -> None: + """corpus_cjk.jsonl is GENERATED, not curated (#295): every + distinct case-table text bearing a codepoint the script table + classifies, sorted -- see build_cjk_corpus.py for why the other + two corpora cannot carry these names. The checked-in file must + equal what the generator would write, so a CJK case row added + without regenerating fails HERE instead of silently narrowing + the differential gate back toward the blind spot #295 closed. + Same promise as the toml pin above, aimed at a generated artifact + instead of a hand copy. + """ + tools = Path(__file__).parents[2] / "tools" / "differential" + spec = importlib.util.spec_from_file_location( + "build_cjk_corpus", tools / "build_cjk_corpus.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + checked_in = [json.loads(line) for line in + (tools / "corpus_cjk.jsonl") + .read_text(encoding="utf-8").splitlines()] + assert checked_in == module.selected_names(), ( + "corpus_cjk.jsonl is stale: regenerate with " + "`uv run python tools/differential/build_cjk_corpus.py`") diff --git a/tools/differential/README.md b/tools/differential/README.md index 9da4b31..3ae3ae0 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -38,7 +38,7 @@ needs widening. The run must exit 0 before a 2.0 release; the classified summary it prints is the source for the "Behavior Changes" section of `docs/release_log.rst`. -## The two corpora +## The three corpora `compare.py` reads **every** `corpus*.jsonl` beside it by default (deduped), because a corpus you have to ask for by name is a corpus @@ -48,11 +48,18 @@ that stops being run. Pass `--corpus PATH` (repeatable) to narrow it. |---|---|---| | `corpus.jsonl` | v1's own test suite at a pinned ref | anything 2.0 added — v1's authors had no reason to test a typographic nickname delimiter or a Cyrillic title | | `corpus_issues.jsonl` | name-like strings harvested from the GitHub issue tracker | anything nobody ever reported | +| `corpus_cjk.jsonl` | the CJK-bearing rows of `tests/v2/cases.py`, via `build_cjk_corpus.py` (#295) | anything the case table itself missed — it re-witnesses reviewed expectations at the 1.4 boundary rather than discovering new shapes | They are deliberately separate rather than merged: `corpus.jsonl` is -reproducible forever from an immutable git ref, while the issue -tracker is mutable, so regenerating the second one is an explicit, -reviewable act that can only add names. +reproducible forever from an immutable git ref, the issue tracker is +mutable, so regenerating `corpus_issues.jsonl` is an explicit, +reviewable act that can only add names — and the CJK corpus exists +because BOTH of those are structurally blind to unspaced CJK (v1's +banks never tested it; `build_issues_corpus.py` requires an internal +space, which unspaced names never have). It regenerates from the case +table, and `tests/v2/test_regex_sync.py` pins the checked-in file +against the generator's selection, so a CJK case row added without +regenerating fails the suite instead of silently narrowing this gate. The issue corpus earned its place on the first run — 166 of its 198 names were not in `corpus.jsonl`, and it immediately surfaced five diff --git a/tools/differential/build_cjk_corpus.py b/tools/differential/build_cjk_corpus.py new file mode 100644 index 0000000..2858546 --- /dev/null +++ b/tools/differential/build_cjk_corpus.py @@ -0,0 +1,62 @@ +"""Regenerate corpus_cjk.jsonl from the CJK rows of the case table. + +The third corpus, with the third provenance (#295): corpus.jsonl +regenerates from v1's test banks at an immutable ref and +corpus_issues.jsonl harvests the issue tracker, but neither can carry +an unspaced CJK name -- v1's authors had no reason to test one, and +build_issues_corpus.py requires an internal space, which is exactly +the shape #271 classifies. This file derives from the reviewed case +table instead: every distinct `text` in tests/v2/cases.py containing +a character the script table classifies, regardless of the row's +policy/locale context -- the corpus carries name STRINGS and the +differential run parses them with the default facade, so a name from +a zh-scoped row is simply one more CJK name to diff. + +Selection is by the shipped table (nameparser._policy._SCRIPT_RANGES, +through the same _script_matcher the parser uses), so a script added +to the table widens the harvest on the next regeneration, and a case +row added for future CJK work (#298's 间隔号 forms, say) enters the +corpus by being written down in the table everyone already reviews. + +Regenerate after editing CJK case rows: + + uv run python tools/differential/build_cjk_corpus.py + +tests/v2/test_regex_sync.py pins the checked-in file against this +module's selection, so a stale corpus fails the suite rather than +silently narrowing the differential gate. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +sys.path.insert(0, str(ROOT)) + +from nameparser._policy import _SCRIPT_RANGES, _script_matcher # noqa: E402 +from tests.v2.cases import CASES # noqa: E402 + +OUT = HERE / "corpus_cjk.jsonl" + +_has_cjk = _script_matcher(*_SCRIPT_RANGES) + + +def selected_names() -> list[str]: + """Every distinct case-table text bearing a classified codepoint, + sorted for a deterministic file.""" + return sorted({case.text for case in CASES if _has_cjk(case.text)}) + + +def main() -> None: + names = selected_names() + with OUT.open("w", encoding="utf-8") as fh: + for name in names: + fh.write(json.dumps(name, ensure_ascii=False) + "\n") + print(f"wrote {len(names)} names to {OUT.name}") + + +if __name__ == "__main__": + main() diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl new file mode 100644 index 0000000..a6ce564 --- /dev/null +++ b/tools/differential/corpus_cjk.jsonl @@ -0,0 +1,31 @@ +"Dr 김민준, Jr." +"John 王" +"〆木 ひろ" +"〆木 太郎" +"〆木太郎" +"みなみ" +"マイケル" +"マイケル ジャクソン" +"マイケル・ジャクソン" +"佐々木 太郎" +"司马相如" +"夏侯惇" +"山田 エミ" +"山田 太郎 (マイケル・ジャクソン)" +"山田「タロ」太郎" +"张伟" +"毛 泽东" +"毛 김" +"毛泽东" +"田中『ハナ』花子" +"諸葛亮" +"阿明" +"高橋 みなみ" +"高橋みなみ" +"高橋・一郎" +"高橋一郎" +"김 민준" +"김민준" +"남궁" +"남궁민수" +"남궁민수, 지훈" diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index 8a72918..163f3ae 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -44,15 +44,13 @@ issue = "fix(#271/#272) native-script CJK: family-first order, hangul segmentati # parity today. The dot is the whole mechanism; the dot is the whole # span. # -# Expected to match nothing against the current corpora, which contain -# no CJK at all -- build_issues_corpus.py requires an internal space, -# and unspaced names are the shape this classifies. That blind spot is -# tracked as #272's and #271's shared gap in issue #295, which also -# records the provenance decision a fix needs; until it is closed the -# behavioral guarantee lives in the fix(#271)/fix(#272) rows of -# tests/v2/cases.py and the fix(shime-mark) rows, whose 〆 span rides -# in on the same HAN entry, and this rule is kept ready for the moment -# a CJK name lands in a corpus. +# corpus_cjk.jsonl (#295) exists so this rule fires in every real +# run: it derives from the fix(#271)/fix(#272)/fix(shime-mark) rows of +# tests/v2/cases.py (build_cjk_corpus.py; the 〆 span rides in on the +# same HAN entry), closing the blind spot the other two corpora have +# by construction -- v1's banks had no reason to test CJK, and +# build_issues_corpus.py requires an internal space, which unspaced +# names never have. name_regex = "[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65]" fields = ["first", "middle", "last"] @@ -123,6 +121,25 @@ issue = "feat(#273) typographic nickname delimiters recognized by default" name_regex = "[“”„«»「」『』（）]" fields = ["middle", "nickname"] +[[change]] +issue = "fix(cjk-delimited-nickname) delimiter recognition compounds with the CJK order flip" +# '山田「タロ」太郎', '山田 太郎 (マイケル・ジャクソン)': one name, two +# intended changes at once -- the corner-bracket/nakaguro handling +# (2.0's typographic delimiters; 2.1's dot separator rendering the +# nickname join with a space) changes `nickname`, while the same +# name's wholly-CJK remainder takes the 2.1 family-first flip in +# first/last. Neither single-change rule may claim the union (the +# delimiter rule's fields exclude first/last, the CJK rule's exclude +# nickname -- each on purpose, so a lone regression in the other's +# fields stays loud). Scoped to the CJK-typographic delimiters and +# the nakaguro only: a name containing 「」『』 or ・ is CJK-adjacent by +# construction, so this cannot absorb a Latin delimiter diff. The +# issue slug deliberately avoids the literal #271/#272 strings -- +# test_regex_sync's differential pin selects THE CJK rule by those +# substrings and asserts it is unique. +name_regex = "[「」『』・･]" +fields = ["first", "middle", "last", "nickname"] + [[change]] issue = "feat(#269) non-Latin titles/conjunctions recognized" # 'г-н Иван Петров' (Cyrillic title), 'Хосе И Мария Сантос' (Cyrillic From 4eff1838d5cf1d583deb4da3fb40af5b759ffa4e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 30 Jul 2026 00:33:28 -0700 Subject: [PATCH 2/2] Tighten the compound rule to names the order flip can reach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review found the delimiter-only regex claiming more than its comment said: 「」『』 sit in CJK Symbols and Punctuation, OUTSIDE every classified span, so 'John 「Jack」 Kennedy' matched and a bare first/last regression on a Latin name classified as intended. The rule now requires a classified codepoint alongside the delimiter (both lookaheads), drops the unearned middle field (all three claimed diffs touch exactly first/last/nickname), and its comment owns the one absorption subset-matching cannot avoid -- a nickname-only diff on a nakaguro name -- instead of claiming 'cannot absorb'. The second lookahead is the toml's second hand copy of the script spans, so it gets the same sync pin as the first, which also pins the delimiter set and records the #271/#272-substring taboo where a rule author will look. Plus the smaller review items: the provenance comment now credits the parity rows (the corpus members that pin NON-diffs), the positional note within the name_regex tier is stated, test_regex_sync's module docstring counts its outside-the-package tests correctly, and the README comma splice is mended. Noted, not changed: the Latin probes the review used do not fall to UNEXPLAINED without the compound rule -- the fields-only fix(suffix-routing) rule absorbs any bare first/last diff on any name, and always has. Pre-existing fields-tier looseness, outside this change. Co-Authored-By: Claude Fable 5 --- tests/v2/test_regex_sync.py | 41 ++++++++++++++++++++++-- tools/differential/README.md | 8 ++--- tools/differential/expected_changes.toml | 40 +++++++++++++++-------- 3 files changed, 69 insertions(+), 20 deletions(-) diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index 486dd24..7c2ddd2 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -12,8 +12,9 @@ Layering is the usual reason for a copy but not the only one, so this module's scope is the PROMISE rather than that one pair of packages: the comma-set pin below reads _pipeline._state instead of config, and -the last test reaches outside the package altogether, to a TOML file -that could not import a Python constant if it wanted to. +the last three tests reach outside the package altogether -- two to a +TOML file that could not import a Python constant if it wanted to, +one to a generated corpus whose generator can, and must stay run. """ import importlib.util import json @@ -236,6 +237,42 @@ def test_differential_cjk_rule_matches_the_script_ranges() -> None: f"_SCRIPT_RANGES' BMP spans are {sorted(expected)}") +def test_differential_compound_rule_matches_the_script_ranges() -> None: + """The fix(cjk-delimited-nickname) rule (#295) carries the SECOND + hand copy of the script spans in the toml: its require-a-classified- + codepoint lookahead exists so the delimiter set alone cannot claim a + Latin name's first/last regression ('John 「Jack」 Kennedy' -- the + corner brackets sit outside every classified span). Pin that copy to + the table exactly as the canonical CJK rule's is, and pin the + delimiter set itself, so widening either (to ASCII quotes, say) is + an explicit decision here rather than a silent absorption change. + + Selection note for future rule authors: the canonical-CJK-rule pin + above selects by the literal '#271'/'#272' substrings and asserts + uniqueness -- this rule's slug avoids them on purpose, and any new + rule's must too. + """ + toml_path = (Path(__file__).parents[2] / "tools" / "differential" + / "expected_changes.toml") + rules = tomllib.loads(toml_path.read_text())["change"] + matched = [r for r in rules if "cjk-delimited-nickname" in r["issue"]] + assert len(matched) == 1 + regex = matched[0]["name_regex"] + assert "[「」『』・･]" in regex, ( + "the compound rule's delimiter set changed; decide deliberately") + declared = { + (int(lo, 16), int(hi, 16)) + for lo, hi in re.findall(r"\\u([0-9A-Fa-f]{4})-\\u([0-9A-Fa-f]{4})", + regex)} + expected = {span + for spans in _policy._SCRIPT_RANGES.values() + for span in spans + if span[1] <= 0xFFFF} | {(0xFF65, 0xFF65)} + assert declared == expected, ( + f"compound rule's codepoint lookahead declares {sorted(declared)}; " + f"_SCRIPT_RANGES' BMP spans are {sorted(expected)}") + + def test_cjk_corpus_matches_the_case_table() -> None: """corpus_cjk.jsonl is GENERATED, not curated (#295): every distinct case-table text bearing a codepoint the script table diff --git a/tools/differential/README.md b/tools/differential/README.md index 3ae3ae0..54cedaa 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -51,10 +51,10 @@ that stops being run. Pass `--corpus PATH` (repeatable) to narrow it. | `corpus_cjk.jsonl` | the CJK-bearing rows of `tests/v2/cases.py`, via `build_cjk_corpus.py` (#295) | anything the case table itself missed — it re-witnesses reviewed expectations at the 1.4 boundary rather than discovering new shapes | They are deliberately separate rather than merged: `corpus.jsonl` is -reproducible forever from an immutable git ref, the issue tracker is -mutable, so regenerating `corpus_issues.jsonl` is an explicit, -reviewable act that can only add names — and the CJK corpus exists -because BOTH of those are structurally blind to unspaced CJK (v1's +reproducible forever from an immutable git ref, while the issue +tracker is mutable, so regenerating `corpus_issues.jsonl` is an +explicit, reviewable act that can only add names — and the CJK corpus +exists because BOTH of those are structurally blind to unspaced CJK (v1's banks never tested it; `build_issues_corpus.py` requires an internal space, which unspaced names never have). It regenerates from the case table, and `tests/v2/test_regex_sync.py` pins the checked-in file diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index 163f3ae..cf03b8b 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -45,12 +45,13 @@ issue = "fix(#271/#272) native-script CJK: family-first order, hangul segmentati # span. # # corpus_cjk.jsonl (#295) exists so this rule fires in every real -# run: it derives from the fix(#271)/fix(#272)/fix(shime-mark) rows of -# tests/v2/cases.py (build_cjk_corpus.py; the 〆 span rides in on the -# same HAN entry), closing the blind spot the other two corpora have -# by construction -- v1's banks had no reason to test CJK, and -# build_issues_corpus.py requires an internal space, which unspaced -# names never have. +# run: build_cjk_corpus.py harvests every CJK-bearing text in +# tests/v2/cases.py -- the fix(#271)/fix(#272)/fix(shime-mark) rows +# AND the parity rows, whose value here is pinning NON-diffs (マイケル +# must not change against 1.4) -- closing the blind spot the other +# two corpora have by construction: v1's banks had no reason to test +# CJK, and build_issues_corpus.py requires an internal space, which +# unspaced names never have. name_regex = "[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65]" fields = ["first", "middle", "last"] @@ -131,14 +132,25 @@ issue = "fix(cjk-delimited-nickname) delimiter recognition compounds with the CJ # first/last. Neither single-change rule may claim the union (the # delimiter rule's fields exclude first/last, the CJK rule's exclude # nickname -- each on purpose, so a lone regression in the other's -# fields stays loud). Scoped to the CJK-typographic delimiters and -# the nakaguro only: a name containing 「」『』 or ・ is CJK-adjacent by -# construction, so this cannot absorb a Latin delimiter diff. The -# issue slug deliberately avoids the literal #271/#272 strings -- -# test_regex_sync's differential pin selects THE CJK rule by those -# substrings and asserts it is unique. -name_regex = "[「」『』・･]" -fields = ["first", "middle", "last", "nickname"] +# fields stays loud). Both lookaheads are required: the delimiters +# alone would match 'John 「Jack」 Kennedy' -- the brackets sit in CJK +# Symbols and Punctuation, OUTSIDE every classified span, so a Latin +# name can carry them -- and would then absorb a bare first/last +# regression on it. Requiring a classified codepoint too confines the +# rule to names the order flip can actually reach. The second +# lookahead's class is the same hand copy of _SCRIPT_RANGES the rule +# above carries, pinned by the same sync test. One absorption is +# inherent and accepted: a nickname-ONLY diff on a nakaguro name +# ('マイケル・ジャクソン') classifies here, because subset-matching +# cannot REQUIRE a field to have changed. Positional note: this rule +# overlaps the CJK rule above and the delimiter rule below; within +# the name_regex tier file order decides which label a diff reports +# under, so it sits after the tighter single-change rules on purpose. +# The issue slug deliberately avoids the literal #271/#272 strings -- +# test_regex_sync's differential pin selects the canonical CJK rule +# by those substrings and asserts it is unique. +name_regex = "(?s)(?=.*[「」『』・･])(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])" +fields = ["first", "last", "nickname"] [[change]] issue = "feat(#269) non-Latin titles/conjunctions recognized"