diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index a8d6839..7c2ddd2 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -12,9 +12,12 @@ Layering is the usual reason for a copy but not the only one, so this module's scope is the PROMISE rather than that one pair of packages: the comma-set pin below reads _pipeline._state instead of config, and -the last test reaches outside the package altogether, to a TOML file -that could not import a Python constant if it wanted to. +the last three tests reach outside the package altogether -- two to a +TOML file that could not import a Python constant if it wanted to, +one to a generated corpus whose generator can, and must stay run. """ +import importlib.util +import json import re import tomllib from pathlib import Path @@ -232,3 +235,64 @@ def test_differential_cjk_rule_matches_the_script_ranges() -> None: assert declared == expected, ( f"{toml_path.name}'s CJK name_regex declares {sorted(declared)}; " f"_SCRIPT_RANGES' BMP spans are {sorted(expected)}") + + +def test_differential_compound_rule_matches_the_script_ranges() -> None: + """The fix(cjk-delimited-nickname) rule (#295) carries the SECOND + hand copy of the script spans in the toml: its require-a-classified- + codepoint lookahead exists so the delimiter set alone cannot claim a + Latin name's first/last regression ('John 「Jack」 Kennedy' -- the + corner brackets sit outside every classified span). Pin that copy to + the table exactly as the canonical CJK rule's is, and pin the + delimiter set itself, so widening either (to ASCII quotes, say) is + an explicit decision here rather than a silent absorption change. + + Selection note for future rule authors: the canonical-CJK-rule pin + above selects by the literal '#271'/'#272' substrings and asserts + uniqueness -- this rule's slug avoids them on purpose, and any new + rule's must too. + """ + toml_path = (Path(__file__).parents[2] / "tools" / "differential" + / "expected_changes.toml") + rules = tomllib.loads(toml_path.read_text())["change"] + matched = [r for r in rules if "cjk-delimited-nickname" in r["issue"]] + assert len(matched) == 1 + regex = matched[0]["name_regex"] + assert "[「」『』・･]" in regex, ( + "the compound rule's delimiter set changed; decide deliberately") + declared = { + (int(lo, 16), int(hi, 16)) + for lo, hi in re.findall(r"\\u([0-9A-Fa-f]{4})-\\u([0-9A-Fa-f]{4})", + regex)} + expected = {span + for spans in _policy._SCRIPT_RANGES.values() + for span in spans + if span[1] <= 0xFFFF} | {(0xFF65, 0xFF65)} + assert declared == expected, ( + f"compound rule's codepoint lookahead declares {sorted(declared)}; " + f"_SCRIPT_RANGES' BMP spans are {sorted(expected)}") + + +def test_cjk_corpus_matches_the_case_table() -> None: + """corpus_cjk.jsonl is GENERATED, not curated (#295): every + distinct case-table text bearing a codepoint the script table + classifies, sorted -- see build_cjk_corpus.py for why the other + two corpora cannot carry these names. The checked-in file must + equal what the generator would write, so a CJK case row added + without regenerating fails HERE instead of silently narrowing + the differential gate back toward the blind spot #295 closed. + Same promise as the toml pin above, aimed at a generated artifact + instead of a hand copy. + """ + tools = Path(__file__).parents[2] / "tools" / "differential" + spec = importlib.util.spec_from_file_location( + "build_cjk_corpus", tools / "build_cjk_corpus.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + checked_in = [json.loads(line) for line in + (tools / "corpus_cjk.jsonl") + .read_text(encoding="utf-8").splitlines()] + assert checked_in == module.selected_names(), ( + "corpus_cjk.jsonl is stale: regenerate with " + "`uv run python tools/differential/build_cjk_corpus.py`") diff --git a/tools/differential/README.md b/tools/differential/README.md index 9da4b31..54cedaa 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -38,7 +38,7 @@ needs widening. The run must exit 0 before a 2.0 release; the classified summary it prints is the source for the "Behavior Changes" section of `docs/release_log.rst`. -## The two corpora +## The three corpora `compare.py` reads **every** `corpus*.jsonl` beside it by default (deduped), because a corpus you have to ask for by name is a corpus @@ -48,11 +48,18 @@ that stops being run. Pass `--corpus PATH` (repeatable) to narrow it. |---|---|---| | `corpus.jsonl` | v1's own test suite at a pinned ref | anything 2.0 added — v1's authors had no reason to test a typographic nickname delimiter or a Cyrillic title | | `corpus_issues.jsonl` | name-like strings harvested from the GitHub issue tracker | anything nobody ever reported | +| `corpus_cjk.jsonl` | the CJK-bearing rows of `tests/v2/cases.py`, via `build_cjk_corpus.py` (#295) | anything the case table itself missed — it re-witnesses reviewed expectations at the 1.4 boundary rather than discovering new shapes | They are deliberately separate rather than merged: `corpus.jsonl` is reproducible forever from an immutable git ref, while the issue -tracker is mutable, so regenerating the second one is an explicit, -reviewable act that can only add names. +tracker is mutable, so regenerating `corpus_issues.jsonl` is an +explicit, reviewable act that can only add names — and the CJK corpus +exists because BOTH of those are structurally blind to unspaced CJK (v1's +banks never tested it; `build_issues_corpus.py` requires an internal +space, which unspaced names never have). It regenerates from the case +table, and `tests/v2/test_regex_sync.py` pins the checked-in file +against the generator's selection, so a CJK case row added without +regenerating fails the suite instead of silently narrowing this gate. The issue corpus earned its place on the first run — 166 of its 198 names were not in `corpus.jsonl`, and it immediately surfaced five diff --git a/tools/differential/build_cjk_corpus.py b/tools/differential/build_cjk_corpus.py new file mode 100644 index 0000000..2858546 --- /dev/null +++ b/tools/differential/build_cjk_corpus.py @@ -0,0 +1,62 @@ +"""Regenerate corpus_cjk.jsonl from the CJK rows of the case table. + +The third corpus, with the third provenance (#295): corpus.jsonl +regenerates from v1's test banks at an immutable ref and +corpus_issues.jsonl harvests the issue tracker, but neither can carry +an unspaced CJK name -- v1's authors had no reason to test one, and +build_issues_corpus.py requires an internal space, which is exactly +the shape #271 classifies. This file derives from the reviewed case +table instead: every distinct `text` in tests/v2/cases.py containing +a character the script table classifies, regardless of the row's +policy/locale context -- the corpus carries name STRINGS and the +differential run parses them with the default facade, so a name from +a zh-scoped row is simply one more CJK name to diff. + +Selection is by the shipped table (nameparser._policy._SCRIPT_RANGES, +through the same _script_matcher the parser uses), so a script added +to the table widens the harvest on the next regeneration, and a case +row added for future CJK work (#298's 间隔号 forms, say) enters the +corpus by being written down in the table everyone already reviews. + +Regenerate after editing CJK case rows: + + uv run python tools/differential/build_cjk_corpus.py + +tests/v2/test_regex_sync.py pins the checked-in file against this +module's selection, so a stale corpus fails the suite rather than +silently narrowing the differential gate. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +sys.path.insert(0, str(ROOT)) + +from nameparser._policy import _SCRIPT_RANGES, _script_matcher # noqa: E402 +from tests.v2.cases import CASES # noqa: E402 + +OUT = HERE / "corpus_cjk.jsonl" + +_has_cjk = _script_matcher(*_SCRIPT_RANGES) + + +def selected_names() -> list[str]: + """Every distinct case-table text bearing a classified codepoint, + sorted for a deterministic file.""" + return sorted({case.text for case in CASES if _has_cjk(case.text)}) + + +def main() -> None: + names = selected_names() + with OUT.open("w", encoding="utf-8") as fh: + for name in names: + fh.write(json.dumps(name, ensure_ascii=False) + "\n") + print(f"wrote {len(names)} names to {OUT.name}") + + +if __name__ == "__main__": + main() diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl new file mode 100644 index 0000000..a6ce564 --- /dev/null +++ b/tools/differential/corpus_cjk.jsonl @@ -0,0 +1,31 @@ +"Dr 김민준, Jr." +"John 王" +"〆木 ひろ" +"〆木 太郎" +"〆木太郎" +"みなみ" +"マイケル" +"マイケル ジャクソン" +"マイケル・ジャクソン" +"佐々木 太郎" +"司马相如" +"夏侯惇" +"山田 エミ" +"山田 太郎 (マイケル・ジャクソン)" +"山田「タロ」太郎" +"张伟" +"毛 泽东" +"毛 김" +"毛泽东" +"田中『ハナ』花子" +"諸葛亮" +"阿明" +"高橋 みなみ" +"高橋みなみ" +"高橋・一郎" +"高橋一郎" +"김 민준" +"김민준" +"남궁" +"남궁민수" +"남궁민수, 지훈" diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index 8a72918..cf03b8b 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -44,15 +44,14 @@ issue = "fix(#271/#272) native-script CJK: family-first order, hangul segmentati # parity today. The dot is the whole mechanism; the dot is the whole # span. # -# Expected to match nothing against the current corpora, which contain -# no CJK at all -- build_issues_corpus.py requires an internal space, -# and unspaced names are the shape this classifies. That blind spot is -# tracked as #272's and #271's shared gap in issue #295, which also -# records the provenance decision a fix needs; until it is closed the -# behavioral guarantee lives in the fix(#271)/fix(#272) rows of -# tests/v2/cases.py and the fix(shime-mark) rows, whose 〆 span rides -# in on the same HAN entry, and this rule is kept ready for the moment -# a CJK name lands in a corpus. +# corpus_cjk.jsonl (#295) exists so this rule fires in every real +# run: build_cjk_corpus.py harvests every CJK-bearing text in +# tests/v2/cases.py -- the fix(#271)/fix(#272)/fix(shime-mark) rows +# AND the parity rows, whose value here is pinning NON-diffs (マイケル +# must not change against 1.4) -- closing the blind spot the other +# two corpora have by construction: v1's banks had no reason to test +# CJK, and build_issues_corpus.py requires an internal space, which +# unspaced names never have. name_regex = "[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65]" fields = ["first", "middle", "last"] @@ -123,6 +122,36 @@ issue = "feat(#273) typographic nickname delimiters recognized by default" name_regex = "[“”„«»「」『』（）]" fields = ["middle", "nickname"] +[[change]] +issue = "fix(cjk-delimited-nickname) delimiter recognition compounds with the CJK order flip" +# '山田「タロ」太郎', '山田 太郎 (マイケル・ジャクソン)': one name, two +# intended changes at once -- the corner-bracket/nakaguro handling +# (2.0's typographic delimiters; 2.1's dot separator rendering the +# nickname join with a space) changes `nickname`, while the same +# name's wholly-CJK remainder takes the 2.1 family-first flip in +# first/last. Neither single-change rule may claim the union (the +# delimiter rule's fields exclude first/last, the CJK rule's exclude +# nickname -- each on purpose, so a lone regression in the other's +# fields stays loud). Both lookaheads are required: the delimiters +# alone would match 'John 「Jack」 Kennedy' -- the brackets sit in CJK +# Symbols and Punctuation, OUTSIDE every classified span, so a Latin +# name can carry them -- and would then absorb a bare first/last +# regression on it. Requiring a classified codepoint too confines the +# rule to names the order flip can actually reach. The second +# lookahead's class is the same hand copy of _SCRIPT_RANGES the rule +# above carries, pinned by the same sync test. One absorption is +# inherent and accepted: a nickname-ONLY diff on a nakaguro name +# ('マイケル・ジャクソン') classifies here, because subset-matching +# cannot REQUIRE a field to have changed. Positional note: this rule +# overlaps the CJK rule above and the delimiter rule below; within +# the name_regex tier file order decides which label a diff reports +# under, so it sits after the tighter single-change rules on purpose. +# The issue slug deliberately avoids the literal #271/#272 strings -- +# test_regex_sync's differential pin selects the canonical CJK rule +# by those substrings and asserts it is unique. +name_regex = "(?s)(?=.*[「」『』・･])(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])" +fields = ["first", "last", "nickname"] + [[change]] issue = "feat(#269) non-Latin titles/conjunctions recognized" # 'г-н Иван Петров' (Cyrillic title), 'Хосе И Мария Сантос' (Cyrillic