diff --git a/tests/unit/test_conflict_unicode.py b/tests/unit/test_conflict_unicode.py new file mode 100644 index 00000000..35a0fa37 --- /dev/null +++ b/tests/unit/test_conflict_unicode.py @@ -0,0 +1,441 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for Unicode-normalization folding in conflict detection (#325). + +``check_conflicts`` grouped only the *relation* axis under a Unicode fold. The +subject axis and the object axis used the raw string, so a KB that mixes NFC and +NFD spellings of the same text (routine on macOS, whose filesystem and IMEs emit +Hangul in NFD) failed in both directions at once: + +* **subject axis, false negative (unsound)** — a real contradiction split into + two singleton groups and the gate passed a KB that does contain one; +* **object axis, false positive** — two objects that render identically on + screen were reported as a contradiction the reader cannot act on. + +The object fold has to happen *before* ``literal_types.normalize``, not only on +the untyped fallback: an NFD-authored typed literal does not parse, degrades to +the ``"raw"`` tag, and never meets its NFC twin under ``"scalar"``. + +Folding is NFC only — compatibility variants (fullwidth) and case stay distinct. +""" +from __future__ import annotations + +import unicodedata + +import check_conflicts +import common + + +def _fact(subject: str, relation: str, obj: str, status: str = "confirmed") -> dict[str, str]: + return { + "subject": subject, + "relation": relation, + "object": obj, + "source": "sources/x.md", + "status": status, + "confidence": "0.9", + "note": "", + } + + +def _nfc(value: str) -> str: + return unicodedata.normalize("NFC", value) + + +def _nfd(value: str) -> str: + return unicodedata.normalize("NFD", value) + + +_AMOUNT_SPEC = common.TypedRelSpec("amount", "revenue") +_TYPED_AMOUNT = {"매출": _AMOUNT_SPEC} +_TYPED_ORDINAL = {"순위": common.TypedRelSpec("ordinal", "rank")} + + +class TestTypedObjectFoldedBeforeParse: + """An NFD-authored typed literal must reach its scalar, not degrade to raw.""" + + def test_amount_nfc_vs_nfd_same_value_no_conflict(self): + # Same amount, one row authored NFD: the 억 unit decomposes, parse_amount + # fails, and the row would key as ("raw", …) against its twin's scalar. + obj = 'amount(5400,"억")' + facts = [ + _fact("갑사", "매출", _nfc(obj)), + _fact("갑사", "매출", _nfd(obj)), + ] + assert check_conflicts.detect_conflicts(facts, {"매출"}, _TYPED_AMOUNT) == {} + + def test_ordinal_nfc_vs_nfd_same_rank_no_conflict(self): + facts = [ + _fact("갑", "순위", _nfc("제3호")), + _fact("갑", "순위", _nfd("제3호")), + ] + assert check_conflicts.detect_conflicts(facts, {"순위"}, _TYPED_ORDINAL) == {} + + def test_all_nfd_kb_cross_notation_amounts_collapse(self): + # 5400억 == 0.54조 == 5.4e11. In an all-NFD KB neither side parsed before, + # so #116's cross-notation equivalence never fired there; now it does. + facts = [ + _fact("갑사", "매출", _nfd('amount(5400,"억")')), + _fact("갑사", "매출", _nfd('amount(0.54,"조")')), + ] + assert check_conflicts.detect_conflicts(facts, {"매출"}, _TYPED_AMOUNT) == {} + + def test_nfd_typed_literals_with_different_values_still_conflict(self): + # Folding must not swallow a genuine typed contradiction. + facts = [ + _fact("갑사", "매출", _nfd('amount(5400,"억")')), + _fact("갑사", "매출", _nfd('amount(1,"조")')), + ] + conflicts = check_conflicts.detect_conflicts(facts, {"매출"}, _TYPED_AMOUNT) + assert list(conflicts) == [("갑사", "매출")] + assert len(conflicts[("갑사", "매출")]) == 2 + + +class TestSubjectAxisFolded: + """Issue case (a): the unsound false negative.""" + + def test_mixed_subject_different_objects_detected(self): + facts = [ + _fact(_nfc("김철수"), "소속", "A사"), + _fact(_nfd("김철수"), "소속", "B사"), + ] + conflicts = check_conflicts.detect_conflicts(facts, {"소속"}, {}) + assert len(conflicts) == 1 + ((subject, relation), objects), = conflicts.items() + assert _nfc(subject) == _nfc("김철수") + assert relation == "소속" + assert objects == ["A사", "B사"] + + def test_mixed_subject_same_object_no_conflict(self): + facts = [ + _fact(_nfc("김철수"), "소속", "A사"), + _fact(_nfd("김철수"), "소속", "A사"), + ] + assert check_conflicts.detect_conflicts(facts, {"소속"}, {}) == {} + + def test_reported_subject_is_a_raw_spelling_actually_present(self): + # Provenance: the reported subject is one of the strings as written, not a + # synthesized NFC form. + raws = [_nfc("김철수"), _nfd("김철수")] + facts = [_fact(raws[0], "소속", "A사"), _fact(raws[1], "소속", "B사")] + conflicts = check_conflicts.detect_conflicts(facts, {"소속"}, {}) + (subject, _), = conflicts + assert subject in raws + + def test_reported_subject_prefers_the_composed_spelling(self): + # Plain min() would always return the NFD form: conjoining jamo (U+1100…) + # sort below precomposed syllables (U+AC00…). That is the spelling that + # will NOT match what a reader types from an NFC editor. + raws = [_nfc("김철수"), _nfd("김철수")] + assert min(raws) == _nfd("김철수") # the trap this avoids + for ordering in ([raws[0], raws[1]], [raws[1], raws[0]]): + facts = [_fact(ordering[0], "소속", "A사"), _fact(ordering[1], "소속", "B사")] + conflicts = check_conflicts.detect_conflicts(facts, {"소속"}, {}) + (subject, _), = conflicts + assert subject == _nfc("김철수") + + def test_distinct_subjects_are_not_merged(self): + facts = [ + _fact("김철수", "소속", "A사"), + _fact("이영희", "소속", "B사"), + ] + assert check_conflicts.detect_conflicts(facts, {"소속"}, {}) == {} + + +class TestUntypedObjectAxisFolded: + """Issue case (b): the unactionable false positive.""" + + def test_mixed_untyped_object_same_value_no_conflict(self): + facts = [ + _fact("연구소", "소속", _nfc("한국대학교")), + _fact("연구소", "소속", _nfd("한국대학교")), + ] + assert check_conflicts.detect_conflicts(facts, {"소속"}, {}) == {} + + def test_reported_object_is_a_raw_string_actually_present(self): + raws = [_nfc("한국대학교"), _nfd("한국대학교")] + facts = [ + _fact("연구소", "소속", raws[0]), + _fact("연구소", "소속", raws[1]), + _fact("연구소", "소속", "서울대학교"), + ] + conflicts = check_conflicts.detect_conflicts(facts, {"소속"}, {}) + values = conflicts[("연구소", "소속")] + # Two equivalence classes, and the merged one reports its composed form. + assert values == ["서울대학교", _nfc("한국대학교")] + + +class TestRepresentativeChoice: + """The reported string is one that was written, and the one likeliest to grep.""" + + def test_representative_prefers_the_composed_spelling(self): + # Plain min() would always return the NFD form: conjoining jamo (U+1100…) + # sort below precomposed syllables (U+AC00…). That is the spelling that + # will NOT match what a reader types from an NFC editor. + raws = {_nfc("한국대학교"), _nfd("한국대학교")} + assert min(raws) == _nfd("한국대학교") # the trap this avoids + assert check_conflicts._representative(raws) == _nfc("한국대학교") + + def test_representative_is_deterministic_when_no_form_is_composed(self): + # Two distinct NFD spellings that fold together cannot both be NFC; the + # choice must still be stable, so it falls back to lexicographic order. + raws = {_nfd("김철수"), _nfd("김철수") + "x"} + assert check_conflicts._representative(raws) == min(raws) + + +class TestNonEquivalentNotationsStayDistinct: + """NFC only: no NFKC, no casefold.""" + + def test_fullwidth_stays_a_separate_value(self): + facts = [_fact("갑", "속성", "ABC"), _fact("갑", "속성", "ABC")] + conflicts = check_conflicts.detect_conflicts(facts, {"속성"}, {}) + assert conflicts[("갑", "속성")] == sorted(["ABC", "ABC"]) + + def test_fullwidth_subject_stays_a_separate_entity(self): + facts = [_fact("ABC", "속성", "x"), _fact("ABC", "속성", "y")] + assert check_conflicts.detect_conflicts(facts, {"속성"}, {}) == {} + + def test_case_stays_a_separate_value(self): + facts = [_fact("갑", "속성", "abc"), _fact("갑", "속성", "ABC")] + conflicts = check_conflicts.detect_conflicts(facts, {"속성"}, {}) + assert conflicts[("갑", "속성")] == ["ABC", "abc"] + + def test_case_subject_stays_a_separate_entity(self): + facts = [_fact("abc", "속성", "x"), _fact("ABC", "속성", "y")] + assert check_conflicts.detect_conflicts(facts, {"속성"}, {}) == {} + + +class TestRelationAxisOutOfScope: + """Regression pin, not a defect claim. + + Folding the *relation* axis is deliberately not part of #325: the reported + relation would stay verbatim under representative restoration, but grouping + semantics would change, and how far #210's "no silent NFC coercion for + non-participating relations" was meant to reach is a maintainer decision. + Tracked as a follow-up; pinned here so the scope boundary is explicit. + """ + + def test_mixed_relation_without_aliases_still_splits(self): + facts = [ + _fact("김철수", _nfc("소속"), "A사"), + _fact("김철수", _nfd("소속"), "B사"), + ] + conflicts = check_conflicts.detect_conflicts( + facts, {_nfc("소속"), _nfd("소속")}, {}, {} + ) + assert conflicts == {} + + def test_nfd_relation_name_reported_verbatim(self): + nfd_rel = _nfd("소속") + facts = [_fact("김철수", nfd_rel, "A사"), _fact("김철수", nfd_rel, "B사")] + conflicts = check_conflicts.detect_conflicts(facts, {nfd_rel}, {}, {}) + (_, relation), = conflicts + assert relation == nfd_rel + + +class TestVariantChannels: + """``collect_conflicts`` exposes the raw spellings behind each reported string. + + ``detect_conflicts`` keeps its established return shape (36 pinned tests + assert it); the spelling maps ride extra channels so ``main`` can report a + merge without duplicating the grouping logic. Both folded axes get a channel — + the information loss is the same on each. + """ + + def test_detect_conflicts_return_shape_unchanged(self): + facts = [_fact("갑", "속성", "x"), _fact("갑", "속성", "y")] + assert check_conflicts.detect_conflicts(facts, {"속성"}, {}) == {("갑", "속성"): ["x", "y"]} + + def test_collect_returns_conflicts_and_both_spelling_maps(self): + raws = [_nfc("김철수"), _nfd("김철수")] + facts = [_fact(raws[0], "소속", "A사"), _fact(raws[1], "소속", "B사")] + conflicts, subjects, objects = check_conflicts.collect_conflicts(facts, {"소속"}, {}) + key = (_nfc("김철수"), "소속") + assert conflicts == {key: ["A사", "B사"]} + assert subjects[key] == sorted(raws) + assert objects[key] == {"A사": ["A사"], "B사": ["B사"]} + + def test_object_channel_lists_the_merged_spellings(self): + raws = [_nfc("한국대학교"), _nfd("한국대학교")] + facts = [ + _fact("연구소", "소속", raws[0]), + _fact("연구소", "소속", raws[1]), + _fact("연구소", "소속", "서울대학교"), + ] + _, _, objects = check_conflicts.collect_conflicts(facts, {"소속"}, {}) + merged = objects[("연구소", "소속")] + assert merged[_nfc("한국대학교")] == sorted(raws) + assert merged["서울대학교"] == ["서울대학교"] + + def test_single_spelling_reports_one_variant_on_both_axes(self): + facts = [_fact("갑", "속성", "x"), _fact("갑", "속성", "y")] + _, subjects, objects = check_conflicts.collect_conflicts(facts, {"속성"}, {}) + assert subjects[("갑", "속성")] == ["갑"] + assert objects[("갑", "속성")] == {"x": ["x"], "y": ["y"]} + + def test_variant_map_is_per_subject_relation_pair(self): + # A per-folded-subject (global) map would let the mixed spelling of one + # relation rewrite the reported subject of an unrelated relation. + raws = [_nfc("김철수"), _nfd("김철수")] + facts = [ + _fact(raws[0], "소속", "A사"), + _fact(raws[1], "소속", "B사"), + _fact(_nfc("김철수"), "직급", "부장"), + _fact(_nfc("김철수"), "직급", "과장"), + ] + conflicts, subjects, _ = check_conflicts.collect_conflicts(facts, {"소속", "직급"}, {}) + assert subjects[(_nfc("김철수"), "소속")] == sorted(raws) + assert subjects[(_nfc("김철수"), "직급")] == [_nfc("김철수")] + assert conflicts[(_nfc("김철수"), "직급")] == ["과장", "부장"] + + +def _run_main(monkeypatch, facts, single_valued, typed=None, aliases=None): + monkeypatch.setattr(check_conflicts, "ensure_dirs", lambda: None) + monkeypatch.setattr(check_conflicts, "load_facts", lambda: facts) + monkeypatch.setattr(check_conflicts, "single_valued_relations", lambda: single_valued) + monkeypatch.setattr(check_conflicts, "typed_relations", lambda: typed or {}) + monkeypatch.setattr(check_conflicts, "relation_aliases", lambda: aliases or {}) + return check_conflicts.main([]) + + +class TestReportExposesTheMerge: + """A merged group must say so, and must show the spellings. + + Representative restoration keeps provenance, but when folding actually merged + two spellings the reader who greps the reported string finds only some of the + rows — and the strings render identically, so a count alone is not actionable. + The disclosure is *additive*: a contradiction that a mixed spelling merely + joined is still a contradiction, so the supersede guidance must not disappear. + """ + + def test_mixed_forms_are_named_in_the_conflict_line(self, monkeypatch, capsys): + raws = [_nfc("김철수"), _nfd("김철수")] + facts = [_fact(raws[0], "소속", "A사"), _fact(raws[1], "소속", "B사")] + assert _run_main(monkeypatch, facts, {"소속"}) == 1 + err = capsys.readouterr().err + assert "(subject written in 2 mixed Unicode normalization forms)" in err + + def test_mixed_subject_spellings_are_printed_escaped(self, monkeypatch, capsys): + raws = [_nfc("김철수"), _nfd("김철수")] + facts = [_fact(raws[0], "소속", "A사"), _fact(raws[1], "소속", "B사")] + _run_main(monkeypatch, facts, {"소속"}) + err = capsys.readouterr().err + assert f" subject spellings: {ascii(raws[1])}, {ascii(raws[0])}\n" in err + + def test_mixed_object_spellings_are_printed_escaped(self, monkeypatch, capsys): + raws = [_nfc("한국대학교"), _nfd("한국대학교")] + facts = [ + _fact("연구소", "소속", raws[0]), + _fact("연구소", "소속", raws[1]), + _fact("연구소", "소속", "서울대학교"), + ] + _run_main(monkeypatch, facts, {"소속"}) + err = capsys.readouterr().err + assert f" value {raws[0]!r} spellings: {ascii(raws[1])}, {ascii(raws[0])}\n" in err + + def test_mixed_forms_get_the_unify_guidance_on_top(self, monkeypatch, capsys): + raws = [_nfc("김철수"), _nfd("김철수")] + facts = [_fact(raws[0], "소속", "A사"), _fact(raws[1], "소속", "B사")] + _run_main(monkeypatch, facts, {"소속"}) + err = capsys.readouterr().err + assert "Unify them to one form" in err + assert "status='superseded'" in err + + def test_ordinary_conflict_keeps_the_supersede_guidance(self, monkeypatch, capsys): + facts = [_fact("갑", "속성", "x"), _fact("갑", "속성", "y")] + assert _run_main(monkeypatch, facts, {"속성"}) == 1 + err = capsys.readouterr().err + assert "status='superseded'" in err + assert "mixed Unicode normalization forms" not in err + assert "Unify them to one form" not in err + + def test_supersede_guidance_survives_a_mixed_spelling_joining_a_real_conflict( + self, monkeypatch, capsys + ): + # The contradiction (A사 vs B사) is detected with or without folding: both + # differing rows carry the NFC subject. A third row spells the subject NFD, + # which makes the pair "mixed" without folding having caused the conflict. + # Superseding is still the correct repair and must still be stated. + facts = [ + _fact(_nfc("김철수"), "소속", "A사"), + _fact(_nfc("김철수"), "소속", "B사"), + _fact(_nfd("김철수"), "소속", "A사"), + ] + assert _run_main(monkeypatch, facts, {"소속"}) == 1 + err = capsys.readouterr().err + assert "mixed Unicode normalization forms" in err + assert "status='superseded'" in err + + def test_mixed_and_ordinary_conflicts_get_both_guidances(self, monkeypatch, capsys): + raws = [_nfc("김철수"), _nfd("김철수")] + facts = [ + _fact(raws[0], "소속", "A사"), + _fact(raws[1], "소속", "B사"), + _fact("갑", "속성", "x"), + _fact("갑", "속성", "y"), + ] + _run_main(monkeypatch, facts, {"소속", "속성"}) + err = capsys.readouterr().err + assert "status='superseded'" in err + assert "Unify them to one form" in err + + +class TestNfcOnlyKbByteIdentical: + """The invariant the issue asks for: an NFC-only KB reports exactly as before.""" + + def test_report_lines_unchanged_for_nfc_only_kb(self, monkeypatch, capsys): + facts = [ + _fact("갑사", "매출", 'amount(5400,"억")'), + _fact("갑사", "매출", 'amount(1,"조")'), + _fact("김철수", "소속", "A사"), + _fact("김철수", "소속", "B사"), + ] + assert _run_main(monkeypatch, facts, {"매출", "소속"}, _TYPED_AMOUNT) == 1 + assert capsys.readouterr().err == ( + "check_conflicts: 2 conflict(s) found\n" + " CONFLICT: single-valued '매출' on '갑사' has 2 values: " + 'amount(1,"조"), amount(5400,"억")\n' + " CONFLICT: single-valued '소속' on '김철수' has 2 values: A사, B사\n" + " Resolve by marking the outdated row(s) status='superseded' in " + "facts/candidates.csv, then re-run.\n" + ) + + def test_report_lines_unchanged_for_nfc_only_kb_with_aliases(self, monkeypatch, capsys): + # The alias path prints an extra suffix and re-reads relation_aliases(); + # pin it too, so the folding change cannot perturb the canonical-name + # report of a KB that has a relation-aliases.md file. + aliases = {"게재연도": "published_year", "발행년도": "published_year"} + facts = [ + _fact("논문A", "게재연도", "2005"), + _fact("논문A", "발행년도", "2007"), + _fact("논문B", "소속", "A사"), + _fact("논문B", "소속", "B사"), + ] + rc = _run_main(monkeypatch, facts, {"published_year", "소속"}, None, aliases) + assert rc == 1 + assert capsys.readouterr().err == ( + "check_conflicts: 2 conflict(s) found\n" + " CONFLICT: single-valued 'published_year' (canonical; incl. surface variants) " + "on '논문A' has 2 values: 2005, 2007\n" + " CONFLICT: single-valued '소속' on '논문B' has 2 values: A사, B사\n" + " Resolve by marking the outdated row(s) status='superseded' in " + "facts/candidates.csv, then re-run.\n" + ) + + def test_no_conflict_path_unchanged(self, monkeypatch, capsys): + facts = [_fact("김철수", "소속", "A사")] + assert _run_main(monkeypatch, facts, {"소속"}) == 0 + assert capsys.readouterr().out == ( + "check_conflicts: 0 conflicts across 1 single-valued relation(s)\n" + ) + + def test_no_single_valued_relations_early_return_unchanged(self, monkeypatch, capsys): + # Early return before collect_conflicts is ever called; pinned so the new + # three-value return cannot leak into this path. + facts = [_fact("김철수", "소속", "A사"), _fact("김철수", "소속", "B사")] + assert _run_main(monkeypatch, facts, set()) == 0 + captured = capsys.readouterr() + assert captured.out == ( + "check_conflicts: no single-valued relations declared " + "(policy/single-valued.md); nothing to check\n" + ) + assert captured.err == "" diff --git a/tools/check_conflicts.py b/tools/check_conflicts.py index cc82b976..0625a018 100644 --- a/tools/check_conflicts.py +++ b/tools/check_conflicts.py @@ -73,6 +73,42 @@ def _canonicalize(relation: str, aliases: dict[str, str]) -> str: return relation +def _fold(value: str) -> str: + """Return *value* under Unicode canonical composition (NFC). + + NFC only — **not** NFKC and **not** casefold. NFC merges strings that are + *canonically equivalent*: the same abstract characters written with + precomposed vs decomposed code points. macOS filesystems and IMEs routinely + emit Hangul decomposed, so a KB assembled from mixed sources naturally + collects both spellings of one string. It does **not** merge compatibility + variants (fullwidth ``ABC`` stays distinct from ``ABC``) nor case — those + are genuinely different values and must keep firing a conflict. + + ``common._canonical_value`` is deliberately *not* reused: it layers + amount-quote normalization on top of the Unicode fold, which would perturb + the typed-literal key space this module builds through ``literal_types``. + """ + return unicodedata.normalize("NFC", value) + + +def _representative(raws: set[str]) -> str: + """Return the string reported on behalf of a folded group of *raws*. + + Deterministic, and always one of the strings as written (provenance). Where + the group holds several normalization forms, the **composed (NFC)** spelling + wins; ties break lexicographically. + + Plain ``min`` would be deterministic too, but it picks the wrong member in + practice: Hangul conjoining jamo (U+1100…) sort below precomposed syllables + (U+AC00…), so ``min`` on a mixed group *always* returns the decomposed form — + the one that will not match if the reader types or pastes the name into a + search from an NFC editor. Preferring NFC makes the reported string the one + most likely to grep, and the full spelling list is printed alongside it for + the rows it cannot reach. + """ + return min(raws, key=lambda s: (s != _fold(s), s)) + + def _group_key(obj: str, spec: TypedRelSpec | None) -> tuple: """Return the equivalence key an *object* string is grouped under. @@ -108,12 +144,135 @@ def _group_key(obj: str, spec: TypedRelSpec | None) -> tuple: < 2**63`` guard). So this checker may group under a scalar the engine would drop. That affects **grouping only** (never insertion) and is harmless: the checker is strictly more willing to merge equivalents, never less. No behaviour - change here — note only.""" + change here — note only. + + **Unicode folding (#325):** *obj* is NFC-folded **once, on entry**, and the + folded string feeds *both* the typed ``normalize`` call and the raw fallback. + Folding before ``normalize`` is required, not cosmetic. An NFD-authored typed + literal fails to parse — ``normalize("amount", NFD('amount(5400,"억")'), units)`` + and ``normalize("ordinal", NFD("제3호"), None)`` both return ``None`` — so it + degrades to ``("raw", …)`` while its NFC twin keys as ``("scalar", …)``. The + two tags never meet, and folding only the fallback would not reach them. + Typed relations are in fact the *more* exposed axis, since amount and ordinal + units are Hangul (억/조/호/위/번/차). The ``scalar``/``raw`` tag split itself is + correct and unchanged. + + Consequence, stated plainly: an **all-NFD typed KB can change output**. Two + NFD rows that previously degraded to distinct raw strings now parse and may + collapse onto one scalar. That is not a regression — it is #116's cross- + notation equivalence (억↔조) starting to work on that KB for the first time. + The invariant held here is the one the issue asks for: an **NFC-only** KB is + byte-identical.""" + folded = _fold(obj) if spec is not None: - scalar = literal_types.normalize(spec.type, obj, spec.units) + scalar = literal_types.normalize(spec.type, folded, spec.units) if scalar is not None: return ("scalar", scalar) - return ("raw", obj) + return ("raw", folded) + + +def collect_conflicts( + facts: list[dict[str, str]], + single_valued: set[str], + typed: dict[str, TypedRelSpec] | None = None, + aliases: dict[str, str] | None = None, +) -> tuple[ + dict[tuple[str, str], list[str]], + dict[tuple[str, str], list[str]], + dict[tuple[str, str], dict[str, list[str]]], +]: + """Return ``(conflicts, subject_variants, object_variants)``. + + *conflicts* is exactly what ``detect_conflicts`` returns (see there). + + The other two are symmetric spelling channels — folding merges strings on the + subject axis and on the object axis alike, and a reported representative + stands in for every spelling it absorbed on **both**: + + * *subject_variants* maps each reported key to the sorted raw subject strings + merged under it; + * *object_variants* maps each reported key to ``{reported object: sorted raw + objects}`` for that key's value groups. + + Either list holds one entry when nothing was merged. More than one means the + KB writes that string in several Unicode normalization forms, so the reported + representative does not grep to every row behind it — which is exactly what + the caller has to disclose. + + These channels exist so ``main`` can report a merge without duplicating the + grouping logic, while ``detect_conflicts`` keeps its established return shape + (a large body of pinned tests asserts that dict directly). + + **Subject axis folding (#325):** rows group under the NFC fold of the + subject, so a contradiction written with the subject NFC on one row and NFD + on another is detected instead of splitting into two singleton groups. That + split was an *unsound* false negative: the finalize gate passed KBs that do + contain a contradiction, and nobody ever saw it. Conversely the untyped + object axis folds too, so two objects that render identically stop being + reported as a contradiction the reader cannot act on. + + The *reported* key preserves provenance: a raw subject actually seen for that + (folded subject, canonical relation) pair, chosen by ``_representative`` (NFC + spelling preferred, ties lexicographic — not a plain ``min``, see there). The + map is keyed per **pair**, never per folded subject globally — a global map + would rewrite the reported subject spelling of unrelated relations and break + byte-identity on inputs where folding merges nothing. + + **Relation axis is out of scope (#325).** Folding it as well is mechanically + possible and the #210 pins survive it (representative restoration keeps the + reported relation verbatim), so "the pins forbid it" would be a false reason. + The real reason is scope: folding the relation axis changes *grouping* + semantics, and how far #210's "no silent NFC coercion for non-participating + relations" was meant to reach is a maintainer's call. Raised as a follow-up. + + **Engine divergence (note only):** ``common.dedup_engine_atoms`` + (``factlog/common.py:1277``) dedups on the **raw** ``(subject, relation, + object)`` triple, so the engine still writes an NFC-spelled and an NFD-spelled + subject into ``accepted.dl`` as two distinct entities. After this change the + checker therefore reports a contradiction the engine cannot see from + ``accepted.dl`` alone. For the gate that direction is safe — the checker is + the stricter of the two, so nothing slips through — but the engine-side + defect remains and is not addressed here. + """ + typed = typed or {} + aliases = aliases or {} + # Precompute the set of canonical single-valued relation names so the + # per-row membership test is O(1). + sv = {_canonicalize(r, aliases) for r in single_valued} + # (folded subject, canonical_relation) -> group key -> set of raw objects. + by_key: dict[tuple[str, str], dict[tuple, set[str]]] = {} + # Same pair -> set of raw subject spellings folded into it. + raw_subjects: dict[tuple[str, str], set[str]] = {} + for row in engine_facts(facts): + relation = row["relation"] + canon = _canonicalize(relation, aliases) + if canon not in sv: + continue + obj = row["object"] + # Typed-spec lookup (#210), NOT a fold of the relation axis: the spec dict + # is keyed by NFC names, so the lookup normalizes to find it. The relation + # used for grouping stays `canon`, verbatim. + spec = typed.get(canon) or typed.get(_fold(relation)) + key = _group_key(obj, spec) + pair = (_fold(row["subject"]), canon) + by_key.setdefault(pair, {}).setdefault(key, set()).add(obj) + raw_subjects.setdefault(pair, set()).add(row["subject"]) + conflicts: dict[tuple[str, str], list[str]] = {} + subject_variants: dict[tuple[str, str], list[str]] = {} + object_variants: dict[tuple[str, str], dict[str, list[str]]] = {} + for pair, groups in by_key.items(): + if len(groups) <= 1: + continue + subjects = raw_subjects[pair] + # Representative restoration on both axes: report strings as written. + # Distinct folded subjects cannot share a representative (the choice is a + # function of the fold), so reported keys stay unique. + reported = (_representative(subjects), pair[1]) + objects = {_representative(raws): sorted(raws) for raws in groups.values()} + conflicts[reported] = sorted(objects) + subject_variants[reported] = sorted(subjects) + object_variants[reported] = objects + return conflicts, subject_variants, object_variants def detect_conflicts( @@ -130,8 +289,8 @@ def detect_conflicts( available, else the raw string — see ``_group_key``), so equivalent typed notations do not false-positive. The reported values, however, preserve the original object strings (provenance): each distinct key contributes one - deterministic representative (the lexicographically smallest raw object seen - for it). Deterministic; never raises. + deterministic representative (an object seen for it, chosen by + ``_representative``). Deterministic; never raises. Two grouping subtleties documented on ``_group_key``: ordinal collapses cross-unit notations onto the shared rank (rank-only contract, #218/#224 A), @@ -159,31 +318,14 @@ def detect_conflicts( from the alias map), then falls back to the NFC form of the raw relation string. This ensures that an NFD-authored relation that also participates in the alias map still reaches its typed spec, so equivalent notations (억↔조) - collapse correctly.""" - typed = typed or {} - aliases = aliases or {} - # Precompute the set of canonical single-valued relation names so the - # per-row membership test is O(1). - sv = {_canonicalize(r, aliases) for r in single_valued} - # (subject, canonical_relation) -> group key -> set of raw object strings. - by_key: dict[tuple[str, str], dict[tuple, set[str]]] = {} - for row in engine_facts(facts): - relation = row["relation"] - canon = _canonicalize(relation, aliases) - if canon not in sv: - continue - obj = row["object"] - # Typed-spec lookup: try canonical name first (NFC by construction when - # it came from the alias map), then NFC of the raw relation (#210). - spec = typed.get(canon) or typed.get(unicodedata.normalize("NFC", relation)) - key = _group_key(obj, spec) - groups = by_key.setdefault((row["subject"], canon), {}) - groups.setdefault(key, set()).add(obj) - return { - key: sorted(min(raws) for raws in groups.values()) - for key, groups in by_key.items() - if len(groups) > 1 - } + collapse correctly. + + **Unicode folding (#325):** the subject and untyped-object axes are folded to + NFC, and the reported strings are restored to raw spellings seen. Callers that + also need to know *which* raw spellings were merged should use + ``collect_conflicts``, which documents the grouping in full and returns those + maps alongside this dict.""" + return collect_conflicts(facts, single_valued, typed, aliases)[0] def main(argv: list[str] | None = None) -> int: @@ -197,25 +339,59 @@ def main(argv: list[str] | None = None) -> int: print("check_conflicts: no single-valued relations declared (policy/single-valued.md); nothing to check") return 0 - conflicts = detect_conflicts(load_facts(), single_valued, typed_relations(), relation_aliases()) + conflicts, subject_variants, object_variants = collect_conflicts( + load_facts(), single_valued, typed_relations(), relation_aliases() + ) if not conflicts: print(f"check_conflicts: 0 conflicts across {len(single_valued)} single-valued relation(s)") return 0 print(f"check_conflicts: {len(conflicts)} conflict(s) found", file=sys.stderr) aliases = relation_aliases() - for (subject, relation), objects in sorted(conflicts.items()): + # Whether folding merged spellings anywhere. This is an *extra* disclosure, + # never a replacement for the supersede guidance: a contradiction that a mixed + # spelling merely joined is still a contradiction, and unifying the spelling + # does not resolve it. + any_mixed = False + for key, objects in sorted(conflicts.items()): + subject, relation = key suffix = " (canonical; incl. surface variants)" if aliases and relation in set(aliases.values()) else "" + subjects = subject_variants[key] + if len(subjects) > 1: + suffix += f" (subject written in {len(subjects)} mixed Unicode normalization forms)" print( f" CONFLICT: single-valued '{relation}'{suffix} on '{subject}' has " f"{len(objects)} values: {', '.join(objects)}", file=sys.stderr, ) + # Print the spellings themselves, escaped: the whole difficulty of this + # class of conflict is that the strings render identically, so naming a + # count without the code points leaves the reader unable to act. + if len(subjects) > 1: + any_mixed = True + print(f" subject spellings: {', '.join(ascii(s) for s in subjects)}", file=sys.stderr) + for obj in objects: + raws = object_variants[key][obj] + if len(raws) > 1: + any_mixed = True + print( + f" value {obj!r} spellings: {', '.join(ascii(r) for r in raws)}", + file=sys.stderr, + ) print( " Resolve by marking the outdated row(s) status='superseded' in " "facts/candidates.csv, then re-run.", file=sys.stderr, ) + if any_mixed: + print( + " Some string(s) above were merged across Unicode normalization forms: they render " + "identically but differ byte-wise, so the reported spelling does not grep to every " + "row behind it (the spellings are listed under each conflict). Unify them to one " + "form in facts/candidates.csv as well — that is a separate repair from superseding, " + "and neither substitutes for the other.", + file=sys.stderr, + ) return 1