From be5f6bdb2f0c70a31d0b945ea4b8d881479ea330 Mon Sep 17 00:00:00 2001 From: Seoyun Lee Date: Wed, 29 Jul 2026 12:06:07 +0900 Subject: [PATCH 1/2] =?UTF-8?q?fix:=20=EB=AA=A8=EC=88=9C=20=EA=B2=80?= =?UTF-8?q?=EC=B6=9C=20=EA=B7=B8=EB=A3=B9=ED=95=91=20=ED=82=A4=EC=9D=98=20?= =?UTF-8?q?=EA=B0=9D=EC=B2=B4=20=EC=B6=95=EC=9D=84=20NFC=20=EB=A1=9C=20?= =?UTF-8?q?=EC=A0=91=EB=8A=94=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 업스트림 #325 (1/2) 세 축 중 관계 축만 유니코드 정규형을 접고 있었다. untyped 객체 축은 원문 그대로라, 같은 값이 NFC/NFD 로 혼용 저술되면 화면상 구별되지 않는 두 값을 모순으로 신고했다. 사용자에게는 같은 문자열 두 개가 모순이라고 표시되므로 자기 완결적 오진이다. - `_group_key` 진입부에서 객체를 한 번 NFC 로 접고, 접은 문자열을 `literal_types.normalize` 와 raw fallback 양쪽에 쓴다. NFD 로 저술된 typed 리터럴은 파싱에 실패해 `("raw", …)` 로 degrade 하므로, fallback 만 접으면 `("scalar", …)` 인 NFC 쌍둥이와 끝내 만나지 못한다. amount·ordinal 단위가 한글(억/조/호/위)이라 typed 경로가 오히려 더 취약하다. - 병합된 그룹의 대표는 `_representative` 로 뽑는다. 단순 `min` 은 결정론적이지만 한글에서 조합형 자모(U+1100~)가 완성형(U+AC00~)보다 항상 작아 혼용 그룹에서 늘 NFD 를 고른다 — NFC 편집기에서 타이핑·복사해도 매치되지 않는 쪽이다. 완성형을 우선하되 동률은 사전순으로 깨 결정론을 유지한다. NFC 는 정준 등가만 접는다. NFKC 도 casefold 도 쓰지 않으므로 전각과 대소문자는 계속 별개 값으로 남는다. `common._canonical_value` 는 amount 인용 정규화가 얹혀 있어 재사용하지 않는다. 자체 부과 불변식은 낮춘다. 전-NFD typed KB 는 출력이 바뀔 수 있다 — 두 NFD 행이 전에는 서로 다른 raw 문자열로 갈렸다가 이제 파싱돼 한 스칼라로 접힌다. 이는 회귀가 아니라 #116 의 교차표기 등가(억↔조)가 그 KB 에서 처음 작동하기 시작하는 것이다. 이슈 AC 가 요구하는 불변식은 NFC-only KB 바이트 동일성이고 그건 유지된다. 테스트: `tests/unit/test_conflict_unicode.py` 10건. --- tests/unit/test_conflict_unicode.py | 140 ++++++++++++++++++++++++++++ tools/check_conflicts.py | 66 +++++++++++-- 2 files changed, 200 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_conflict_unicode.py diff --git a/tests/unit/test_conflict_unicode.py b/tests/unit/test_conflict_unicode.py new file mode 100644 index 00000000..091fef10 --- /dev/null +++ b/tests/unit/test_conflict_unicode.py @@ -0,0 +1,140 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for Unicode-normalization folding in conflict detection (#325). + +``check_conflicts`` folded only the *relation* axis. The object axis used the raw +string, so two objects that render identically but are spelled NFC on one row and +NFD on another were reported as a contradiction — a false positive the reader +cannot act on, because the two values look the same on screen. + +The fold has to happen *before* ``literal_types.normalize``, not only on the +untyped fallback: an NFD-authored typed literal does not parse, degrades to the +``"raw"`` tag, and never meets its NFC twin under ``"scalar"``. + +Folding is NFC only — compatibility variants (fullwidth) and case stay distinct. +""" +from __future__ import annotations + +import unicodedata + +import check_conflicts +import common + + +def _fact(subject: str, relation: str, obj: str, status: str = "confirmed") -> dict[str, str]: + return { + "subject": subject, + "relation": relation, + "object": obj, + "source": "sources/x.md", + "status": status, + "confidence": "0.9", + "note": "", + } + + +def _nfc(value: str) -> str: + return unicodedata.normalize("NFC", value) + + +def _nfd(value: str) -> str: + return unicodedata.normalize("NFD", value) + + +_AMOUNT_SPEC = common.TypedRelSpec("amount", "revenue") +_TYPED_AMOUNT = {"매출": _AMOUNT_SPEC} +_TYPED_ORDINAL = {"순위": common.TypedRelSpec("ordinal", "rank")} + + +class TestTypedObjectFoldedBeforeParse: + """An NFD-authored typed literal must reach its scalar, not degrade to raw.""" + + def test_amount_nfc_vs_nfd_same_value_no_conflict(self): + # Same amount, one row authored NFD: the 억 unit decomposes, parse_amount + # fails, and the row would key as ("raw", …) against its twin's scalar. + obj = 'amount(5400,"억")' + facts = [ + _fact("갑사", "매출", _nfc(obj)), + _fact("갑사", "매출", _nfd(obj)), + ] + assert check_conflicts.detect_conflicts(facts, {"매출"}, _TYPED_AMOUNT) == {} + + def test_ordinal_nfc_vs_nfd_same_rank_no_conflict(self): + facts = [ + _fact("갑", "순위", _nfc("제3호")), + _fact("갑", "순위", _nfd("제3호")), + ] + assert check_conflicts.detect_conflicts(facts, {"순위"}, _TYPED_ORDINAL) == {} + + def test_all_nfd_kb_cross_notation_amounts_collapse(self): + # 5400억 == 0.54조 == 5.4e11. In an all-NFD KB neither side parsed before, + # so #116's cross-notation equivalence never fired there; now it does. + facts = [ + _fact("갑사", "매출", _nfd('amount(5400,"억")')), + _fact("갑사", "매출", _nfd('amount(0.54,"조")')), + ] + assert check_conflicts.detect_conflicts(facts, {"매출"}, _TYPED_AMOUNT) == {} + + def test_nfd_typed_literals_with_different_values_still_conflict(self): + # Folding must not swallow a genuine typed contradiction. + facts = [ + _fact("갑사", "매출", _nfd('amount(5400,"억")')), + _fact("갑사", "매출", _nfd('amount(1,"조")')), + ] + conflicts = check_conflicts.detect_conflicts(facts, {"매출"}, _TYPED_AMOUNT) + assert list(conflicts) == [("갑사", "매출")] + assert len(conflicts[("갑사", "매출")]) == 2 + + +class TestUntypedObjectAxisFolded: + """Issue case (b): the unactionable false positive.""" + + def test_mixed_untyped_object_same_value_no_conflict(self): + facts = [ + _fact("연구소", "소속", _nfc("한국대학교")), + _fact("연구소", "소속", _nfd("한국대학교")), + ] + assert check_conflicts.detect_conflicts(facts, {"소속"}, {}) == {} + + def test_reported_object_is_a_raw_string_actually_present(self): + raws = [_nfc("한국대학교"), _nfd("한국대학교")] + facts = [ + _fact("연구소", "소속", raws[0]), + _fact("연구소", "소속", raws[1]), + _fact("연구소", "소속", "서울대학교"), + ] + conflicts = check_conflicts.detect_conflicts(facts, {"소속"}, {}) + values = conflicts[("연구소", "소속")] + # Two equivalence classes, and the merged one reports its composed form. + assert values == ["서울대학교", _nfc("한국대학교")] + + +class TestRepresentativeChoice: + """The reported string is one that was written, and the one likeliest to grep.""" + + def test_representative_prefers_the_composed_spelling(self): + # Plain min() would always return the NFD form: conjoining jamo (U+1100…) + # sort below precomposed syllables (U+AC00…). That is the spelling that + # will NOT match what a reader types from an NFC editor. + raws = {_nfc("한국대학교"), _nfd("한국대학교")} + assert min(raws) == _nfd("한국대학교") # the trap this avoids + assert check_conflicts._representative(raws) == _nfc("한국대학교") + + def test_representative_is_deterministic_when_no_form_is_composed(self): + # Two distinct NFD spellings that fold together cannot both be NFC; the + # choice must still be stable, so it falls back to lexicographic order. + raws = {_nfd("김철수"), _nfd("김철수") + "x"} + assert check_conflicts._representative(raws) == min(raws) + + +class TestNonEquivalentNotationsStayDistinct: + """NFC only: no NFKC, no casefold.""" + + def test_fullwidth_stays_a_separate_value(self): + facts = [_fact("갑", "속성", "ABC"), _fact("갑", "속성", "ＡＢＣ")] + conflicts = check_conflicts.detect_conflicts(facts, {"속성"}, {}) + assert conflicts[("갑", "속성")] == sorted(["ABC", "ＡＢＣ"]) + + def test_case_stays_a_separate_value(self): + facts = [_fact("갑", "속성", "abc"), _fact("갑", "속성", "ABC")] + conflicts = check_conflicts.detect_conflicts(facts, {"속성"}, {}) + assert conflicts[("갑", "속성")] == ["ABC", "abc"] diff --git a/tools/check_conflicts.py b/tools/check_conflicts.py index cc82b976..4ec678c0 100644 --- a/tools/check_conflicts.py +++ b/tools/check_conflicts.py @@ -73,6 +73,41 @@ def _canonicalize(relation: str, aliases: dict[str, str]) -> str: return relation +def _fold(value: str) -> str: + """Return *value* under Unicode canonical composition (NFC). + + NFC only — **not** NFKC and **not** casefold. NFC merges strings that are + *canonically equivalent*: the same abstract characters written with + precomposed vs decomposed code points. macOS filesystems and IMEs routinely + emit Hangul decomposed, so a KB assembled from mixed sources naturally + collects both spellings of one string. It does **not** merge compatibility + variants (fullwidth ``ＡＢＣ`` stays distinct from ``ABC``) nor case — those + are genuinely different values and must keep firing a conflict. + + ``common._canonical_value`` is deliberately *not* reused: it layers + amount-quote normalization on top of the Unicode fold, which would perturb + the typed-literal key space this module builds through ``literal_types``. + """ + return unicodedata.normalize("NFC", value) + + +def _representative(raws: set[str]) -> str: + """Return the string reported on behalf of a folded group of *raws*. + + Deterministic, and always one of the strings as written (provenance). Where + the group holds several normalization forms, the **composed (NFC)** spelling + wins; ties break lexicographically. + + Plain ``min`` would be deterministic too, but it picks the wrong member in + practice: Hangul conjoining jamo (U+1100…) sort below precomposed syllables + (U+AC00…), so ``min`` on a mixed group *always* returns the decomposed form — + the one that will not match if the reader types or pastes the name into a + search from an NFC editor. Preferring NFC makes the reported string the one + most likely to grep. + """ + return min(raws, key=lambda s: (s != _fold(s), s)) + + def _group_key(obj: str, spec: TypedRelSpec | None) -> tuple: """Return the equivalence key an *object* string is grouped under. @@ -108,12 +143,31 @@ def _group_key(obj: str, spec: TypedRelSpec | None) -> tuple: < 2**63`` guard). So this checker may group under a scalar the engine would drop. That affects **grouping only** (never insertion) and is harmless: the checker is strictly more willing to merge equivalents, never less. No behaviour - change here — note only.""" + change here — note only. + + **Unicode folding (#325):** *obj* is NFC-folded **once, on entry**, and the + folded string feeds *both* the typed ``normalize`` call and the raw fallback. + Folding before ``normalize`` is required, not cosmetic. An NFD-authored typed + literal fails to parse — ``normalize("amount", NFD('amount(5400,"억")'), units)`` + and ``normalize("ordinal", NFD("제3호"), None)`` both return ``None`` — so it + degrades to ``("raw", …)`` while its NFC twin keys as ``("scalar", …)``. The + two tags never meet, and folding only the fallback would not reach them. + Typed relations are in fact the *more* exposed axis, since amount and ordinal + units are Hangul (억/조/호/위/번/차). The ``scalar``/``raw`` tag split itself is + correct and unchanged. + + Consequence, stated plainly: an **all-NFD typed KB can change output**. Two + NFD rows that previously degraded to distinct raw strings now parse and may + collapse onto one scalar. That is not a regression — it is #116's cross- + notation equivalence (억↔조) starting to work on that KB for the first time. + The invariant held here is the one the issue asks for: an **NFC-only** KB is + byte-identical.""" + folded = _fold(obj) if spec is not None: - scalar = literal_types.normalize(spec.type, obj, spec.units) + scalar = literal_types.normalize(spec.type, folded, spec.units) if scalar is not None: return ("scalar", scalar) - return ("raw", obj) + return ("raw", folded) def detect_conflicts( @@ -130,8 +184,8 @@ def detect_conflicts( available, else the raw string — see ``_group_key``), so equivalent typed notations do not false-positive. The reported values, however, preserve the original object strings (provenance): each distinct key contributes one - deterministic representative (the lexicographically smallest raw object seen - for it). Deterministic; never raises. + deterministic representative (an object seen for it, chosen by + ``_representative``). Deterministic; never raises. Two grouping subtleties documented on ``_group_key``: ordinal collapses cross-unit notations onto the shared rank (rank-only contract, #218/#224 A), @@ -180,7 +234,7 @@ def detect_conflicts( groups = by_key.setdefault((row["subject"], canon), {}) groups.setdefault(key, set()).add(obj) return { - key: sorted(min(raws) for raws in groups.values()) + key: sorted(_representative(raws) for raws in groups.values()) for key, groups in by_key.items() if len(groups) > 1 } From d765648ac548cf5f3fb49a3b51b483984c4a2fa2 Mon Sep 17 00:00:00 2001 From: Seoyun Lee Date: Wed, 29 Jul 2026 12:07:25 +0900 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20=EB=AA=A8=EC=88=9C=20=EA=B2=80?= =?UTF-8?q?=EC=B6=9C=20=EA=B7=B8=EB=A3=B9=ED=95=91=20=ED=82=A4=EC=9D=98=20?= =?UTF-8?q?=EC=A3=BC=EC=96=B4=20=EC=B6=95=EC=9D=84=20NFC=20=EB=A1=9C=20?= =?UTF-8?q?=EC=A0=91=EB=8A=94=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 업스트림 #325 (2/2) 주어 축이 원문 그대로라, 같은 주체의 참 모순이 NFC/NFD 로 혼용 저술되면 두 개의 단일 그룹으로 쪼개져 검출되지 않았다. unsound 다 — finalize 의 모순 게이트가 같은 함수를 쓰므로, 게이트가 통과시킨 KB 에 실제 모순이 남고 사람이 볼 기회가 없다. - 그룹 키의 주어 축을 NFC 로 접고, 리포트 키는 그 (접은 주어, 정규 관계) 쌍에서 실제로 관측된 원문 주어로 복원한다. 맵 세분도는 전역이 아니라 쌍별이다. 전역으로 잡으면 폴딩이 아무것도 병합하지 않는 입력에서도 다른 관계의 주어 표기가 바뀌어 바이트 동일성이 깨진다. - `collect_conflicts` 를 추가해 병합된 원문 표기를 별도 채널로 노출한다. 주어와 객체 두 축 모두에 채널을 둔다 — 정보 손실이 양쪽에서 동일하다. `detect_conflicts` 의 반환 형태는 그대로다(핀 36건이 그 dict 을 직접 assert). - 폴딩이 병합한 표기를 리포트에 노출한다. 대표 복원만으로는 리포트의 문자열을 grep 해도 그 뒤의 모든 행이 나오지 않는다. 개수만으로는 조치할 수 없으므로 (두 표기가 화면상 동일하다) 표기 자체를 `ascii()` 로 이스케이프해 함께 찍는다. - 표기 혼용 안내는 supersede 안내를 대체하지 않고 그 위에 덧붙인다. 혼용 판정은 "폴딩이 이 모순을 만들어냈다" 가 아니라 "이 쌍에 표기가 섞여 있다" 이므로, 폴딩 이전에도 검출되던 참 모순에 NFD 행이 하나 끼는 경우가 있다. 그때 supersede 안내를 걷어내면 사용자는 표기만 통일하고 재실행하게 되고, 남은 진짜 모순의 올바른 조치는 두 번째 실행에서야 알게 된다. 두 조치는 서로를 대체하지 않는다. typed 스펙 조회의 `unicodedata.normalize("NFC", relation)` 은 `_fold` 로 표기를 통일하고, 그것이 관계 축 폴딩이 아니라 NFC 키로 저장된 스펙 dict 을 찾기 위한 조회 정규화(#210)임을 주석으로 남긴다. 그룹핑에 쓰는 관계명은 `canon` 그대로다. 범위에서 뺀 것 — 관계 축. 폴딩 자체는 기계적으로 가능하고 #210 핀도 살아남지만 (대표 복원으로 관계명이 verbatim 유지된다) 그룹핑 의미가 바뀌므로, #210 의 "no silent NFC coercion" 이 어디까지 의도한 것인지는 메인테이너 판정 사안이다. 후속 이슈로 올린다. 남는 결함 — `factlog/common.py:1277` 의 `dedup_engine_atoms` 는 원문 3튜플로 dedup 하므로 엔진은 NFC 주어와 NFD 주어를 서로 다른 개체로 `accepted.dl` 에 넣는다. 즉 이 수정 이후 체커는 엔진이 `accepted.dl` 만 보고서는 볼 수 없는 모순을 보고한다. 게이트 방향으로는 안전하나(체커가 더 엄격하다) 엔진 쪽 결함은 그대로 남는다. 테스트: 25건 추가(누계 35건). NFC-only KB 의 stderr 전문 동일성 핀은 별칭 없는 경로·별칭 있는 경로·무모순 경로·단일값 관계 미선언 조기 return 을 덮는다. --- tests/unit/test_conflict_unicode.py | 315 +++++++++++++++++++++++++++- tools/check_conflicts.py | 178 +++++++++++++--- 2 files changed, 458 insertions(+), 35 deletions(-) diff --git a/tests/unit/test_conflict_unicode.py b/tests/unit/test_conflict_unicode.py index 091fef10..35a0fa37 100644 --- a/tests/unit/test_conflict_unicode.py +++ b/tests/unit/test_conflict_unicode.py @@ -1,14 +1,19 @@ # SPDX-License-Identifier: Apache-2.0 """Unit tests for Unicode-normalization folding in conflict detection (#325). -``check_conflicts`` folded only the *relation* axis. The object axis used the raw -string, so two objects that render identically but are spelled NFC on one row and -NFD on another were reported as a contradiction — a false positive the reader -cannot act on, because the two values look the same on screen. +``check_conflicts`` grouped only the *relation* axis under a Unicode fold. The +subject axis and the object axis used the raw string, so a KB that mixes NFC and +NFD spellings of the same text (routine on macOS, whose filesystem and IMEs emit +Hangul in NFD) failed in both directions at once: -The fold has to happen *before* ``literal_types.normalize``, not only on the -untyped fallback: an NFD-authored typed literal does not parse, degrades to the -``"raw"`` tag, and never meets its NFC twin under ``"scalar"``. +* **subject axis, false negative (unsound)** — a real contradiction split into + two singleton groups and the gate passed a KB that does contain one; +* **object axis, false positive** — two objects that render identically on + screen were reported as a contradiction the reader cannot act on. + +The object fold has to happen *before* ``literal_types.normalize``, not only on +the untyped fallback: an NFD-authored typed literal does not parse, degrades to +the ``"raw"`` tag, and never meets its NFC twin under ``"scalar"``. Folding is NFC only — compatibility variants (fullwidth) and case stay distinct. """ @@ -85,6 +90,57 @@ def test_nfd_typed_literals_with_different_values_still_conflict(self): assert len(conflicts[("갑사", "매출")]) == 2 +class TestSubjectAxisFolded: + """Issue case (a): the unsound false negative.""" + + def test_mixed_subject_different_objects_detected(self): + facts = [ + _fact(_nfc("김철수"), "소속", "A사"), + _fact(_nfd("김철수"), "소속", "B사"), + ] + conflicts = check_conflicts.detect_conflicts(facts, {"소속"}, {}) + assert len(conflicts) == 1 + ((subject, relation), objects), = conflicts.items() + assert _nfc(subject) == _nfc("김철수") + assert relation == "소속" + assert objects == ["A사", "B사"] + + def test_mixed_subject_same_object_no_conflict(self): + facts = [ + _fact(_nfc("김철수"), "소속", "A사"), + _fact(_nfd("김철수"), "소속", "A사"), + ] + assert check_conflicts.detect_conflicts(facts, {"소속"}, {}) == {} + + def test_reported_subject_is_a_raw_spelling_actually_present(self): + # Provenance: the reported subject is one of the strings as written, not a + # synthesized NFC form. + raws = [_nfc("김철수"), _nfd("김철수")] + facts = [_fact(raws[0], "소속", "A사"), _fact(raws[1], "소속", "B사")] + conflicts = check_conflicts.detect_conflicts(facts, {"소속"}, {}) + (subject, _), = conflicts + assert subject in raws + + def test_reported_subject_prefers_the_composed_spelling(self): + # Plain min() would always return the NFD form: conjoining jamo (U+1100…) + # sort below precomposed syllables (U+AC00…). That is the spelling that + # will NOT match what a reader types from an NFC editor. + raws = [_nfc("김철수"), _nfd("김철수")] + assert min(raws) == _nfd("김철수") # the trap this avoids + for ordering in ([raws[0], raws[1]], [raws[1], raws[0]]): + facts = [_fact(ordering[0], "소속", "A사"), _fact(ordering[1], "소속", "B사")] + conflicts = check_conflicts.detect_conflicts(facts, {"소속"}, {}) + (subject, _), = conflicts + assert subject == _nfc("김철수") + + def test_distinct_subjects_are_not_merged(self): + facts = [ + _fact("김철수", "소속", "A사"), + _fact("이영희", "소속", "B사"), + ] + assert check_conflicts.detect_conflicts(facts, {"소속"}, {}) == {} + + class TestUntypedObjectAxisFolded: """Issue case (b): the unactionable false positive.""" @@ -134,7 +190,252 @@ def test_fullwidth_stays_a_separate_value(self): conflicts = check_conflicts.detect_conflicts(facts, {"속성"}, {}) assert conflicts[("갑", "속성")] == sorted(["ABC", "ＡＢＣ"]) + def test_fullwidth_subject_stays_a_separate_entity(self): + facts = [_fact("ABC", "속성", "x"), _fact("ＡＢＣ", "속성", "y")] + assert check_conflicts.detect_conflicts(facts, {"속성"}, {}) == {} + def test_case_stays_a_separate_value(self): facts = [_fact("갑", "속성", "abc"), _fact("갑", "속성", "ABC")] conflicts = check_conflicts.detect_conflicts(facts, {"속성"}, {}) assert conflicts[("갑", "속성")] == ["ABC", "abc"] + + def test_case_subject_stays_a_separate_entity(self): + facts = [_fact("abc", "속성", "x"), _fact("ABC", "속성", "y")] + assert check_conflicts.detect_conflicts(facts, {"속성"}, {}) == {} + + +class TestRelationAxisOutOfScope: + """Regression pin, not a defect claim. + + Folding the *relation* axis is deliberately not part of #325: the reported + relation would stay verbatim under representative restoration, but grouping + semantics would change, and how far #210's "no silent NFC coercion for + non-participating relations" was meant to reach is a maintainer decision. + Tracked as a follow-up; pinned here so the scope boundary is explicit. + """ + + def test_mixed_relation_without_aliases_still_splits(self): + facts = [ + _fact("김철수", _nfc("소속"), "A사"), + _fact("김철수", _nfd("소속"), "B사"), + ] + conflicts = check_conflicts.detect_conflicts( + facts, {_nfc("소속"), _nfd("소속")}, {}, {} + ) + assert conflicts == {} + + def test_nfd_relation_name_reported_verbatim(self): + nfd_rel = _nfd("소속") + facts = [_fact("김철수", nfd_rel, "A사"), _fact("김철수", nfd_rel, "B사")] + conflicts = check_conflicts.detect_conflicts(facts, {nfd_rel}, {}, {}) + (_, relation), = conflicts + assert relation == nfd_rel + + +class TestVariantChannels: + """``collect_conflicts`` exposes the raw spellings behind each reported string. + + ``detect_conflicts`` keeps its established return shape (36 pinned tests + assert it); the spelling maps ride extra channels so ``main`` can report a + merge without duplicating the grouping logic. Both folded axes get a channel — + the information loss is the same on each. + """ + + def test_detect_conflicts_return_shape_unchanged(self): + facts = [_fact("갑", "속성", "x"), _fact("갑", "속성", "y")] + assert check_conflicts.detect_conflicts(facts, {"속성"}, {}) == {("갑", "속성"): ["x", "y"]} + + def test_collect_returns_conflicts_and_both_spelling_maps(self): + raws = [_nfc("김철수"), _nfd("김철수")] + facts = [_fact(raws[0], "소속", "A사"), _fact(raws[1], "소속", "B사")] + conflicts, subjects, objects = check_conflicts.collect_conflicts(facts, {"소속"}, {}) + key = (_nfc("김철수"), "소속") + assert conflicts == {key: ["A사", "B사"]} + assert subjects[key] == sorted(raws) + assert objects[key] == {"A사": ["A사"], "B사": ["B사"]} + + def test_object_channel_lists_the_merged_spellings(self): + raws = [_nfc("한국대학교"), _nfd("한국대학교")] + facts = [ + _fact("연구소", "소속", raws[0]), + _fact("연구소", "소속", raws[1]), + _fact("연구소", "소속", "서울대학교"), + ] + _, _, objects = check_conflicts.collect_conflicts(facts, {"소속"}, {}) + merged = objects[("연구소", "소속")] + assert merged[_nfc("한국대학교")] == sorted(raws) + assert merged["서울대학교"] == ["서울대학교"] + + def test_single_spelling_reports_one_variant_on_both_axes(self): + facts = [_fact("갑", "속성", "x"), _fact("갑", "속성", "y")] + _, subjects, objects = check_conflicts.collect_conflicts(facts, {"속성"}, {}) + assert subjects[("갑", "속성")] == ["갑"] + assert objects[("갑", "속성")] == {"x": ["x"], "y": ["y"]} + + def test_variant_map_is_per_subject_relation_pair(self): + # A per-folded-subject (global) map would let the mixed spelling of one + # relation rewrite the reported subject of an unrelated relation. + raws = [_nfc("김철수"), _nfd("김철수")] + facts = [ + _fact(raws[0], "소속", "A사"), + _fact(raws[1], "소속", "B사"), + _fact(_nfc("김철수"), "직급", "부장"), + _fact(_nfc("김철수"), "직급", "과장"), + ] + conflicts, subjects, _ = check_conflicts.collect_conflicts(facts, {"소속", "직급"}, {}) + assert subjects[(_nfc("김철수"), "소속")] == sorted(raws) + assert subjects[(_nfc("김철수"), "직급")] == [_nfc("김철수")] + assert conflicts[(_nfc("김철수"), "직급")] == ["과장", "부장"] + + +def _run_main(monkeypatch, facts, single_valued, typed=None, aliases=None): + monkeypatch.setattr(check_conflicts, "ensure_dirs", lambda: None) + monkeypatch.setattr(check_conflicts, "load_facts", lambda: facts) + monkeypatch.setattr(check_conflicts, "single_valued_relations", lambda: single_valued) + monkeypatch.setattr(check_conflicts, "typed_relations", lambda: typed or {}) + monkeypatch.setattr(check_conflicts, "relation_aliases", lambda: aliases or {}) + return check_conflicts.main([]) + + +class TestReportExposesTheMerge: + """A merged group must say so, and must show the spellings. + + Representative restoration keeps provenance, but when folding actually merged + two spellings the reader who greps the reported string finds only some of the + rows — and the strings render identically, so a count alone is not actionable. + The disclosure is *additive*: a contradiction that a mixed spelling merely + joined is still a contradiction, so the supersede guidance must not disappear. + """ + + def test_mixed_forms_are_named_in_the_conflict_line(self, monkeypatch, capsys): + raws = [_nfc("김철수"), _nfd("김철수")] + facts = [_fact(raws[0], "소속", "A사"), _fact(raws[1], "소속", "B사")] + assert _run_main(monkeypatch, facts, {"소속"}) == 1 + err = capsys.readouterr().err + assert "(subject written in 2 mixed Unicode normalization forms)" in err + + def test_mixed_subject_spellings_are_printed_escaped(self, monkeypatch, capsys): + raws = [_nfc("김철수"), _nfd("김철수")] + facts = [_fact(raws[0], "소속", "A사"), _fact(raws[1], "소속", "B사")] + _run_main(monkeypatch, facts, {"소속"}) + err = capsys.readouterr().err + assert f" subject spellings: {ascii(raws[1])}, {ascii(raws[0])}\n" in err + + def test_mixed_object_spellings_are_printed_escaped(self, monkeypatch, capsys): + raws = [_nfc("한국대학교"), _nfd("한국대학교")] + facts = [ + _fact("연구소", "소속", raws[0]), + _fact("연구소", "소속", raws[1]), + _fact("연구소", "소속", "서울대학교"), + ] + _run_main(monkeypatch, facts, {"소속"}) + err = capsys.readouterr().err + assert f" value {raws[0]!r} spellings: {ascii(raws[1])}, {ascii(raws[0])}\n" in err + + def test_mixed_forms_get_the_unify_guidance_on_top(self, monkeypatch, capsys): + raws = [_nfc("김철수"), _nfd("김철수")] + facts = [_fact(raws[0], "소속", "A사"), _fact(raws[1], "소속", "B사")] + _run_main(monkeypatch, facts, {"소속"}) + err = capsys.readouterr().err + assert "Unify them to one form" in err + assert "status='superseded'" in err + + def test_ordinary_conflict_keeps_the_supersede_guidance(self, monkeypatch, capsys): + facts = [_fact("갑", "속성", "x"), _fact("갑", "속성", "y")] + assert _run_main(monkeypatch, facts, {"속성"}) == 1 + err = capsys.readouterr().err + assert "status='superseded'" in err + assert "mixed Unicode normalization forms" not in err + assert "Unify them to one form" not in err + + def test_supersede_guidance_survives_a_mixed_spelling_joining_a_real_conflict( + self, monkeypatch, capsys + ): + # The contradiction (A사 vs B사) is detected with or without folding: both + # differing rows carry the NFC subject. A third row spells the subject NFD, + # which makes the pair "mixed" without folding having caused the conflict. + # Superseding is still the correct repair and must still be stated. + facts = [ + _fact(_nfc("김철수"), "소속", "A사"), + _fact(_nfc("김철수"), "소속", "B사"), + _fact(_nfd("김철수"), "소속", "A사"), + ] + assert _run_main(monkeypatch, facts, {"소속"}) == 1 + err = capsys.readouterr().err + assert "mixed Unicode normalization forms" in err + assert "status='superseded'" in err + + def test_mixed_and_ordinary_conflicts_get_both_guidances(self, monkeypatch, capsys): + raws = [_nfc("김철수"), _nfd("김철수")] + facts = [ + _fact(raws[0], "소속", "A사"), + _fact(raws[1], "소속", "B사"), + _fact("갑", "속성", "x"), + _fact("갑", "속성", "y"), + ] + _run_main(monkeypatch, facts, {"소속", "속성"}) + err = capsys.readouterr().err + assert "status='superseded'" in err + assert "Unify them to one form" in err + + +class TestNfcOnlyKbByteIdentical: + """The invariant the issue asks for: an NFC-only KB reports exactly as before.""" + + def test_report_lines_unchanged_for_nfc_only_kb(self, monkeypatch, capsys): + facts = [ + _fact("갑사", "매출", 'amount(5400,"억")'), + _fact("갑사", "매출", 'amount(1,"조")'), + _fact("김철수", "소속", "A사"), + _fact("김철수", "소속", "B사"), + ] + assert _run_main(monkeypatch, facts, {"매출", "소속"}, _TYPED_AMOUNT) == 1 + assert capsys.readouterr().err == ( + "check_conflicts: 2 conflict(s) found\n" + " CONFLICT: single-valued '매출' on '갑사' has 2 values: " + 'amount(1,"조"), amount(5400,"억")\n' + " CONFLICT: single-valued '소속' on '김철수' has 2 values: A사, B사\n" + " Resolve by marking the outdated row(s) status='superseded' in " + "facts/candidates.csv, then re-run.\n" + ) + + def test_report_lines_unchanged_for_nfc_only_kb_with_aliases(self, monkeypatch, capsys): + # The alias path prints an extra suffix and re-reads relation_aliases(); + # pin it too, so the folding change cannot perturb the canonical-name + # report of a KB that has a relation-aliases.md file. + aliases = {"게재연도": "published_year", "발행년도": "published_year"} + facts = [ + _fact("논문A", "게재연도", "2005"), + _fact("논문A", "발행년도", "2007"), + _fact("논문B", "소속", "A사"), + _fact("논문B", "소속", "B사"), + ] + rc = _run_main(monkeypatch, facts, {"published_year", "소속"}, None, aliases) + assert rc == 1 + assert capsys.readouterr().err == ( + "check_conflicts: 2 conflict(s) found\n" + " CONFLICT: single-valued 'published_year' (canonical; incl. surface variants) " + "on '논문A' has 2 values: 2005, 2007\n" + " CONFLICT: single-valued '소속' on '논문B' has 2 values: A사, B사\n" + " Resolve by marking the outdated row(s) status='superseded' in " + "facts/candidates.csv, then re-run.\n" + ) + + def test_no_conflict_path_unchanged(self, monkeypatch, capsys): + facts = [_fact("김철수", "소속", "A사")] + assert _run_main(monkeypatch, facts, {"소속"}) == 0 + assert capsys.readouterr().out == ( + "check_conflicts: 0 conflicts across 1 single-valued relation(s)\n" + ) + + def test_no_single_valued_relations_early_return_unchanged(self, monkeypatch, capsys): + # Early return before collect_conflicts is ever called; pinned so the new + # three-value return cannot leak into this path. + facts = [_fact("김철수", "소속", "A사"), _fact("김철수", "소속", "B사")] + assert _run_main(monkeypatch, facts, set()) == 0 + captured = capsys.readouterr() + assert captured.out == ( + "check_conflicts: no single-valued relations declared " + "(policy/single-valued.md); nothing to check\n" + ) + assert captured.err == "" diff --git a/tools/check_conflicts.py b/tools/check_conflicts.py index 4ec678c0..0625a018 100644 --- a/tools/check_conflicts.py +++ b/tools/check_conflicts.py @@ -103,7 +103,8 @@ def _representative(raws: set[str]) -> str: (U+AC00…), so ``min`` on a mixed group *always* returns the decomposed form — the one that will not match if the reader types or pastes the name into a search from an NFC editor. Preferring NFC makes the reported string the one - most likely to grep. + most likely to grep, and the full spelling list is printed alongside it for + the rows it cannot reach. """ return min(raws, key=lambda s: (s != _fold(s), s)) @@ -170,6 +171,110 @@ def _group_key(obj: str, spec: TypedRelSpec | None) -> tuple: return ("raw", folded) +def collect_conflicts( + facts: list[dict[str, str]], + single_valued: set[str], + typed: dict[str, TypedRelSpec] | None = None, + aliases: dict[str, str] | None = None, +) -> tuple[ + dict[tuple[str, str], list[str]], + dict[tuple[str, str], list[str]], + dict[tuple[str, str], dict[str, list[str]]], +]: + """Return ``(conflicts, subject_variants, object_variants)``. + + *conflicts* is exactly what ``detect_conflicts`` returns (see there). + + The other two are symmetric spelling channels — folding merges strings on the + subject axis and on the object axis alike, and a reported representative + stands in for every spelling it absorbed on **both**: + + * *subject_variants* maps each reported key to the sorted raw subject strings + merged under it; + * *object_variants* maps each reported key to ``{reported object: sorted raw + objects}`` for that key's value groups. + + Either list holds one entry when nothing was merged. More than one means the + KB writes that string in several Unicode normalization forms, so the reported + representative does not grep to every row behind it — which is exactly what + the caller has to disclose. + + These channels exist so ``main`` can report a merge without duplicating the + grouping logic, while ``detect_conflicts`` keeps its established return shape + (a large body of pinned tests asserts that dict directly). + + **Subject axis folding (#325):** rows group under the NFC fold of the + subject, so a contradiction written with the subject NFC on one row and NFD + on another is detected instead of splitting into two singleton groups. That + split was an *unsound* false negative: the finalize gate passed KBs that do + contain a contradiction, and nobody ever saw it. Conversely the untyped + object axis folds too, so two objects that render identically stop being + reported as a contradiction the reader cannot act on. + + The *reported* key preserves provenance: a raw subject actually seen for that + (folded subject, canonical relation) pair, chosen by ``_representative`` (NFC + spelling preferred, ties lexicographic — not a plain ``min``, see there). The + map is keyed per **pair**, never per folded subject globally — a global map + would rewrite the reported subject spelling of unrelated relations and break + byte-identity on inputs where folding merges nothing. + + **Relation axis is out of scope (#325).** Folding it as well is mechanically + possible and the #210 pins survive it (representative restoration keeps the + reported relation verbatim), so "the pins forbid it" would be a false reason. + The real reason is scope: folding the relation axis changes *grouping* + semantics, and how far #210's "no silent NFC coercion for non-participating + relations" was meant to reach is a maintainer's call. Raised as a follow-up. + + **Engine divergence (note only):** ``common.dedup_engine_atoms`` + (``factlog/common.py:1277``) dedups on the **raw** ``(subject, relation, + object)`` triple, so the engine still writes an NFC-spelled and an NFD-spelled + subject into ``accepted.dl`` as two distinct entities. After this change the + checker therefore reports a contradiction the engine cannot see from + ``accepted.dl`` alone. For the gate that direction is safe — the checker is + the stricter of the two, so nothing slips through — but the engine-side + defect remains and is not addressed here. + """ + typed = typed or {} + aliases = aliases or {} + # Precompute the set of canonical single-valued relation names so the + # per-row membership test is O(1). + sv = {_canonicalize(r, aliases) for r in single_valued} + # (folded subject, canonical_relation) -> group key -> set of raw objects. + by_key: dict[tuple[str, str], dict[tuple, set[str]]] = {} + # Same pair -> set of raw subject spellings folded into it. + raw_subjects: dict[tuple[str, str], set[str]] = {} + for row in engine_facts(facts): + relation = row["relation"] + canon = _canonicalize(relation, aliases) + if canon not in sv: + continue + obj = row["object"] + # Typed-spec lookup (#210), NOT a fold of the relation axis: the spec dict + # is keyed by NFC names, so the lookup normalizes to find it. The relation + # used for grouping stays `canon`, verbatim. + spec = typed.get(canon) or typed.get(_fold(relation)) + key = _group_key(obj, spec) + pair = (_fold(row["subject"]), canon) + by_key.setdefault(pair, {}).setdefault(key, set()).add(obj) + raw_subjects.setdefault(pair, set()).add(row["subject"]) + conflicts: dict[tuple[str, str], list[str]] = {} + subject_variants: dict[tuple[str, str], list[str]] = {} + object_variants: dict[tuple[str, str], dict[str, list[str]]] = {} + for pair, groups in by_key.items(): + if len(groups) <= 1: + continue + subjects = raw_subjects[pair] + # Representative restoration on both axes: report strings as written. + # Distinct folded subjects cannot share a representative (the choice is a + # function of the fold), so reported keys stay unique. + reported = (_representative(subjects), pair[1]) + objects = {_representative(raws): sorted(raws) for raws in groups.values()} + conflicts[reported] = sorted(objects) + subject_variants[reported] = sorted(subjects) + object_variants[reported] = objects + return conflicts, subject_variants, object_variants + + def detect_conflicts( facts: list[dict[str, str]], single_valued: set[str], @@ -213,31 +318,14 @@ def detect_conflicts( from the alias map), then falls back to the NFC form of the raw relation string. This ensures that an NFD-authored relation that also participates in the alias map still reaches its typed spec, so equivalent notations (억↔조) - collapse correctly.""" - typed = typed or {} - aliases = aliases or {} - # Precompute the set of canonical single-valued relation names so the - # per-row membership test is O(1). - sv = {_canonicalize(r, aliases) for r in single_valued} - # (subject, canonical_relation) -> group key -> set of raw object strings. - by_key: dict[tuple[str, str], dict[tuple, set[str]]] = {} - for row in engine_facts(facts): - relation = row["relation"] - canon = _canonicalize(relation, aliases) - if canon not in sv: - continue - obj = row["object"] - # Typed-spec lookup: try canonical name first (NFC by construction when - # it came from the alias map), then NFC of the raw relation (#210). - spec = typed.get(canon) or typed.get(unicodedata.normalize("NFC", relation)) - key = _group_key(obj, spec) - groups = by_key.setdefault((row["subject"], canon), {}) - groups.setdefault(key, set()).add(obj) - return { - key: sorted(_representative(raws) for raws in groups.values()) - for key, groups in by_key.items() - if len(groups) > 1 - } + collapse correctly. + + **Unicode folding (#325):** the subject and untyped-object axes are folded to + NFC, and the reported strings are restored to raw spellings seen. Callers that + also need to know *which* raw spellings were merged should use + ``collect_conflicts``, which documents the grouping in full and returns those + maps alongside this dict.""" + return collect_conflicts(facts, single_valued, typed, aliases)[0] def main(argv: list[str] | None = None) -> int: @@ -251,25 +339,59 @@ def main(argv: list[str] | None = None) -> int: print("check_conflicts: no single-valued relations declared (policy/single-valued.md); nothing to check") return 0 - conflicts = detect_conflicts(load_facts(), single_valued, typed_relations(), relation_aliases()) + conflicts, subject_variants, object_variants = collect_conflicts( + load_facts(), single_valued, typed_relations(), relation_aliases() + ) if not conflicts: print(f"check_conflicts: 0 conflicts across {len(single_valued)} single-valued relation(s)") return 0 print(f"check_conflicts: {len(conflicts)} conflict(s) found", file=sys.stderr) aliases = relation_aliases() - for (subject, relation), objects in sorted(conflicts.items()): + # Whether folding merged spellings anywhere. This is an *extra* disclosure, + # never a replacement for the supersede guidance: a contradiction that a mixed + # spelling merely joined is still a contradiction, and unifying the spelling + # does not resolve it. + any_mixed = False + for key, objects in sorted(conflicts.items()): + subject, relation = key suffix = " (canonical; incl. surface variants)" if aliases and relation in set(aliases.values()) else "" + subjects = subject_variants[key] + if len(subjects) > 1: + suffix += f" (subject written in {len(subjects)} mixed Unicode normalization forms)" print( f" CONFLICT: single-valued '{relation}'{suffix} on '{subject}' has " f"{len(objects)} values: {', '.join(objects)}", file=sys.stderr, ) + # Print the spellings themselves, escaped: the whole difficulty of this + # class of conflict is that the strings render identically, so naming a + # count without the code points leaves the reader unable to act. + if len(subjects) > 1: + any_mixed = True + print(f" subject spellings: {', '.join(ascii(s) for s in subjects)}", file=sys.stderr) + for obj in objects: + raws = object_variants[key][obj] + if len(raws) > 1: + any_mixed = True + print( + f" value {obj!r} spellings: {', '.join(ascii(r) for r in raws)}", + file=sys.stderr, + ) print( " Resolve by marking the outdated row(s) status='superseded' in " "facts/candidates.csv, then re-run.", file=sys.stderr, ) + if any_mixed: + print( + " Some string(s) above were merged across Unicode normalization forms: they render " + "identically but differ byte-wise, so the reported spelling does not grep to every " + "row behind it (the spellings are listed under each conflict). Unify them to one " + "form in facts/candidates.csv as well — that is a separate repair from superseding, " + "and neither substitutes for the other.", + file=sys.stderr, + ) return 1