From ff171381a06a1d9cf0b040e9baf26d25a6cb9092 Mon Sep 17 00:00:00 2001 From: Seoyun Lee Date: Wed, 29 Jul 2026 11:37:00 +0900 Subject: [PATCH 1/9] =?UTF-8?q?fix:=20=ED=83=80=EC=9E=85=20=EB=A6=AC?= =?UTF-8?q?=ED=84=B0=EB=9F=B4=20=ED=8C=8C=EC=84=9C=EA=B0=80=20=EC=A0=84?= =?UTF-8?q?=EA=B0=81=20=EC=88=AB=EC=9E=90=EB=A5=BC=20=EA=B1=B0=EB=B6=80?= =?UTF-8?q?=ED=95=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit literal_types 의 아홉 정규식이 모두 `\d` 로 숫자를 인식했다. 파이썬의 `\d` 는 유니코드 Nd 범주 전체에 매치하므로 `１００억` 이 `100억` 과 **같은 스칼라**로 파싱되면서 relation/3 에 저장되는 객체 문자열은 서로 달랐다. 그래서 "같은 값인가" 에 대한 답이 코드 경로마다 갈렸다 — typed 관계 아래서는 병합되고, 객체 매치 질의에서는 조용히 미스하고, typed 선언이 없으면 별개 엔티티로 남았다. 수치 그룹 21곳을 정규식 단위로 다시 써 `[0-9]` 로 좁힌다. 전각 값은 "파싱 실패 → 타입 없이 적재" 라는 기존 계약을 그대로 타고, _project_typed_relations 의 경고로 표면화된다. 접기(fold)를 택하지 않은 이유는 그것이 저장된 사실 문자열을 다시 쓰는 결정이기 때문이다. 거부가 접기보다 일관된 근거가 하나 더 있다. cli.py 의 status 경로는 이미 원문 객체로 conflict 를 세는데 check_conflicts.py 만 스칼라로 병합해 0 을 보고한다. 두 경로가 현재 서로 모순이며, 전각을 거부하면 그 불일치가 해소된다. re.ASCII 는 쓰지 않는다. 같은 플래그가 `\s` 도 좁혀 `date(2020,1)` 이 조용히 깨진다. `\D+` 단위 그룹은 그대로 둔다 — `\D` 는 Nd 의 여집합이라 이미 전각을 배제한다. 모듈 밖 세 경로의 동작이 함께 바뀐다(의도된 결과, 데이터 재작성 없음). merge_candidates 의 dedup 키가 전각 두 표기를 더는 합치지 않고, 기존 KB 에 저장된 전각 값은 _canonical_value 로 정준화되지 않아 질의가 미스하며, humanize 는 전각 term 을 원문 그대로 돌려준다. 모듈 docstring 에 적어 두었다. 잔여 증상도 함께 적었다. 표면화는 typed 로 **선언된** 관계에서만 일어난다. attribute 선언만 있고 typed 가 아닌 관계의 전각 값은 여전히 어디에도 뜨지 않는다. 업스트림 #331 --- factlog/literal_types.py | 70 +++++++++++++--- tests/unit/test_literal_types.py | 137 +++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 13 deletions(-) diff --git a/factlog/literal_types.py b/factlog/literal_types.py index 4554093f..c42fabae 100644 --- a/factlog/literal_types.py +++ b/factlog/literal_types.py @@ -17,6 +17,37 @@ a sub-base-unit fraction is rounded to the nearest int (ROUND_HALF_UP). The engine has no float column, so the base-unit value MUST be an exact integer — see ``parse_amount``. + +**Digits are ASCII-only.** Python's ``\\d`` matches the whole Unicode ``Nd`` +category, so a full-width ``１００억`` used to parse to the *same scalar* as +``100억`` while ``relation/3`` kept the differing object string — one value with +three different answers to "is this the same?" depending on the code path. The +policy is **reject, not fold**: every numeric group below is spelled ``[0-9]``, +so a full-width value takes the ordinary "does not parse -> untyped" path and +surfaces as a typed-projection warning instead of merging silently. Rewriting the +stored string would have been the silent fold this repo consistently refuses. + +Three call sites outside this module change behaviour as a consequence. All three +are intended, and none of them rewrites data: + +- ``tools/merge_candidates.py`` dedup key — ``canonical_amount`` returns ``None`` + for a full-width term, so ``amount(１００,억)`` and ``amount(１００,"억")`` stay + two rows instead of collapsing into one. Collapsing them is exactly the fold + this policy rejects. +- ``common._canonical_value`` — a fact already stored as ``amount(１００,"억")`` + in an existing KB is no longer canonicalised, so a query written as + ``amount(１００,억)`` misses it. Intended: under this policy ``amount(１００,억)`` + is not a valid amount term at all, so there is no canonical form to map it to. + The fix is to correct the source to ASCII and re-collect, not to fold here. +- ``humanize`` — a full-width compound term is now returned verbatim rather than + rendered as ``１００억``. + +Residual symptom, NOT fixed here: the warning only fires for relations **declared +typed**. ``common._project_typed_relations`` skips a relation with no typed spec +without warning, so a full-width value under a relation that is declared as an +attribute but not typed still surfaces nowhere — ``tools/entity_audit.py`` lists +``declared_literals`` as an unflagged sorted list, and ``100억`` / ``１００억`` +sort far apart in it. """ from __future__ import annotations @@ -41,7 +72,13 @@ "조": 10**12, } -_DATE_RE = re.compile(r"^(\d{4})[.\-/](\d{1,2})(?:[.\-/](\d{1,2}))?$") +# Every numeric group below is written ``[0-9]``, never ``\d`` — see the ASCII-only +# paragraph in the module docstring. Do NOT reach for ``re.ASCII`` to get the same +# effect: the flag also narrows ``\s``, and U+3000 (ideographic space) inside +# ``date(2020,1)`` would silently stop parsing. The ``\D+`` unit group in +# ``_AMOUNT_RE`` needs no change: ``\D`` is the complement of ``Nd``, so it already +# excludes full-width digits (``1２3억`` therefore does not match either half). +_DATE_RE = re.compile(r"^([0-9]{4})[.\-/]([0-9]{1,2})(?:[.\-/]([0-9]{1,2}))?$") # The compound form is year-precision friendly: month AND day are optional, so # ``date(2020)`` parses (a bibliographic record normally knows only the year). # This mirrors the prose path, where a missing day already defaults to ``01`` @@ -50,29 +87,29 @@ # it is indistinguishable from a plain number, so only the explicitly typed # compound term opts into year precision. _DATE_COMPOUND_RE = re.compile( - r"^date\(\s*(\d{4})(?:\s*,\s*(\d{1,2})(?:\s*,\s*(\d{1,2}))?)?\s*\)$", + r"^date\(\s*([0-9]{4})(?:\s*,\s*([0-9]{1,2})(?:\s*,\s*([0-9]{1,2}))?)?\s*\)$", re.IGNORECASE, ) -_NUMBER_RE = re.compile(r"^-?\d[\d,]*(?:\.\d+)?$") +_NUMBER_RE = re.compile(r"^-?[0-9][0-9,]*(?:\.[0-9]+)?$") _NUMBER_COMPOUND_RE = re.compile( - r"^number\(\s*\"?(-?\d[\d,]*(?:\.\d+)?)\"?\s*\)$", + r"^number\(\s*\"?(-?[0-9][0-9,]*(?:\.[0-9]+)?)\"?\s*\)$", re.IGNORECASE, ) -_ORDINAL_KO_RE = re.compile(r"^제?(\d+)\s*(?:호|위|번|차|등|째)$") -_ORDINAL_EN_RE = re.compile(r"^(\d+)\s*(?:st|nd|rd|th)$", re.IGNORECASE) -_ORDINAL_COMPOUND_RE = re.compile(r"^ordinal\(\s*(\d+)\s*\)$", re.IGNORECASE) +_ORDINAL_KO_RE = re.compile(r"^제?([0-9]+)\s*(?:호|위|번|차|등|째)$") +_ORDINAL_EN_RE = re.compile(r"^([0-9]+)\s*(?:st|nd|rd|th)$", re.IGNORECASE) +_ORDINAL_COMPOUND_RE = re.compile(r"^ordinal\(\s*([0-9]+)\s*\)$", re.IGNORECASE) # , contiguous OR a single space between them. The number part is a # plain/comma/decimal magnitude with an OPTIONAL leading sign (a loss/credit may be # negative); the unit is validated against the table by the caller. A leading `제` # (ordinal marker) can't match because the `num` group is anchored to an optional -# sign + leading digit (`^-?\d…`), so `제3호`-style ordinals never match (the first -# char `제` is neither `-` nor a digit → no match). -_AMOUNT_RE = re.compile(r"^(?P-?\d[\d,]*(?:\.\d+)?) ?(?P\D+)$") +# sign + leading digit (`^-?[0-9]…`), so `제3호`-style ordinals never match (the +# first char `제` is neither `-` nor a digit → no match). +_AMOUNT_RE = re.compile(r"^(?P-?[0-9][0-9,]*(?:\.[0-9]+)?) ?(?P\D+)$") # Compound amount: the unit may be quoted ("...", allowing spaces and commas) or # bare (no comma/paren/quote). The number is optionally quoted. Canonicalisation # always emits the quoted unit form (see ``canonical_amount``). _AMOUNT_COMPOUND_RE = re.compile( - r'^amount\(\s*"?(?P-?\d[\d,]*(?:\.\d+)?)"?\s*,\s*' + r'^amount\(\s*"?(?P-?[0-9][0-9,]*(?:\.[0-9]+)?)"?\s*,\s*' r'(?:"(?P[^"]*)"|(?P[^,)"]+))\s*\)$', re.IGNORECASE, ) @@ -250,7 +287,12 @@ def canonical_amount(raw: str) -> str | None: reaches ``facts/accepted.dl`` as ``"amount(7,\\"억\\")"`` and loads cleanly. Both the bare (``amount(7,억)``) and quoted (``amount(7,"억")``) input forms canonicalise to the same quoted output, so a re-merge is idempotent and the - dedup key collapses the two.""" + dedup key collapses the two. + + A full-width number (``amount(１００,억)``) is not an amount compound term, so + it returns ``None`` and the two spellings stay separate rows in the + ``tools/merge_candidates.py`` dedup key — the intended consequence of the + ASCII-only digit policy (see the module docstring).""" m = _AMOUNT_COMPOUND_RE.match(raw.strip()) if not m: return None @@ -298,7 +340,9 @@ def humanize(value: str) -> str: ``ordinal(N)`` is intentionally NOT humanized: the source unit (호/위/번) is lost at normalization, so a bare rank would be ambiguous. Any non-compound or unrecognized string is returned verbatim, so a KB that emits no compound - objects is byte-identical.""" + objects is byte-identical. A full-width term (``amount(１００,"억")``) is not + recognized under the ASCII-only digit policy, so it too comes back verbatim + rather than rendered.""" text = value.strip() m = _DATE_COMPOUND_RE.match(text) if m: diff --git a/tests/unit/test_literal_types.py b/tests/unit/test_literal_types.py index c74ba094..38a3b000 100644 --- a/tests/unit/test_literal_types.py +++ b/tests/unit/test_literal_types.py @@ -377,3 +377,140 @@ def test_detected_and_parsed(self, raw, type_tag, expected): from entity_audit import _LITERAL_RE assert _LITERAL_RE.match(raw), f"entity_audit no longer detects {raw!r}" assert lt.normalize(type_tag, raw) == expected + + +class TestFullWidthDigitsRejected: + """ASCII-only digits (#331). Python's ``\\d`` covers the whole Unicode ``Nd`` + category, so a full-width ``１００억`` used to normalize to the SAME scalar as + ``100억`` while ``relation/3`` stored a different object string — the two + spellings merged under a typed relation but stayed separate entities and + missed each other in object-match queries. The policy is reject, not fold: a + full-width value takes the ordinary "does not parse -> untyped" path.""" + + @pytest.mark.parametrize("type_tag,raw", [ + # Each of these returned a scalar before the fix (20200101 / 20200101 / + # 123000 / 3 / 10000000000 / 10000000000), never None. + ("date", "date(２０２０,１)"), + ("date", "２０２０.１"), + ("number", "number(１２３)"), + ("number", "１２３"), + ("ordinal", "ordinal(３)"), + ("ordinal", "３위"), + ("ordinal", "제３호"), + ("amount", "１００억"), + ("amount", 'amount(１００,"억")'), + ]) + def test_normalize_rejects(self, type_tag, raw): + assert lt.normalize(type_tag, raw) is None + + @pytest.mark.parametrize("parser,raw", [ + (lt.parse_date, "date(２０２０,１)"), + (lt.parse_number, "number(１２３)"), + (lt.parse_number_scaled, "number(１２３)"), + (lt.parse_ordinal, "ordinal(３)"), + ]) + def test_public_parsers_reject(self, parser, raw): + assert parser(raw) is None + + def test_parse_amount_rejects(self): + assert lt.parse_amount("１００억", lt.DEFAULT_AMOUNT_UNITS) is None + + @pytest.mark.parametrize("type_tag,raw", [ + # Half-and-half spellings are the realistic accident (an IME left in + # full-width mode mid-token), and they must not parse either. + ("amount", "1２3억"), + ("amount", "amount('１0억')"), + ("amount", 'amount("１００","억")'), + ("number", "number('1２3')"), + ("number", "1２3"), + ("date", "date(20２0,1)"), + ("ordinal", "ordinal(1２)"), + ]) + def test_mixed_width_rejects(self, type_tag, raw): + assert lt.normalize(type_tag, raw) is None + + def test_full_width_no_longer_shares_a_scalar_with_ascii(self): + # The motivating bug in one line: same scalar, different stored string. + assert lt.normalize("amount", "100억") == 10000000000 + assert lt.normalize("amount", "１００억") is None + + @pytest.mark.parametrize("raw", ['amount(１００,억)', 'amount(１００,"억")']) + def test_canonical_amount_rejects(self, raw): + # Consequence, by design: merge_candidates' dedup key no longer collapses + # the bare and quoted full-width spellings into one row. Folding them is + # the silent rewrite this policy refuses. + assert lt.canonical_amount(raw) is None + + @pytest.mark.parametrize("raw", ['amount(１００,"억")', "date(２０２０,１)", "number(１２３)"]) + def test_humanize_returns_full_width_verbatim(self, raw): + # Unrecognized -> verbatim, the documented humanize fallback. + assert lt.humanize(raw) == raw + + +class TestFullWidthSurfaces: + """AC3 (#331): a rejected full-width value must reach a user-visible path at + least once. Under a relation declared typed, the projection loop warns on + stderr and loads the fact untyped — the same path any malformed literal takes. + + Residual symptom, deliberately NOT covered: a relation declared as an + attribute but NOT typed has no spec, so ``_project_typed_relations`` skips it + without warning and the full-width value surfaces nowhere.""" + + class _FakeSession: + """Records inserts; _project_typed_relations touches nothing else.""" + + def __init__(self): + self.inserts = [] + + def intern(self, value): + return 1 + + def insert(self, alias, payload): + self.inserts.append((alias, payload)) + + def test_projection_warns_and_loads_untyped(self, capsys): + import common + + specs = {"출시일": common.TypedRelSpec("date", "launch_date")} + rows = [ + {"subject": "제품", "relation": "출시일", "object": "date(２０２０,１)"}, + {"subject": "제품2", "relation": "출시일", "object": "date(2020,1)"}, + ] + session = self._FakeSession() + common._project_typed_relations(session, specs, rows) + err = capsys.readouterr().err + assert "date(２０２０,１)" in err + assert "does not parse as date" in err + # Only the ASCII row projects; the full-width fact still loads untyped. + assert session.inserts == [("launch_date", (1, 20200101))] + + +class TestAsciiDigitsUnchanged: + """Regression guard for the narrowing in #331: the ASCII forms that already + parsed must keep parsing, including the ones a careless narrowing would break + (a space-separated unit, a comma inside the number, and — the reason + ``re.ASCII`` is NOT used — a U+3000 ideographic space as separator whitespace). + These pin existing behaviour; they do not prove the fix.""" + + @pytest.mark.parametrize("type_tag,raw,expected", [ + ("amount", "100 억", 10000000000), + ("amount", "100억", 10000000000), + ("amount", 'amount(1,000,"억")', 100000000000), + ("amount", "1,000원", 1000), + ("date", "date(2020)", 20200101), + ("date", "date(2020,　1)", 20200101), # U+3000 stays whitespace + ("date", "2030.1", 20300101), + ("number", "1,000", 1000000), + ("number", "3.14", 3140), + ("ordinal", "제3호", 3), + ("ordinal", "3rd", 3), + ]) + def test_ascii_forms_still_parse(self, type_tag, raw, expected): + assert lt.normalize(type_tag, raw) == expected + + def test_ascii_canonical_amount_unchanged(self): + assert lt.canonical_amount("amount(1,000,억)") == 'amount(1000,"억")' + + def test_ascii_humanize_unchanged(self): + assert lt.humanize('amount(7,"억")') == "7억" + assert lt.humanize("date(2030,1,15)") == "2030-01-15" From 0c9fcd788aa00b3744471d36db2c7cdf323ea476 Mon Sep 17 00:00:00 2001 From: Seoyun Lee Date: Wed, 29 Jul 2026 11:37:18 +0900 Subject: [PATCH 2/9] =?UTF-8?q?docs:=20=EC=A0=84=EA=B0=81=20=EC=88=AB?= =?UTF-8?q?=EC=9E=90=20=EA=B1=B0=EB=B6=80=20=EC=A0=95=EC=B1=85=EA=B3=BC=20?= =?UTF-8?q?=EA=B8=B0=EC=A1=B4=20KB=20=EB=A7=88=EC=9D=B4=EA=B7=B8=EB=A0=88?= =?UTF-8?q?=EC=9D=B4=EC=85=98=EC=9D=84=20=EC=A0=81=EB=8A=94=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 타입 지정 리터럴은 ASCII 숫자만 파싱한다는 계약과, 전각을 조용히 접지 않는 이유(저장된 사실 문자열을 다시 쓰게 된다)를 ko/en 양쪽에 적는다. 마이그레이션 주의도 함께 적는다. 이 규칙 이전에 수집된 전각 값이 남아 있으면 check_conflicts 가 새로 exit 1 로 실패할 수 있다 — 경고가 아니라 게이트 실패다. 전에는 `１００억` 과 `100억` 이 같은 스칼라로 접혀 한 값이었지만 이제 전각 쪽이 원문 문자열로 남아, 같은 주어의 단일값 관계에서 두 값이 되기 때문이다. 해소는 소스를 ASCII 로 고쳐 재수집하는 것이다. 레포에 CHANGELOG 가 없어 이 문서가 유일한 안내 지점이다. 업스트림 #331 --- docs/reference/typed-relations.en.md | 16 ++++++++++++++++ docs/reference/typed-relations.md | 14 ++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/docs/reference/typed-relations.en.md b/docs/reference/typed-relations.en.md index c1d2a8cb..7d3c6373 100644 --- a/docs/reference/typed-relations.en.md +++ b/docs/reference/typed-relations.en.md @@ -50,5 +50,21 @@ to `2030-01`. A bare `2030` with no `date(…)` wrapper still does NOT parse as date — with neither a separator nor the wrapper it is indistinguishable from a plain number. +Digits must be **ASCII**. A value carrying full-width digits — `１００억`, +`date(２０２０,１)`, the half-and-half `1２3억` — does NOT parse as any of +date/number/ordinal/amount. It takes the ordinary "does not parse → load +untyped" path and surfaces as a `typed-relations: … does not parse as …` +warning. Full-width is not folded to ASCII silently, because folding would +rewrite the stored fact string — the fix is to correct the source to ASCII and +re-collect. Under a relation that is not declared typed the parsers never run at +all, so there the two spellings simply stay separate values. + +⚠️ **Migrating an existing KB.** If full-width values collected before this rule +are still in the KB, `tools/check_conflicts.py` may now exit **1** — a gate +failure, not a warning. `１００억` and `100억` used to fold onto the same scalar +and count as one value; the full-width one now keys on its raw string, so for the +same subject a single-valued relation sees two values. Correct the source of +those facts to ASCII and re-collect to clear it. + `factlog vocab` shows declared typed relations with a `[typed:]` tag (e.g. `[attribute, typed:date]`). diff --git a/docs/reference/typed-relations.md b/docs/reference/typed-relations.md index 2ac6fb5a..5fbbc29a 100644 --- a/docs/reference/typed-relations.md +++ b/docs/reference/typed-relations.md @@ -45,5 +45,19 @@ 날짜로 파싱되지 않습니다 — 구분자도 래퍼도 없으면 일반 수치와 구별할 수 없기 때문입니다. +숫자는 **ASCII 숫자만** 파싱됩니다. `１００억`, `date(２０２０,１)`, `1２3억` 처럼 +전각 숫자가 섞인 값은 date/number/ordinal/amount 어느 타입으로도 파싱되지 않고 +"파싱 실패 → 타입 없이 적재" 경로로 떨어져 `typed-relations: … does not parse as +…` 경고로 표면화됩니다. 전각을 ASCII 로 조용히 접지 않는 이유는, 그것이 저장된 +사실 문자열을 다시 쓰는 결정이기 때문입니다 — 해소는 소스를 ASCII 숫자로 고쳐 다시 +수집하는 것입니다. 타입 지정이 없는 관계는 파서를 타지 않으므로 두 표기가 그대로 +별개 값으로 남습니다. + +⚠️ **기존 KB 마이그레이션.** 이 규칙 이전에 수집된 전각 값이 KB 에 남아 있다면 +`tools/check_conflicts.py` 가 새로 **exit 1** 로 실패할 수 있습니다 — 경고가 아니라 +게이트 실패입니다. 전에는 `１００억` 과 `100억` 이 같은 스칼라로 접혀 한 값으로 +보였지만, 이제 전각 쪽은 원문 문자열 그대로 남아 같은 주어의 단일값 관계에서 두 +값이 되기 때문입니다. 해당 사실의 소스를 ASCII 로 고쳐 재수집하면 해소됩니다. + `factlog vocab` 은 선언된 타입 지정 관계에 `[typed:]` 태그를 붙여 보여 줍니다(예: `[attribute, typed:date]`). From 881efb51e2a490636563c78c1e8fb7c47d3607d5 Mon Sep 17 00:00:00 2001 From: Seoyun Lee Date: Wed, 29 Jul 2026 11:37:18 +0900 Subject: [PATCH 3/9] =?UTF-8?q?docs:=20entity=5Faudit=20=EC=9D=98=20?= =?UTF-8?q?=EB=A6=AC=ED=84=B0=EB=9F=B4=20=ED=83=90=EC=A7=80=EA=B0=80=20?= =?UTF-8?q?=EB=8A=90=EC=8A=A8=ED=95=9C=20=EA=B2=83=EC=9D=B4=20=EC=9D=98?= =?UTF-8?q?=EB=8F=84=EC=9E=84=EC=9D=84=20=EB=B0=9D=ED=9E=8C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _LITERAL_RE 는 냄새 탐지기라 literal_types 의 엄격한 ASCII 전용 계약과 일부러 다르다. 여기서는 `\d` 가 전각까지 잡아야, 파서가 거부한 전각 값이 audit 에는 남아 사람 눈에 띈다. 좁히면 그 값이 어느 경로에서도 안 보이게 된다. 정규식은 그대로 두고 주석만 남긴다. 업스트림 #331 --- tools/entity_audit.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/entity_audit.py b/tools/entity_audit.py index d29d95c9..b0814d7e 100755 --- a/tools/entity_audit.py +++ b/tools/entity_audit.py @@ -57,6 +57,8 @@ # 100억, 제3호). Advisory only — a human confirms before declaring the relation; # a few false positives (e.g. a named concept like '4차 산업혁명') are acceptable # in exchange for not missing the motivating value forms. +# Deliberately looser than literal_types' ASCII-only contract: `\d` still matches +# full-width digits here, so a rejected value stays visible to the audit (#331). _LITERAL_RE = re.compile( r"^\d{4}[.\-/]\d{1,2}([.\-/]\d{1,2})?$" # date r"|^\d[\d,]*(\.\d+)?$" # number / comma / decimal From b0816b2b0df565cbe902ee787f18f4224cafb159 Mon Sep 17 00:00:00 2001 From: Seoyun Lee Date: Wed, 29 Jul 2026 11:38:58 +0900 Subject: [PATCH 4/9] =?UTF-8?q?test:=20=EC=A0=84=EA=B0=81=20=ED=98=BC?= =?UTF-8?q?=ED=95=A9=20=EC=BC=80=EC=9D=B4=EC=8A=A4=EB=A5=BC=20=EC=8B=A4?= =?UTF-8?q?=EC=A0=9C=EB=A1=9C=20baseline=20=EC=97=90=EC=84=9C=20=EA=B9=A8?= =?UTF-8?q?=EC=A7=80=EB=8A=94=20=EC=9E=85=EB=A0=A5=EC=9C=BC=EB=A1=9C=20?= =?UTF-8?q?=EB=B0=94=EA=BE=BC=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `amount('１0억')` 과 `number('1２3')` 은 홑따옴표를 쓴다. compound 문법에는 홑따옴표가 없으므로 이 둘은 전각과 무관하게 baseline 에서도 이미 None 이었다 — 즉 전각 거부를 증명하지 못하는 vacuous 케이스였다. baseline 에서 각각 1000000000 / 123000 을 돌려주던 `amount(１0,"억")` 과 `number("1２3")` 로 바꾼다. 이제 두 케이스 모두 이 수정이 없으면 실패한다. 업스트림 #331 --- tests/unit/test_literal_types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_literal_types.py b/tests/unit/test_literal_types.py index 38a3b000..8c65fdd6 100644 --- a/tests/unit/test_literal_types.py +++ b/tests/unit/test_literal_types.py @@ -419,9 +419,9 @@ def test_parse_amount_rejects(self): # Half-and-half spellings are the realistic accident (an IME left in # full-width mode mid-token), and they must not parse either. ("amount", "1２3억"), - ("amount", "amount('１0억')"), + ("amount", 'amount(１0,"억")'), ("amount", 'amount("１００","억")'), - ("number", "number('1２3')"), + ("number", 'number("1２3")'), ("number", "1２3"), ("date", "date(20２0,1)"), ("ordinal", "ordinal(1２)"), From 4d02c8f775e88994efa0bb80c3eb85c1f9a0e4a6 Mon Sep 17 00:00:00 2001 From: Seoyun Lee Date: Wed, 29 Jul 2026 12:00:12 +0900 Subject: [PATCH 5/9] =?UTF-8?q?docs:=20=EC=A0=84=EA=B0=81=20=EA=B1=B0?= =?UTF-8?q?=EB=B6=80=EA=B0=80=20=EA=B2=8C=EC=9D=B4=ED=8A=B8=EB=A5=BC=20?= =?UTF-8?q?=EB=92=A4=EC=A7=91=EB=8A=94=20=EB=84=A4=20=EB=B2=88=EC=A7=B8=20?= =?UTF-8?q?=EA=B2=BD=EB=A1=9C=EB=A5=BC=20docstring=20=EC=97=90=20=EC=A0=81?= =?UTF-8?q?=EB=8A=94=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 모듈 밖 소비자를 전수로 다시 세니 셋이 아니라 넷이다. 빠져 있던 check_conflicts._group_key 가 나머지 셋보다 무거운 경로다 — 나머지는 "데이터 재작성 없음" 이 요점이지만, 이쪽은 통과하던 게이트가 exit 1 로 뒤집힌다. 전각 객체가 ("raw", obj) 로 떨어져 ASCII 쌍둥이의 ("scalar", …) 와 갈리고, 단일값 관계에서 두 값이 되기 때문이다. 사용자 문서에는 적혀 있었지만 코드를 읽는 사람은 놓칠 자리였다. merge dedup 서술도 결과를 절반만 적고 있었다. 안 합쳐지는 것에 더해, 새로 머지되는 전각 행의 저장 문자열 자체가 따옴표 정준형으로 재작성되지 않고 원문 그대로 남는다(merge_candidates 는 canonical_amount 가 None 이 아닐 때만 객체를 덮어쓴다). 기존 KB 행이 불변이라는 주장은 그대로다. 표면화 경로도 정확히 다시 적었다. 이슈 AC3 이 "경고 또는 audit" 이라 쓴 대로 경로는 둘이다 — typed 선언된 관계는 투영 경고, attribute 로 선언되지 **않은** 관계는 entity_audit 의 literal suspects(_LITERAL_RE 가 일부러 느슨해 전각을 계속 잡는다). 진짜 구멍은 그 사이, attribute 선언은 있고 typed 는 아닌 관계 하나뿐이다. 이전 문구는 이 구멍을 실제보다 넓게 적고 있었다. 업스트림 #331 --- factlog/literal_types.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/factlog/literal_types.py b/factlog/literal_types.py index c42fabae..3c23f238 100644 --- a/factlog/literal_types.py +++ b/factlog/literal_types.py @@ -27,13 +27,21 @@ surfaces as a typed-projection warning instead of merging silently. Rewriting the stored string would have been the silent fold this repo consistently refuses. -Three call sites outside this module change behaviour as a consequence. All three -are intended, and none of them rewrites data: +Four call sites outside this module change behaviour as a consequence. All four +are intended, and none of them rewrites data already stored: - ``tools/merge_candidates.py`` dedup key — ``canonical_amount`` returns ``None`` for a full-width term, so ``amount(１００,억)`` and ``amount(１００,"억")`` stay two rows instead of collapsing into one. Collapsing them is exactly the fold - this policy rejects. + this policy rejects. A newly merged full-width row also keeps its **source + string verbatim** instead of being rewritten to the quoted canonical form; + rows already in the KB are untouched either way. +- ``tools/check_conflicts.py`` ``_group_key`` — a full-width object no longer + normalizes to a scalar, so it keys as ``("raw", obj)`` while its ASCII twin + keys as ``("scalar", …)``. Under a single-valued relation those are **two + values**, so a KB that used to pass can now report a conflict and the command + exits **1**. This is the one consequence that flips a green gate red — see the + migration note in ``docs/reference/typed-relations.md``. - ``common._canonical_value`` — a fact already stored as ``amount(１００,"억")`` in an existing KB is no longer canonicalised, so a query written as ``amount(１００,억)`` misses it. Intended: under this policy ``amount(１００,억)`` @@ -42,12 +50,18 @@ - ``humanize`` — a full-width compound term is now returned verbatim rather than rendered as ``１００억``. -Residual symptom, NOT fixed here: the warning only fires for relations **declared -typed**. ``common._project_typed_relations`` skips a relation with no typed spec -without warning, so a full-width value under a relation that is declared as an -attribute but not typed still surfaces nowhere — ``tools/entity_audit.py`` lists -``declared_literals`` as an unflagged sorted list, and ``100억`` / ``１００억`` -sort far apart in it. +A rejected value has two user-visible paths, matching the two the issue asked +for. Under a relation declared **typed**, ``common._project_typed_relations`` +warns and loads the fact untyped. Under a relation that is **not declared** as an +attribute at all, ``tools/entity_audit.py`` reports it as a *literal suspect* — +its ``_LITERAL_RE`` is deliberately looser than this module and still matches +full-width digits, so narrowing that detector would close this path (a pinning +test guards the divergence). + +Residual symptom, NOT fixed here: a relation declared as an **attribute but not +typed** falls between the two. It has no spec, so the projection loop skips it +without warning, and its objects land in ``entity_audit``'s ``declared_literals`` +— an unflagged sorted list in which ``100억`` and ``１００억`` sort far apart. """ from __future__ import annotations From 392bca490623aa783803f2b9de6cb5ceba8da438 Mon Sep 17 00:00:00 2001 From: Seoyun Lee Date: Wed, 29 Jul 2026 12:00:12 +0900 Subject: [PATCH 6/9] =?UTF-8?q?test:=20entity=5Faudit=20=EC=99=80=20litera?= =?UTF-8?q?l=5Ftypes=20=EC=9D=98=20=EC=9D=98=EB=8F=84=EB=90=9C=20=EB=B0=9C?= =?UTF-8?q?=EC=82=B0=EC=9D=84=20=EB=B6=99=EB=93=A0=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestLiteralReConsistency 는 "must not drift" 를 표방하면서 ASCII 8건만 핀했다. 그래서 누가 일관성을 이유로 _LITERAL_RE 를 [0-9] 로 좁혀도 아무 테스트도 깨지지 않는다 — 그런데 그 좁힘은 표면화 경로 하나를 통째로 닫는다. attribute 로 선언되지 않은 관계의 전각 값은 _LITERAL_RE 가 잡아 주어야 literal suspects 로 뜬다. _LITERAL_RE.match 가 True 인 것과 normalize 가 None 인 것을 한 테스트에서 함께 단언해, 느슨함과 엄격함이 **일부러** 다르다는 계약을 테스트가 들고 있게 한다. 뮤턴트 확인: _LITERAL_RE 의 \d 를 [0-9] 로 좁히면 이 4건이 실패하고 나머지는 전부 통과한다(= 이전에는 이 계약을 아무도 붙들지 않았다는 뜻이기도 하다). 업스트림 #331 --- tests/unit/test_literal_types.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/unit/test_literal_types.py b/tests/unit/test_literal_types.py index 8c65fdd6..0daf7c54 100644 --- a/tests/unit/test_literal_types.py +++ b/tests/unit/test_literal_types.py @@ -378,6 +378,26 @@ def test_detected_and_parsed(self, raw, type_tag, expected): assert _LITERAL_RE.match(raw), f"entity_audit no longer detects {raw!r}" assert lt.normalize(type_tag, raw) == expected + @pytest.mark.parametrize("raw,type_tag", [ + ("１００억", "amount"), + ("２０２６", "number"), + ("제３호", "ordinal"), + ("２０３０.１", "date"), + ]) + def test_full_width_divergence_is_intended(self, raw, type_tag): + """The two must NOT be made consistent for full-width digits (#331). + + entity_audit's detector is a loose smell test; these normalizers are a + strict ASCII-only contract. That divergence is load-bearing: a relation + that is not declared an attribute puts a matching object into + ``literal_suspects``, which is the second user-visible path a rejected + full-width value takes. Narrowing ``_LITERAL_RE`` to ``[0-9]`` "for + consistency" would silently close it — so pin both halves together. + """ + from entity_audit import _LITERAL_RE + assert _LITERAL_RE.match(raw), f"entity_audit must still SEE {raw!r}" + assert lt.normalize(type_tag, raw) is None, f"{raw!r} must NOT parse" + class TestFullWidthDigitsRejected: """ASCII-only digits (#331). Python's ``\\d`` covers the whole Unicode ``Nd`` From afb6cefa49423cf0bc659bf1991a87654b6bcba5 Mon Sep 17 00:00:00 2001 From: Seoyun Lee Date: Wed, 29 Jul 2026 12:00:13 +0900 Subject: [PATCH 7/9] =?UTF-8?q?test:=20=EC=A0=84=EA=B0=81=20=EA=B0=92?= =?UTF-8?q?=EC=9D=B4=20raw=20=ED=82=A4=EB=A1=9C=20=EB=96=A8=EC=96=B4?= =?UTF-8?q?=EC=A0=B8=20conflict=20=EA=B0=80=20=EB=9C=A8=EB=8A=94=20?= =?UTF-8?q?=EA=B2=83=EC=9D=84=20=EA=B3=A0=EC=A0=95=ED=95=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 문서가 경고한 check_conflicts exit 1 은 이번 변경이 사용자 파이프라인을 실제로 깨뜨릴 수 있는 유일한 동작인데, 문서 문장 하나에만 기대고 있었다. 나중에 _group_key 에 전각 폴백이 들어가 조용히 되돌아가도 아무것도 실패하지 않는다. _group_key('１００억') == ("raw", …) 와 ASCII 쌍둥이의 ("scalar", …) 가 갈리는 것, 그래서 단일값 관계에서 두 값이 되어 CONFLICT 가 뜨는 것을 amount·date 로 고정한다. 전각 한 표기만 있는 KB 는 여전히 conflict 가 아니라는 것도 함께 — raw 로의 강등이 없던 모순을 만들어내면 안 된다. check_conflicts.py 는 읽기만 했고 수정하지 않았다. 뮤턴트 확인: _group_key 가 normalize 전에 NFKC 로 접게 만들면 이 중 3건이 실패한다(전각 단독 케이스는 폴드 하에서도 초록이라 통과 — 의도대로다). 업스트림 #331 --- tests/unit/test_conflict_scalar.py | 48 ++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/unit/test_conflict_scalar.py b/tests/unit/test_conflict_scalar.py index 182dc274..d31073d5 100644 --- a/tests/unit/test_conflict_scalar.py +++ b/tests/unit/test_conflict_scalar.py @@ -155,6 +155,54 @@ def test_raw_integer_string_does_not_collide_with_scalar_key(self): assert conflicts[("갑사", "매출")] == ["540000000000", 'amount(5400,"억")'] +class TestFullWidthDegradesToRaw: + """The migration risk of the ASCII-only digit policy (#331), pinned. + + Full-width digits stopped parsing, so a full-width object degrades to the + ``("raw", obj)`` key while its ASCII twin keys as ``("scalar", …)``. Under a + single-valued relation that is **two values**, so a KB carrying both + spellings now reports a CONFLICT and ``main`` returns 1 where it used to + return 0. That gate flip is the one user-visible break this policy causes and + it is intended: the resolution is to correct the source to ASCII and + re-collect, NOT to fold full-width to ASCII in ``_group_key``. These tests + fail if such a fold is ever added, which is the point of having them — + ``docs/reference/typed-relations.md`` documents the same contract. + """ + + def test_full_width_amount_keys_raw_not_scalar(self): + assert check_conflicts._group_key("100억", _AMOUNT_SPEC) == ("scalar", 10000000000) + assert check_conflicts._group_key("１００억", _AMOUNT_SPEC) == ("raw", "１００억") + + def test_full_width_and_ascii_twin_conflict(self): + # The same value in two spellings is now two values -> CONFLICT (exit 1). + facts = [ + _fact("갑사", "매출", "100억"), + _fact("갑사", "매출", "１００억"), + ] + conflicts = check_conflicts.detect_conflicts(facts, {"매출"}, _TYPED_AMOUNT) + # Reported values keep the original object strings (provenance), so a + # reviewer can see which row carries the full-width digits. + assert conflicts[("갑사", "매출")] == ["100억", "１００억"] + + def test_full_width_date_and_ascii_twin_conflict(self): + facts = [ + _fact("기서비스", "출시", "2030.1"), + _fact("기서비스", "출시", "２０３０.１"), + ] + conflicts = check_conflicts.detect_conflicts(facts, {"출시"}, _TYPED_DATE) + assert conflicts[("기서비스", "출시")] == ["2030.1", "２０３０.１"] + + def test_full_width_alone_is_not_a_conflict(self): + # Degrading to raw must not invent a conflict out of one repeated value: + # only a KB that carries BOTH spellings flips from green to red. + facts = [ + _fact("갑사", "매출", "１００억"), + _fact("갑사", "매출", "１００억"), + ] + conflicts = check_conflicts.detect_conflicts(facts, {"매출"}, _TYPED_AMOUNT) + assert conflicts == {} + + class TestMultiValuedNotFlagged: def test_relation_not_single_valued_never_conflicts(self): # Not declared single-valued -> ignored entirely, even with typed spec. From 538bcad56507a1db65daddbd02df6bb4b0a20e2d Mon Sep 17 00:00:00 2001 From: Seoyun Lee Date: Wed, 29 Jul 2026 12:13:24 +0900 Subject: [PATCH 8/9] =?UTF-8?q?docs:=20=EB=A7=88=EC=9D=B4=EA=B7=B8?= =?UTF-8?q?=EB=A0=88=EC=9D=B4=EC=85=98=20=EC=A0=88=EC=97=90=20=EC=A1=B0?= =?UTF-8?q?=EC=9A=A9=ED=95=9C=20=EC=A7=88=EC=9D=98=20=EB=AF=B8=EC=8A=A4?= =?UTF-8?q?=EB=A5=BC=20=EC=B6=94=EA=B0=80=ED=95=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 마이그레이션 절에는 check_conflicts exit 1 만 있었다. 그런데 이슈 본문이 문제로 지목한 것은 오히려 조용한 쪽이다 — "검증된 부정(engine says no)과 구분되지 않는다". 시끄러운 실패보다 사용자 대면 문서에 들어갈 자격이 더 있다. 기존 KB 에 전각 amount compound term 이 저장돼 있으면 따옴표 없이 쓴 질의가 이제 조용히 미스한다. 전각 항이 더는 유효한 amount 가 아니라서 저장본과 같은 정준형으로 접히지 않기 때문이다. 실측: before: canon('amount(１００,억)') = 'amount(１００,"억")' -> 저장본과 match after : canon('amount(１００,억)') = 'amount(１００,억)' -> match 안 됨 해소는 두 경우 모두 동일해서 마지막 문장을 공유하도록 문단을 다시 묶었다. ko/en 같은 커밋. humanize 불릿만 소비자 파일명을 안 밝히고 있어 나머지 셋과 통일했다 (tools/ask_router.py — pretty 가 원문과 같아지므로 "(= …)" 접미가 붙지 않는다). 업스트림 #331 --- docs/reference/typed-relations.en.md | 13 +++++++++++-- docs/reference/typed-relations.md | 10 +++++++++- factlog/literal_types.py | 5 +++-- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/docs/reference/typed-relations.en.md b/docs/reference/typed-relations.en.md index 7d3c6373..5fa3ab3c 100644 --- a/docs/reference/typed-relations.en.md +++ b/docs/reference/typed-relations.en.md @@ -63,8 +63,17 @@ all, so there the two spellings simply stay separate values. are still in the KB, `tools/check_conflicts.py` may now exit **1** — a gate failure, not a warning. `１００억` and `100억` used to fold onto the same scalar and count as one value; the full-width one now keys on its raw string, so for the -same subject a single-valued relation sees two values. Correct the source of -those facts to ASCII and re-collect to clear it. +same subject a single-valued relation sees two values. + +That gate failure is loud; there is also a **quiet** one. If an existing KB holds +a full-width amount compound term (`amount(１００,"억")`), a query written without +the quotes — `amount(１００,억)` — now **misses silently**, because a full-width +term is no longer a valid amount and so no longer folds to the same canonical +form as the stored value. That miss is indistinguishable from an engine-verified +"no such fact", which makes it harder to notice than the failure. + +Both cases clear the same way: correct the source of those facts to ASCII and +re-collect. `factlog vocab` shows declared typed relations with a `[typed:]` tag (e.g. `[attribute, typed:date]`). diff --git a/docs/reference/typed-relations.md b/docs/reference/typed-relations.md index 5fbbc29a..2163521c 100644 --- a/docs/reference/typed-relations.md +++ b/docs/reference/typed-relations.md @@ -57,7 +57,15 @@ `tools/check_conflicts.py` 가 새로 **exit 1** 로 실패할 수 있습니다 — 경고가 아니라 게이트 실패입니다. 전에는 `１００억` 과 `100억` 이 같은 스칼라로 접혀 한 값으로 보였지만, 이제 전각 쪽은 원문 문자열 그대로 남아 같은 주어의 단일값 관계에서 두 -값이 되기 때문입니다. 해당 사실의 소스를 ASCII 로 고쳐 재수집하면 해소됩니다. +값이 되기 때문입니다. + +게이트 실패는 시끄럽지만, **조용한** 쪽도 하나 있습니다. 기존 KB 에 전각 amount +compound term(`amount(１００,"억")`)이 저장돼 있다면, 따옴표 없이 쓴 +`amount(１００,억)` 질의가 이제 **조용히 미스**합니다 — 전각 항은 더 이상 유효한 +amount 가 아니라서 저장본과 같은 정준형으로 접히지 않기 때문입니다. 이 미스는 +엔진이 확인한 "없음" 과 구분되지 않으므로 실패보다 알아채기 어렵습니다. + +두 경우 모두 해당 사실의 소스를 ASCII 로 고쳐 재수집하면 해소됩니다. `factlog vocab` 은 선언된 타입 지정 관계에 `[typed:]` 태그를 붙여 보여 줍니다(예: `[attribute, typed:date]`). diff --git a/factlog/literal_types.py b/factlog/literal_types.py index 3c23f238..3e056862 100644 --- a/factlog/literal_types.py +++ b/factlog/literal_types.py @@ -47,8 +47,9 @@ ``amount(１００,억)`` misses it. Intended: under this policy ``amount(１００,억)`` is not a valid amount term at all, so there is no canonical form to map it to. The fix is to correct the source to ASCII and re-collect, not to fold here. -- ``humanize`` — a full-width compound term is now returned verbatim rather than - rendered as ``１００억``. +- ``tools/ask_router.py`` answer annotation — ``humanize`` returns a full-width + compound term verbatim rather than rendering it as ``１００억``, so the display + suffix (``… (= 100억)``) is simply omitted for such a row. A rejected value has two user-visible paths, matching the two the issue asked for. Under a relation declared **typed**, ``common._project_typed_relations`` From 8eb9e0ab7401f22e4aff38cf2d8d4b0f64aa850e Mon Sep 17 00:00:00 2001 From: Seoyun Lee Date: Wed, 29 Jul 2026 12:15:34 +0900 Subject: [PATCH 9/9] =?UTF-8?q?test:=20=EC=98=81=EB=AC=B8=20=EC=84=9C?= =?UTF-8?q?=EC=88=98=20=EC=A0=95=EA=B7=9C=EC=8B=9D=EC=9D=98=20=EC=A0=84?= =?UTF-8?q?=EA=B0=81=20=EA=B1=B0=EB=B6=80=EB=A5=BC=20=EA=B3=A0=EC=A0=95?= =?UTF-8?q?=ED=95=9C=EB=8B=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 정규식 9개를 하나씩 \d 로 되돌려 스위트를 돌리면 _ORDINAL_EN_RE 만 뮤턴트가 살아남았다. 동작 자체는 고쳐져 있었지만(HEAD 에서 '３rd' -> None, baseline 에서는 3) 그것을 고정하는 테스트가 없었다. 신규 ordinal 커버리지가 ordinal(３)·３위· 제３호 뿐이고 영문 서수형이 빠져 있었던 탓이다. 미래 리팩터가 이 한 줄을 조용히 되돌려도 아무것도 안 깨지는 상태였다. 전각 '３rd' 와 혼합폭 '1２th' 을 각각 거부 목록에 넣는다. 업스트림 #331 --- tests/unit/test_literal_types.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/test_literal_types.py b/tests/unit/test_literal_types.py index 0daf7c54..e73064db 100644 --- a/tests/unit/test_literal_types.py +++ b/tests/unit/test_literal_types.py @@ -417,6 +417,7 @@ class TestFullWidthDigitsRejected: ("ordinal", "ordinal(３)"), ("ordinal", "３위"), ("ordinal", "제３호"), + ("ordinal", "３rd"), # the English ordinal form has its own regex ("amount", "１００억"), ("amount", 'amount(１００,"억")'), ]) @@ -445,6 +446,7 @@ def test_parse_amount_rejects(self): ("number", "1２3"), ("date", "date(20２0,1)"), ("ordinal", "ordinal(1２)"), + ("ordinal", "1２th"), ]) def test_mixed_width_rejects(self, type_tag, raw): assert lt.normalize(type_tag, raw) is None