From 6c5e63570fc1889c414d27f2834b84d69e6fb638 Mon Sep 17 00:00:00 2001 From: Gordon YUEN Date: Tue, 28 Jul 2026 23:05:23 +0800 Subject: [PATCH] feat: normalize Traditional Chinese to Simplified for BM25 cross-match MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add OpenCC t2s (Traditional→Simplified) normalization before jieba tokenization in both _tokenize_cjk_segment and tokenize2stw_remove. This is the single chokepoint used by both ingest (build_content_search_text, build_path_search_text via tokenize_contents_for_retrieval) and query (lexical_ranker via tokenize_for_retrieval), so symmetry is automatic. Without this, '營收' (Traditional) and '营收' (Simplified) produce different BM25 tokens and never cross-match. With OpenCC, both normalize to '营收' before jieba cuts, so a 繁体 document is findable by a 简体 query and vice versa. OpenCC only converts CJK characters — English, numbers, and punctuation are untouched. PostgreSQL 'simple' tsvector config is unchanged; the normalization happens entirely in the Python application layer before text reaches the database. Existing content_search_tsv rows are pre-normalization and require re-ingest to benefit fully. No migration needed — new ingestions produce normalized tokens going forward. --- packages/shared-python/pyproject.toml | 1 + .../shared/tests/test_text_utils_cjk.py | 98 +++++++++++++++++++ .../shared-python/shared/utils/text_utils.py | 31 +++++- uv.lock | 15 ++- 4 files changed, 139 insertions(+), 6 deletions(-) create mode 100644 packages/shared-python/shared/tests/test_text_utils_cjk.py diff --git a/packages/shared-python/pyproject.toml b/packages/shared-python/pyproject.toml index 260d831d4..5fd8e393f 100644 --- a/packages/shared-python/pyproject.toml +++ b/packages/shared-python/pyproject.toml @@ -68,6 +68,7 @@ dependencies = [ # Text processing "blingfire>=0.1.8", "syntok>=1.4.4", + "opencc-python-reimplemented>=0.1.7", # Image processing (used by ZipResultService) "pillow==12.2.0", diff --git a/packages/shared-python/shared/tests/test_text_utils_cjk.py b/packages/shared-python/shared/tests/test_text_utils_cjk.py new file mode 100644 index 000000000..06e4af381 --- /dev/null +++ b/packages/shared-python/shared/tests/test_text_utils_cjk.py @@ -0,0 +1,98 @@ +"""Tests for CJK text normalization in shared.utils.text_utils. + +Verifies that Traditional Chinese is normalized to Simplified before +jieba tokenization, so BM25 cross-matches 繁/简 variants (e.g. +'營收' ↔ '营收'). English, numbers, and punctuation must be untouched. +""" + +from __future__ import annotations + +import os + +os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") +os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test") +os.environ.setdefault("S3_BUCKET_NAME", "test-uploads") +os.environ.setdefault("S3_ACCESS_KEY_ID", "test") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test") +os.environ.setdefault("S3_TEMP_PATH", "/tmp") + +from shared.utils.text_utils import ( # noqa: E402 + _normalize_cjk_to_simplified, + tokenize_for_retrieval, + tokenize_contents_for_retrieval, + tokenize2stw_remove, +) + + +def test_normalize_traditional_to_simplified() -> None: + assert _normalize_cjk_to_simplified("營收") == "营收" + assert _normalize_cjk_to_simplified("電腦") == "电脑" + assert _normalize_cjk_to_simplified("計算機") == "计算机" + + +def test_normalize_leaves_english_unchanged() -> None: + assert _normalize_cjk_to_simplified("NVIDIA FY27 revenue") == "NVIDIA FY27 revenue" + assert _normalize_cjk_to_simplified("hello world 123") == "hello world 123" + + +def test_normalize_leaves_punctuation_unchanged() -> None: + assert _normalize_cjk_to_simplified("营收，同比增长。") == "营收，同比增长。" + + +def test_normalize_handles_empty() -> None: + assert _normalize_cjk_to_simplified("") == "" + + +def test_normalize_mixed_traditional_simplified() -> None: + # Mixed 繁/简 in the same string — both should normalize to simplified + assert _normalize_cjk_to_simplified("營收增长") == "营收增长" + + +def test_tokenize_for_retrieval_normalizes_traditional() -> None: + traditional_tokens = tokenize_for_retrieval("營收同比增長") + simplified_tokens = tokenize_for_retrieval("营收同比增长") + assert traditional_tokens == simplified_tokens + + +def test_tokenize_for_retrieval_normalizes_mixed() -> None: + mixed_tokens = tokenize_for_retrieval("營收增长 NVIDIA") + simplified_tokens = tokenize_for_retrieval("营收增长 NVIDIA") + assert mixed_tokens == simplified_tokens + + +def test_tokenize_for_retrieval_leaves_english_unchanged() -> None: + tokens = tokenize_for_retrieval("NVIDIA quarterly revenue") + lowered = [t.lower() for t in tokens] + assert "nvidia" in lowered + assert "quarterly" in lowered + + +def test_tokenize_contents_for_retrieval_normalizes_traditional() -> None: + traditional = tokenize_contents_for_retrieval(["營收同比增長"], link_char=" ") + simplified = tokenize_contents_for_retrieval(["营收同比增长"], link_char=" ") + assert traditional == simplified + + +def test_tokenize2stw_remove_normalizes_traditional() -> None: + traditional = tokenize2stw_remove(["營收同比增長"]) + simplified = tokenize2stw_remove(["营收同比增长"]) + assert traditional == simplified + + +def test_traditional_and_simplified_produce_same_tokens_cross_match() -> None: + # The core regression: a doc ingested with 繁体 should be findable + # by a query in 简体, and vice versa. + ingest_tokens = tokenize_contents_for_retrieval( + ["台積電營收創歷史新高"], link_char=" " + ) + query_tokens_traditional = tokenize_for_retrieval("台積電營收") + query_tokens_simplified = tokenize_for_retrieval("台积电营收") + + # Both query variants should produce tokens that appear in the ingest tokens + ingest_token_set = set(ingest_tokens[0].split()) if ingest_tokens else set() + for query_tokens in [query_tokens_traditional, query_tokens_simplified]: + for token in query_tokens: + if token in ingest_token_set: + break + else: + assert False, f"Query token {query_tokens} not found in ingest tokens {ingest_token_set}" diff --git a/packages/shared-python/shared/utils/text_utils.py b/packages/shared-python/shared/utils/text_utils.py index dd61bfdd2..7bf7fa53a 100644 --- a/packages/shared-python/shared/utils/text_utils.py +++ b/packages/shared-python/shared/utils/text_utils.py @@ -18,6 +18,27 @@ _blingfire_text_to_words = None +try: + from opencc import OpenCC as _OpenCC + + _T2S_CONVERTER = _OpenCC("t2s") +except (ImportError, OSError): + _T2S_CONVERTER = None + + +def _normalize_cjk_to_simplified(text: str) -> str: + """Normalize Traditional Chinese characters to Simplified. + + Applied before jieba tokenization so that 繁/简 variants produce the same + tokens on both ingest and query sides, letting BM25 cross-match. OpenCC + only converts CJK characters — English, numbers, and punctuation are + untouched. Falls through unchanged when ``opencc`` is not installed. + """ + if _T2S_CONVERTER is None or not text: + return text + return _T2S_CONVERTER.convert(text) + + class _JiebaModule(Protocol): def lcut(self, sentence: str) -> list[str]: ... def cut(self, sentence: str) -> list[str]: ... @@ -200,10 +221,11 @@ def _tokenize_english_segment(text: str) -> list[str]: def _tokenize_cjk_segment(text: str) -> list[str]: if not text.strip(): return [] + normalized = _normalize_cjk_to_simplified(text) try: - return list(_jieba.lcut(text)) + return list(_jieba.lcut(normalized)) except AttributeError: - return list(_jieba.cut(text)) + return list(_jieba.cut(normalized)) def _resolve_retrieval_stopwords( @@ -302,10 +324,11 @@ def tokenize2stw_remove(contents: List[str], stopwords: Optional[List[str]] = No for content in contents: # Pre-clean: remove IMAGE_/TABLE_ markers and reference labels content = _CHUNK_MARKER_RE.sub('', content) + normalized = _normalize_cjk_to_simplified(content) try: - raw_tokens = _jieba.lcut(content) + raw_tokens = _jieba.lcut(normalized) except AttributeError: - raw_tokens = list(_jieba.cut(content)) + raw_tokens = list(_jieba.cut(normalized)) # Filter: keep only tokens with meaningful characters (Chinese/English/numbers) tokens = [t for t in raw_tokens if _is_meaningful_token(t)] # Remove stopwords diff --git a/uv.lock b/uv.lock index 4f6aadc56..7e254ceca 100644 --- a/uv.lock +++ b/uv.lock @@ -1493,6 +1493,7 @@ dependencies = [ { name = "kombu" }, { name = "loguru" }, { name = "openai" }, + { name = "opencc-python-reimplemented" }, { name = "pgvector" }, { name = "pillow" }, { name = "posthog" }, @@ -1539,6 +1540,7 @@ requires-dist = [ { name = "kombu", specifier = "==5.4.0" }, { name = "loguru", specifier = "==0.7.3" }, { name = "openai", specifier = ">=1.0.0" }, + { name = "opencc-python-reimplemented", specifier = ">=0.1.7" }, { name = "pgvector", specifier = "==0.2.4" }, { name = "pillow", specifier = "==12.2.0" }, { name = "posthog", specifier = "==7.18.1" }, @@ -1580,8 +1582,8 @@ dependencies = [ { name = "knowhere-shared" }, { name = "logfire", extra = ["celery", "fastapi", "httpx", "sqlalchemy"] }, { name = "lxml" }, - { name = "markitdown" }, { name = "markdownify" }, + { name = "markitdown" }, { name = "numpy" }, { name = "openai" }, { name = "openpyxl" }, @@ -1618,8 +1620,8 @@ requires-dist = [ { name = "knowhere-shared", editable = "packages/shared-python" }, { name = "logfire", extras = ["celery", "fastapi", "httpx", "sqlalchemy"], specifier = ">=4.25.0" }, { name = "lxml", specifier = "==6.1.0" }, - { name = "markitdown", specifier = "==0.1.2" }, { name = "markdownify", specifier = "==1.2.2" }, + { name = "markitdown", specifier = "==0.1.2" }, { name = "numpy", specifier = "==2.2.6" }, { name = "openai", specifier = "==1.93.3" }, { name = "openpyxl", specifier = "==3.1.2" }, @@ -2334,6 +2336,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/b9/0df6351b25c6bd494c534d2a8191dc9460fb5bb09c88b1427775d49fde05/openai-1.93.3-py3-none-any.whl", hash = "sha256:41aaa7594c7d141b46eed0a58dcd75d20edcc809fdd2c931ecbb4957dc98a892", size = 755132, upload-time = "2025-07-09T14:08:25.533Z" }, ] +[[package]] +name = "opencc-python-reimplemented" +version = "0.1.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8d/6d/c6f37eed651dd6b752e50f80a93396cdaa42a6acc6ce05ad7452303ea511/opencc-python-reimplemented-0.1.7.tar.gz", hash = "sha256:4f777ea3461a25257a7b876112cfa90bb6acabc6dfb843bf4d11266e43579dee", size = 482566, upload-time = "2023-02-11T03:58:42.25Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/6b/055b7806f320cc8f2cdf23c5f70221c0dc1683fca9ffaf76dfc2ad4b91b6/opencc_python_reimplemented-0.1.7-py2.py3-none-any.whl", hash = "sha256:41b3b92943c7bed291f448e9c7fad4b577c8c2eae30fcfe5a74edf8818493aa6", size = 481813, upload-time = "2023-02-11T03:58:39.66Z" }, +] + [[package]] name = "openpyxl" version = "3.1.2"