feat: normalize Traditional Chinese to Simplified for BM25 cross-match

packages/shared-python/pyproject.toml

@@ -68,6 +68,7 @@ dependencies = [
     # Text processing
     "blingfire>=0.1.8",
     "syntok>=1.4.4",
+    "opencc-python-reimplemented>=0.1.7",
 
     # Image processing (used by ZipResultService)
     "pillow==12.2.0",

packages/shared-python/shared/tests/test_text_utils_cjk.py

@@ -0,0 +1,98 @@
+"""Tests for CJK text normalization in shared.utils.text_utils.
+
+Verifies that Traditional Chinese is normalized to Simplified before
+jieba tokenization, so BM25 cross-matches 繁/简 variants (e.g.
+'營收' ↔ '营收'). English, numbers, and punctuation must be untouched.
+"""
+
+from __future__ import annotations
+
+import os
+
+os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test")
+os.environ.setdefault("TMP_PATH", "/tmp/knowhere-test")
+os.environ.setdefault("S3_BUCKET_NAME", "test-uploads")
+os.environ.setdefault("S3_ACCESS_KEY_ID", "test")
+os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test")
+os.environ.setdefault("S3_TEMP_PATH", "/tmp")
+
+from shared.utils.text_utils import (  # noqa: E402
+    _normalize_cjk_to_simplified,
+    tokenize_for_retrieval,
+    tokenize_contents_for_retrieval,
+    tokenize2stw_remove,
+)
+
+
+def test_normalize_traditional_to_simplified() -> None:
+    assert _normalize_cjk_to_simplified("營收") == "营收"
+    assert _normalize_cjk_to_simplified("電腦") == "电脑"
+    assert _normalize_cjk_to_simplified("計算機") == "计算机"
+
+
+def test_normalize_leaves_english_unchanged() -> None:
+    assert _normalize_cjk_to_simplified("NVIDIA FY27 revenue") == "NVIDIA FY27 revenue"
+    assert _normalize_cjk_to_simplified("hello world 123") == "hello world 123"
+
+
+def test_normalize_leaves_punctuation_unchanged() -> None:
+    assert _normalize_cjk_to_simplified("营收，同比增长。") == "营收，同比增长。"
+
+
+def test_normalize_handles_empty() -> None:
+    assert _normalize_cjk_to_simplified("") == ""
+
+
+def test_normalize_mixed_traditional_simplified() -> None:
+    # Mixed 繁/简 in the same string — both should normalize to simplified
+    assert _normalize_cjk_to_simplified("營收增长") == "营收增长"
+
+
+def test_tokenize_for_retrieval_normalizes_traditional() -> None:
+    traditional_tokens = tokenize_for_retrieval("營收同比增長")
+    simplified_tokens = tokenize_for_retrieval("营收同比增长")
+    assert traditional_tokens == simplified_tokens
+
+
+def test_tokenize_for_retrieval_normalizes_mixed() -> None:
+    mixed_tokens = tokenize_for_retrieval("營收增长 NVIDIA")
+    simplified_tokens = tokenize_for_retrieval("营收增长 NVIDIA")
+    assert mixed_tokens == simplified_tokens
+
+
+def test_tokenize_for_retrieval_leaves_english_unchanged() -> None:
+    tokens = tokenize_for_retrieval("NVIDIA quarterly revenue")
+    lowered = [t.lower() for t in tokens]
+    assert "nvidia" in lowered
+    assert "quarterly" in lowered
+
+
+def test_tokenize_contents_for_retrieval_normalizes_traditional() -> None:
+    traditional = tokenize_contents_for_retrieval(["營收同比增長"], link_char=" ")
+    simplified = tokenize_contents_for_retrieval(["营收同比增长"], link_char=" ")
+    assert traditional == simplified
+
+
+def test_tokenize2stw_remove_normalizes_traditional() -> None:
+    traditional = tokenize2stw_remove(["營收同比增長"])
+    simplified = tokenize2stw_remove(["营收同比增长"])
+    assert traditional == simplified
+
+
+def test_traditional_and_simplified_produce_same_tokens_cross_match() -> None:
+    # The core regression: a doc ingested with 繁体 should be findable
+    # by a query in 简体, and vice versa.
+    ingest_tokens = tokenize_contents_for_retrieval(
+        ["台積電營收創歷史新高"], link_char=" "
+    )
+    query_tokens_traditional = tokenize_for_retrieval("台積電營收")
+    query_tokens_simplified = tokenize_for_retrieval("台积电营收")
+
+    # Both query variants should produce tokens that appear in the ingest tokens
+    ingest_token_set = set(ingest_tokens[0].split()) if ingest_tokens else set()
+    for query_tokens in [query_tokens_traditional, query_tokens_simplified]:
+        for token in query_tokens:
+            if token in ingest_token_set:
+                break
+        else:
+            assert False, f"Query token {query_tokens} not found in ingest tokens {ingest_token_set}"

packages/shared-python/shared/utils/text_utils.py

@@ -18,6 +18,27 @@
     _blingfire_text_to_words = None
 
 
+try:
+    from opencc import OpenCC as _OpenCC
+
+    _T2S_CONVERTER = _OpenCC("t2s")
+except (ImportError, OSError):
+    _T2S_CONVERTER = None
+
+
+def _normalize_cjk_to_simplified(text: str) -> str:
+    """Normalize Traditional Chinese characters to Simplified.
+
+    Applied before jieba tokenization so that 繁/简 variants produce the same
+    tokens on both ingest and query sides, letting BM25 cross-match. OpenCC
+    only converts CJK characters — English, numbers, and punctuation are
+    untouched. Falls through unchanged when ``opencc`` is not installed.
+    """
+    if _T2S_CONVERTER is None or not text:
+        return text
+    return _T2S_CONVERTER.convert(text)
+
+
 class _JiebaModule(Protocol):
     def lcut(self, sentence: str) -> list[str]: ...
     def cut(self, sentence: str) -> list[str]: ...
@@ -200,10 +221,11 @@ def _tokenize_english_segment(text: str) -> list[str]:
 def _tokenize_cjk_segment(text: str) -> list[str]:
     if not text.strip():
         return []
+    normalized = _normalize_cjk_to_simplified(text)
     try:
-        return list(_jieba.lcut(text))
+        return list(_jieba.lcut(normalized))
     except AttributeError:
-        return list(_jieba.cut(text))
+        return list(_jieba.cut(normalized))
 
 
 def _resolve_retrieval_stopwords(
@@ -302,10 +324,11 @@ def tokenize2stw_remove(contents: List[str], stopwords: Optional[List[str]] = No
     for content in contents:
         # Pre-clean: remove IMAGE_/TABLE_ markers and reference labels
         content = _CHUNK_MARKER_RE.sub('', content)
+        normalized = _normalize_cjk_to_simplified(content)
         try:
-            raw_tokens = _jieba.lcut(content)
+            raw_tokens = _jieba.lcut(normalized)
         except AttributeError:
-            raw_tokens = list(_jieba.cut(content))
+            raw_tokens = list(_jieba.cut(normalized))
         # Filter: keep only tokens with meaningful characters (Chinese/English/numbers)
         tokens = [t for t in raw_tokens if _is_meaningful_token(t)]
         # Remove stopwords