From 51dee55fa55bcd41f3af3f810f20d59fb42e8b9d Mon Sep 17 00:00:00 2001 From: up <1457368987@qq.com> Date: Wed, 22 Jul 2026 13:25:05 +0800 Subject: [PATCH 1/3] fix(search): preserve CJK relevance signals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 背景 / Background：ASCII-only tokenization dropped CJK queries, making lexical and rare-term ranking signals zero / ASCII-only tokenization removed CJK queries and erased relevance signals. - 改动 / Changes：新增 bounded Unicode tokenizer，Latin/数字保留原语义，Han/Hiragana/Katakana/Hangul 使用 bigram，并支持日文长音符 / Add a bounded Unicode tokenizer with CJK bigrams and Japanese prolonged-mark support while preserving Latin/digit behavior. - 文件 / Files：src/search/core/text-tokenizer.ts, lexical-alignment.ts, rare-terms.ts, related unit tests - 验证 / Verification：24 files / 236 tests passed; npm run lint passed; npm run build passed; git diff --check passed. --- src/search/core/lexical-alignment.ts | 8 ++-- src/search/core/rare-terms.ts | 9 ++-- src/search/core/text-tokenizer.ts | 41 +++++++++++++++++++ .../search/core/lexical-alignment.test.ts | 9 ++++ tests/unit/search/core/rare-terms.test.ts | 36 ++++++++++++++++ tests/unit/search/core/text-tokenizer.test.ts | 29 +++++++++++++ 6 files changed, 122 insertions(+), 10 deletions(-) create mode 100644 src/search/core/text-tokenizer.ts create mode 100644 tests/unit/search/core/text-tokenizer.test.ts diff --git a/src/search/core/lexical-alignment.ts b/src/search/core/lexical-alignment.ts index ea954643..d7514fb5 100644 --- a/src/search/core/lexical-alignment.ts +++ b/src/search/core/lexical-alignment.ts @@ -6,6 +6,8 @@ // (typical brand-collision pattern: query about a technology, result is // a retail homepage with no technical tokens). +import { tokenizeRankingText } from './text-tokenizer.js'; + const STOPWORDS: ReadonlySet = new Set([ 'the', 'a', 'an', 'what', 'is', 'are', 'was', 'were', 'how', 'why', 'when', 'where', 'who', @@ -22,11 +24,7 @@ const STOPWORDS: ReadonlySet = new Set([ ]); function tokenize(s: string): string[] { - return s - .toLowerCase() - .replace(/[^a-z0-9]+/g, ' ') - .split(/\s+/) - .filter((t) => t.length >= 2 && !STOPWORDS.has(t)); + return tokenizeRankingText(s).filter((t) => t.length >= 2 && !STOPWORDS.has(t)); } /** diff --git a/src/search/core/rare-terms.ts b/src/search/core/rare-terms.ts index 30d7bfb8..073cf717 100644 --- a/src/search/core/rare-terms.ts +++ b/src/search/core/rare-terms.ts @@ -6,6 +6,8 @@ // Multi-word concept queries are scored by the longest in-order run of query // content-tokens present in the doc (Reciprocal-Rank-Fusion vs "Reciprocal"). +import { tokenizeRankingText } from './text-tokenizer.js'; + export interface RareTerms { compoundTokens: string[]; conceptPhrase: string[] | null; @@ -57,10 +59,7 @@ function classifyCompound(raw: string): string | null { } function contentTokens(query: string): string[] { - return query - .toLowerCase() - .split(/\s+/) - .map(stripEdges) + return tokenizeRankingText(query) .filter((t) => t.length >= 2 && !STOPWORDS.has(t)); } @@ -85,7 +84,7 @@ export function detectRareTerms(query: string): RareTerms { } function tokenizeDoc(s: string): string[] { - return s.toLowerCase().replace(/[^a-z0-9]+/g, ' ').split(/\s+/).filter(Boolean); + return tokenizeRankingText(s); } // Longest contiguous run of `phrase` tokens (in their query order) that appears diff --git a/src/search/core/text-tokenizer.ts b/src/search/core/text-tokenizer.ts new file mode 100644 index 00000000..c0a221da --- /dev/null +++ b/src/search/core/text-tokenizer.ts @@ -0,0 +1,41 @@ +const LATIN_OR_DIGIT_RE = /[\p{Script=Latin}\p{N}]/u; +const CJK_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u; +// `Script_Extensions` 将 ー 等日文共享符号保留在 Katakana 序列中， +// 避免把 ニュース 错分成 ニュ + ス。 +const CJK_RUN = String.raw`[\p{Script_Extensions=Han}\p{Script_Extensions=Hiragana}\p{Script_Extensions=Katakana}\p{Script_Extensions=Hangul}]+`; +const TOKEN_PART_RE = new RegExp(String.raw`[\p{Script=Latin}\p{N}]+|${CJK_RUN}`, 'gu'); +const MAX_INPUT_CODE_UNITS = 4096; +const MAX_TOKENS = 256; + +/** + * 对 ranking 文本做 Unicode-aware 分词，避免丢失 CJK 字符。 + * Latin/数字沿用小写单词语义，连续 CJK 文本生成重叠 bigram，保证无空格查询可比较。 + */ +export function tokenizeRankingText(text: string): string[] { + const tokens: string[] = []; + // 先限制待处理输入，再执行 lowercase/regex/Array.from，避免超长 query + // 在 token cap 生效前产生完整中间数组或触发 RegExp stack overflow。 + const boundedText = text.slice(0, MAX_INPUT_CODE_UNITS).toLowerCase(); + const parts = boundedText.match(TOKEN_PART_RE) ?? []; + + for (const part of parts) { + if (tokens.length >= MAX_TOKENS) break; + if (LATIN_OR_DIGIT_RE.test(part)) { + tokens.push(part); + continue; + } + if (!CJK_RE.test(part)) continue; + + const chars = Array.from(part); + if (chars.length === 1) { + tokens.push(chars[0]); + continue; + } + for (let i = 0; i < chars.length - 1; i++) { + tokens.push(chars[i] + chars[i + 1]); + if (tokens.length >= MAX_TOKENS) break; + } + } + + return tokens; +} diff --git a/tests/unit/search/core/lexical-alignment.test.ts b/tests/unit/search/core/lexical-alignment.test.ts index fec641b1..b99a9cc9 100644 --- a/tests/unit/search/core/lexical-alignment.test.ts +++ b/tests/unit/search/core/lexical-alignment.test.ts @@ -55,4 +55,13 @@ describe('lexicalAlignment', () => { const a = lexicalAlignment('Next.js', 'NEXT-JS Docs', ''); expect(a).toBe(1); }); + + it('scores a topically aligned CJK result above unrelated fresh content', () => { + const query = '北京人工智能大会最新消息'; + const relevant = lexicalAlignment(query, '北京人工智能大会发布最新成果', '大模型产业动态'); + const irrelevant = lexicalAlignment(query, '今日黄历与北京天气预报', '出行和生活指数'); + + expect(relevant).toBeGreaterThan(0); + expect(relevant).toBeGreaterThan(irrelevant); + }); }); diff --git a/tests/unit/search/core/rare-terms.test.ts b/tests/unit/search/core/rare-terms.test.ts index 76beb554..94b701d9 100644 --- a/tests/unit/search/core/rare-terms.test.ts +++ b/tests/unit/search/core/rare-terms.test.ts @@ -41,6 +41,15 @@ describe('detectRareTerms', () => { expect(phrase).not.toBeNull(); expect(phrase!.length).toBeLessThanOrEqual(32); }); + + it('emits a bounded concept phrase for an unsegmented CJK query', () => { + const phrase = detectRareTerms('北京人工智能大会最新消息').conceptPhrase; + + expect(phrase).not.toBeNull(); + expect(phrase).toContain('人工'); + expect(phrase).toContain('智能'); + expect(phrase!.length).toBeLessThanOrEqual(32); + }); }); describe('rareTermFactor', () => { @@ -75,6 +84,20 @@ describe('rareTermFactor', () => { it('returns 1.0 for plain queries with no rare terms', () => { expect(rareTermFactor({ title: 'x', url: 'https://x.com', snippet: 'y' }, detectRareTerms('best laptop'))).toBe(1); }); + + it('boosts a contiguous CJK topic match above unrelated calendar content', () => { + const rare = detectRareTerms('北京人工智能大会最新消息'); + const relevant = rareTermFactor( + { title: '北京人工智能大会发布最新成果', url: 'https://example.com/ai', snippet: '产业动态' }, + rare, + ); + const irrelevant = rareTermFactor( + { title: '今日黄历与北京天气预报', url: 'https://example.com/calendar', snippet: '出行指数' }, + rare, + ); + + expect(relevant).toBeGreaterThan(irrelevant); + }); }); describe('isRareTermMiss', () => { @@ -91,6 +114,19 @@ describe('isRareTermMiss', () => { expect(isRareTermMiss({ title: 'Reciprocal Rank Fusion', url: 'https://e.com', snippet: 'how RRF works' }, phrase)).toBe(false); }); + it('distinguishes a CJK topic hit from unrelated calendar content', () => { + const cjk = detectRareTerms('北京人工智能大会最新消息'); + + expect(isRareTermMiss( + { title: '北京人工智能大会发布最新成果', url: 'https://e.com/ai', snippet: '产业动态' }, + cjk, + )).toBe(false); + expect(isRareTermMiss( + { title: '今日黄历与北京天气预报', url: 'https://e.com/calendar', snippet: '出行指数' }, + cjk, + )).toBe(true); + }); + it('is never a miss for a single-token query with no rare terms', () => { // one token => no compound and no concept phrase => nothing to miss expect(isRareTermMiss({ title: 'x', url: 'https://x.com', snippet: 'y' }, detectRareTerms('laptop'))).toBe(false); diff --git a/tests/unit/search/core/text-tokenizer.test.ts b/tests/unit/search/core/text-tokenizer.test.ts new file mode 100644 index 00000000..487453b6 --- /dev/null +++ b/tests/unit/search/core/text-tokenizer.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from 'vitest'; +import { tokenizeRankingText } from '../../../../src/search/core/text-tokenizer.js'; + +describe('tokenizeRankingText', () => { + it('preserves Latin and digit runs while emitting CJK bigrams', () => { + expect(tokenizeRankingText('Hermes Agent 中文配置 2026')).toEqual([ + 'hermes', + 'agent', + '中文', + '文配', + '配置', + '2026', + ]); + }); + + it('keeps Japanese prolonged sound marks inside Katakana bigrams', () => { + expect(tokenizeRankingText('AIニュース')).toEqual(['ai', 'ニュ', 'ュー', 'ース']); + }); + + it('emits Hangul bigrams for unsegmented Korean text', () => { + expect(tokenizeRankingText('인공지능뉴스')).toEqual(['인공', '공지', '지능', '능뉴', '뉴스']); + }); + + it('caps token output for pathological unsegmented input', () => { + const tokens = tokenizeRankingText('中'.repeat(10_000_000)); + + expect(tokens.length).toBeLessThanOrEqual(256); + }); +}); \ No newline at end of file From ee9166691fef7adda879fab254787afb455f32a1 Mon Sep 17 00:00:00 2001 From: up <1457368987@qq.com> Date: Fri, 7 Aug 2026 10:32:55 +0800 Subject: [PATCH 2/3] docs(search): translate tokenizer comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 背景 / Background：维护者要求将 tokenizer 中的中文注释改为英文，以保持代码库风格一致 / The maintainer requested English comments for consistency across the codebase. - 改动 / Changes：翻译 Script_Extensions、Unicode-aware tokenization 与输入上限相关注释，不改动运行逻辑 / Translated comments about Script_Extensions, Unicode-aware tokenization, and input bounds without changing runtime logic. - 文件 / Files：src/search/core/text-tokenizer.ts - 验证 / Verification：28 targeted tests passed；npm run lint passed；npm run build passed；git diff --check passed；full npm test reached 7837 passed with 22 unrelated failures / 28 targeted tests passed; lint, build, and diff check passed; the full suite completed with 7837 passes and 22 failures outside this comment-only scope. --- src/search/core/text-tokenizer.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/search/core/text-tokenizer.ts b/src/search/core/text-tokenizer.ts index c0a221da..8c170d19 100644 --- a/src/search/core/text-tokenizer.ts +++ b/src/search/core/text-tokenizer.ts @@ -1,20 +1,20 @@ const LATIN_OR_DIGIT_RE = /[\p{Script=Latin}\p{N}]/u; const CJK_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u; -// `Script_Extensions` 将 ー 等日文共享符号保留在 Katakana 序列中， -// 避免把 ニュース 错分成 ニュ + ス。 +// `Script_Extensions` keeps shared Japanese marks such as ー inside Katakana runs, +// preventing ニュース from being split into ニュ + ス. const CJK_RUN = String.raw`[\p{Script_Extensions=Han}\p{Script_Extensions=Hiragana}\p{Script_Extensions=Katakana}\p{Script_Extensions=Hangul}]+`; const TOKEN_PART_RE = new RegExp(String.raw`[\p{Script=Latin}\p{N}]+|${CJK_RUN}`, 'gu'); const MAX_INPUT_CODE_UNITS = 4096; const MAX_TOKENS = 256; /** - * 对 ranking 文本做 Unicode-aware 分词，避免丢失 CJK 字符。 - * Latin/数字沿用小写单词语义，连续 CJK 文本生成重叠 bigram，保证无空格查询可比较。 + * Tokenize ranking text with Unicode awareness so CJK characters are not discarded. + * Preserve lowercase word semantics for Latin/digits and emit overlapping bigrams for contiguous CJK text. */ export function tokenizeRankingText(text: string): string[] { const tokens: string[] = []; - // 先限制待处理输入，再执行 lowercase/regex/Array.from，避免超长 query - // 在 token cap 生效前产生完整中间数组或触发 RegExp stack overflow。 + // Bound input before lowercasing, regex matching, or Array.from so an oversized query + // cannot allocate complete intermediate arrays or overflow the RegExp stack before the token cap applies. const boundedText = text.slice(0, MAX_INPUT_CODE_UNITS).toLowerCase(); const parts = boundedText.match(TOKEN_PART_RE) ?? []; From 3acd57758a6d9cef585aa25bb638159b19a548f1 Mon Sep 17 00:00:00 2001 From: up <1457368987@qq.com> Date: Fri, 7 Aug 2026 10:50:40 +0800 Subject: [PATCH 3/3] fix(search): exclude CJK punctuation from token runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 背景 / Background：CodeRabbit 指出 Script_Extensions 会把 、。・并入 CJK run，生成含标点的 bigram / CodeRabbit found that Script_Extensions allowed CJK punctuation into token runs and bigrams. - 改动 / Changes：改用基础 Script 属性并仅显式保留 U+30FC，同时增加 CJK 标点分隔回归测试 / Switched to base Script properties with explicit U+30FC support and added a punctuation-boundary regression test. - 文件 / Files：src/search/core/text-tokenizer.ts, tests/unit/search/core/text-tokenizer.test.ts - 验证 / Verification：先确认新增测试失败，再确认 29 个定向测试、npm run lint、npm run build、git diff --check 通过 / Confirmed the new test failed first, then 29 targeted tests, lint, build, and diff check passed. --- src/search/core/text-tokenizer.ts | 6 +++--- tests/unit/search/core/text-tokenizer.test.ts | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/search/core/text-tokenizer.ts b/src/search/core/text-tokenizer.ts index 8c170d19..0a817f7b 100644 --- a/src/search/core/text-tokenizer.ts +++ b/src/search/core/text-tokenizer.ts @@ -1,8 +1,8 @@ const LATIN_OR_DIGIT_RE = /[\p{Script=Latin}\p{N}]/u; const CJK_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u; -// `Script_Extensions` keeps shared Japanese marks such as ー inside Katakana runs, -// preventing ニュース from being split into ニュ + ス. -const CJK_RUN = String.raw`[\p{Script_Extensions=Han}\p{Script_Extensions=Hiragana}\p{Script_Extensions=Katakana}\p{Script_Extensions=Hangul}]+`; +// Include the prolonged sound mark explicitly so it stays inside Katakana runs, +// while punctuation such as 、, 。, and ・ still separates CJK token parts. +const CJK_RUN = String.raw`[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\u30FC]+`; const TOKEN_PART_RE = new RegExp(String.raw`[\p{Script=Latin}\p{N}]+|${CJK_RUN}`, 'gu'); const MAX_INPUT_CODE_UNITS = 4096; const MAX_TOKENS = 256; diff --git a/tests/unit/search/core/text-tokenizer.test.ts b/tests/unit/search/core/text-tokenizer.test.ts index 487453b6..c4fa7222 100644 --- a/tests/unit/search/core/text-tokenizer.test.ts +++ b/tests/unit/search/core/text-tokenizer.test.ts @@ -17,6 +17,15 @@ describe('tokenizeRankingText', () => { expect(tokenizeRankingText('AIニュース')).toEqual(['ai', 'ニュ', 'ュー', 'ース']); }); + it('splits CJK runs around punctuation', () => { + expect(tokenizeRankingText('東京、京都。大阪・神戸')).toEqual([ + '東京', + '京都', + '大阪', + '神戸', + ]); + }); + it('emits Hangul bigrams for unsegmented Korean text', () => { expect(tokenizeRankingText('인공지능뉴스')).toEqual(['인공', '공지', '지능', '능뉴', '뉴스']); });