fix(search): preserve CJK relevance signals

src/search/core/lexical-alignment.ts

@@ -6,6 +6,8 @@
 // (typical brand-collision pattern: query about a technology, result is
 // a retail homepage with no technical tokens).
 
+import { tokenizeRankingText } from './text-tokenizer.js';
+
 const STOPWORDS: ReadonlySet<string> = new Set([
   'the', 'a', 'an',
   'what', 'is', 'are', 'was', 'were', 'how', 'why', 'when', 'where', 'who',
@@ -22,11 +24,7 @@ const STOPWORDS: ReadonlySet<string> = new Set([
 ]);
 
 function tokenize(s: string): string[] {
-  return s
-    .toLowerCase()
-    .replace(/[^a-z0-9]+/g, ' ')
-    .split(/\s+/)
-    .filter((t) => t.length >= 2 && !STOPWORDS.has(t));
+  return tokenizeRankingText(s).filter((t) => t.length >= 2 && !STOPWORDS.has(t));
 }
 
 /**

src/search/core/rare-terms.ts

@@ -6,6 +6,8 @@
 // Multi-word concept queries are scored by the longest in-order run of query
 // content-tokens present in the doc (Reciprocal-Rank-Fusion vs "Reciprocal").
 
+import { tokenizeRankingText } from './text-tokenizer.js';
+
 export interface RareTerms {
   compoundTokens: string[];
   conceptPhrase: string[] | null;
@@ -57,10 +59,7 @@ function classifyCompound(raw: string): string | null {
 }
 
 function contentTokens(query: string): string[] {
-  return query
-    .toLowerCase()
-    .split(/\s+/)
-    .map(stripEdges)
+  return tokenizeRankingText(query)
     .filter((t) => t.length >= 2 && !STOPWORDS.has(t));
 }
 
@@ -85,7 +84,7 @@ export function detectRareTerms(query: string): RareTerms {
 }
 
 function tokenizeDoc(s: string): string[] {
-  return s.toLowerCase().replace(/[^a-z0-9]+/g, ' ').split(/\s+/).filter(Boolean);
+  return tokenizeRankingText(s);
 }
 
 // Longest contiguous run of `phrase` tokens (in their query order) that appears

src/search/core/text-tokenizer.ts

@@ -0,0 +1,41 @@
+const LATIN_OR_DIGIT_RE = /[\p{Script=Latin}\p{N}]/u;
+const CJK_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
+// Include the prolonged sound mark explicitly so it stays inside Katakana runs,
+// while punctuation such as 、, 。, and ・ still separates CJK token parts.
+const CJK_RUN = String.raw`[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\u30FC]+`;
+const TOKEN_PART_RE = new RegExp(String.raw`[\p{Script=Latin}\p{N}]+|${CJK_RUN}`, 'gu');
+const MAX_INPUT_CODE_UNITS = 4096;
+const MAX_TOKENS = 256;
+
+/**
+ * Tokenize ranking text with Unicode awareness so CJK characters are not discarded.
+ * Preserve lowercase word semantics for Latin/digits and emit overlapping bigrams for contiguous CJK text.
+ */
+export function tokenizeRankingText(text: string): string[] {
+  const tokens: string[] = [];
+  // Bound input before lowercasing, regex matching, or Array.from so an oversized query
+  // cannot allocate complete intermediate arrays or overflow the RegExp stack before the token cap applies.
+  const boundedText = text.slice(0, MAX_INPUT_CODE_UNITS).toLowerCase();
+  const parts = boundedText.match(TOKEN_PART_RE) ?? [];
+
+  for (const part of parts) {
+    if (tokens.length >= MAX_TOKENS) break;
+    if (LATIN_OR_DIGIT_RE.test(part)) {
+      tokens.push(part);
+      continue;
+    }
+    if (!CJK_RE.test(part)) continue;
+
+    const chars = Array.from(part);
+    if (chars.length === 1) {
+      tokens.push(chars[0]);
+      continue;
+    }
+    for (let i = 0; i < chars.length - 1; i++) {
+      tokens.push(chars[i] + chars[i + 1]);
+      if (tokens.length >= MAX_TOKENS) break;
+    }
+  }
+
+  return tokens;
+}

tests/unit/search/core/lexical-alignment.test.ts

@@ -55,4 +55,13 @@ describe('lexicalAlignment', () => {
     const a = lexicalAlignment('Next.js', 'NEXT-JS Docs', '');
     expect(a).toBe(1);
   });
+
+  it('scores a topically aligned CJK result above unrelated fresh content', () => {
+    const query = '北京人工智能大会最新消息';
+    const relevant = lexicalAlignment(query, '北京人工智能大会发布最新成果', '大模型产业动态');
+    const irrelevant = lexicalAlignment(query, '今日黄历与北京天气预报', '出行和生活指数');
+
+    expect(relevant).toBeGreaterThan(0);
+    expect(relevant).toBeGreaterThan(irrelevant);
+  });
 });

tests/unit/search/core/rare-terms.test.ts

@@ -41,6 +41,15 @@ describe('detectRareTerms', () => {
     expect(phrase).not.toBeNull();
     expect(phrase!.length).toBeLessThanOrEqual(32);
   });
+
+  it('emits a bounded concept phrase for an unsegmented CJK query', () => {
+    const phrase = detectRareTerms('北京人工智能大会最新消息').conceptPhrase;
+
+    expect(phrase).not.toBeNull();
+    expect(phrase).toContain('人工');
+    expect(phrase).toContain('智能');
+    expect(phrase!.length).toBeLessThanOrEqual(32);
+  });
 });
 
 describe('rareTermFactor', () => {
@@ -75,6 +84,20 @@ describe('rareTermFactor', () => {
   it('returns 1.0 for plain queries with no rare terms', () => {
     expect(rareTermFactor({ title: 'x', url: 'https://x.com', snippet: 'y' }, detectRareTerms('best laptop'))).toBe(1);
   });
+
+  it('boosts a contiguous CJK topic match above unrelated calendar content', () => {
+    const rare = detectRareTerms('北京人工智能大会最新消息');
+    const relevant = rareTermFactor(
+      { title: '北京人工智能大会发布最新成果', url: 'https://example.com/ai', snippet: '产业动态' },
+      rare,
+    );
+    const irrelevant = rareTermFactor(
+      { title: '今日黄历与北京天气预报', url: 'https://example.com/calendar', snippet: '出行指数' },
+      rare,
+    );
+
+    expect(relevant).toBeGreaterThan(irrelevant);
+  });
 });
 
 describe('isRareTermMiss', () => {
@@ -91,6 +114,19 @@ describe('isRareTermMiss', () => {
     expect(isRareTermMiss({ title: 'Reciprocal Rank Fusion', url: 'https://e.com', snippet: 'how RRF works' }, phrase)).toBe(false);
   });
 
+  it('distinguishes a CJK topic hit from unrelated calendar content', () => {
+    const cjk = detectRareTerms('北京人工智能大会最新消息');
+
+    expect(isRareTermMiss(
+      { title: '北京人工智能大会发布最新成果', url: 'https://e.com/ai', snippet: '产业动态' },
+      cjk,
+    )).toBe(false);
+    expect(isRareTermMiss(
+      { title: '今日黄历与北京天气预报', url: 'https://e.com/calendar', snippet: '出行指数' },
+      cjk,
+    )).toBe(true);
+  });
+
   it('is never a miss for a single-token query with no rare terms', () => {
     // one token => no compound and no concept phrase => nothing to miss
     expect(isRareTermMiss({ title: 'x', url: 'https://x.com', snippet: 'y' }, detectRareTerms('laptop'))).toBe(false);

tests/unit/search/core/text-tokenizer.test.ts

@@ -0,0 +1,38 @@
+import { describe, expect, it } from 'vitest';
+import { tokenizeRankingText } from '../../../../src/search/core/text-tokenizer.js';
+
+describe('tokenizeRankingText', () => {
+  it('preserves Latin and digit runs while emitting CJK bigrams', () => {
+    expect(tokenizeRankingText('Hermes Agent 中文配置 2026')).toEqual([
+      'hermes',
+      'agent',
+      '中文',
+      '文配',
+      '配置',
+      '2026',
+    ]);
+  });
+
+  it('keeps Japanese prolonged sound marks inside Katakana bigrams', () => {
+    expect(tokenizeRankingText('AIニュース')).toEqual(['ai', 'ニュ', 'ュー', 'ース']);
+  });
+
+  it('splits CJK runs around punctuation', () => {
+    expect(tokenizeRankingText('東京、京都。大阪・神戸')).toEqual([
+      '東京',
+      '京都',
+      '大阪',
+      '神戸',
+    ]);
+  });
+
+  it('emits Hangul bigrams for unsegmented Korean text', () => {
+    expect(tokenizeRankingText('인공지능뉴스')).toEqual(['인공', '공지', '지능', '능뉴', '뉴스']);
+  });
+
+  it('caps token output for pathological unsegmented input', () => {
+    const tokens = tokenizeRankingText('中'.repeat(10_000_000));
+
+    expect(tokens.length).toBeLessThanOrEqual(256);
+  });
+});