From 36908c81500dc58e913764a32ea1423bd0c7f2b8 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:59:06 +0800 Subject: [PATCH 1/3] feat: rank parallels by textual relevance --- src/query.rs | 81 +++++++++++++++++++++++------------- tests/query.rs | 111 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 30 deletions(-) diff --git a/src/query.rs b/src/query.rs index 657ba49..db44aa5 100644 --- a/src/query.rs +++ b/src/query.rs @@ -1,7 +1,7 @@ use crate::model::{MatchGroup, Parallel}; use rusqlite::Connection; use std::cmp::Ordering; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; struct Row { zh_text: String, @@ -64,12 +64,20 @@ fn fetch_rows(conn: &Connection, norm_query: &str) -> rusqlite::Result> iter.collect() } +#[derive(Hash, PartialEq, Eq)] +struct GroupKey { + zh_text: String, + cbeta_id: Option, + juan_num: Option, +} + struct Acc { zh_text: String, cbeta_id: Option, title_zh: Option, juan_num: Option, - contains: bool, + exact: bool, + excess_chars: usize, max_conf: f64, parallels: Vec, } @@ -81,6 +89,8 @@ fn group_and_rank( top: usize, ) -> Vec { let mut accs: Vec = Vec::new(); + let mut acc_idx: HashMap = HashMap::new(); + let query_chars = norm_query.chars().count(); for row in rows { if let Some(filter) = langs { if !filter.iter().any(|l| l == &row.foreign_lang) { @@ -88,10 +98,18 @@ fn group_and_rank( } } let contains = row.zh_norm.contains(norm_query); - let idx = accs.iter().position(|a| { - a.zh_text == row.zh_text && a.cbeta_id == row.cbeta_id && a.juan_num == row.juan_num - }); - let idx = match idx { + let exact = row.zh_norm == norm_query; + let excess_chars = if contains { + row.zh_norm.chars().count().saturating_sub(query_chars) + } else { + row.zh_norm.chars().count() + }; + let key = GroupKey { + zh_text: row.zh_text.clone(), + cbeta_id: row.cbeta_id.clone(), + juan_num: row.juan_num, + }; + let idx = match acc_idx.get(&key).copied() { Some(i) => i, None => { accs.push(Acc { @@ -99,18 +117,24 @@ fn group_and_rank( cbeta_id: row.cbeta_id.clone(), title_zh: row.title_zh.clone(), juan_num: row.juan_num, - contains: false, + exact: false, + excess_chars, max_conf: 0.0, parallels: Vec::new(), }); - accs.len() - 1 + let idx = accs.len() - 1; + acc_idx.insert(key, idx); + idx } }; let conf = row.confidence.unwrap_or(0.0); if conf > accs[idx].max_conf { accs[idx].max_conf = conf; } - accs[idx].contains |= contains; + accs[idx].exact |= exact; + if excess_chars < accs[idx].excess_chars { + accs[idx].excess_chars = excess_chars; + } accs[idx].parallels.push(Parallel { lang: row.foreign_lang, text: row.foreign_text, @@ -118,28 +142,25 @@ fn group_and_rank( }); } - let mut ranked: Vec<(bool, f64, MatchGroup)> = accs - .into_iter() - .map(|a| { - ( - a.contains, - a.max_conf, - MatchGroup { - zh_text: a.zh_text, - cbeta_id: a.cbeta_id, - title_zh: a.title_zh, - juan_num: a.juan_num, - parallels: cap_per_lang(a.parallels, top), - }, - ) - }) - .collect(); - - ranked.sort_by(|a, b| { - b.0.cmp(&a.0) - .then(b.1.partial_cmp(&a.1).unwrap_or(Ordering::Equal)) + accs.sort_by(|a, b| { + b.exact + .cmp(&a.exact) + .then(a.excess_chars.cmp(&b.excess_chars)) + .then_with(|| b.max_conf.total_cmp(&a.max_conf)) + .then_with(|| a.cbeta_id.cmp(&b.cbeta_id)) + .then_with(|| a.juan_num.cmp(&b.juan_num)) + .then_with(|| a.zh_text.cmp(&b.zh_text)) }); - ranked.into_iter().map(|(_, _, g)| g).collect() + + accs.into_iter() + .map(|a| MatchGroup { + zh_text: a.zh_text, + cbeta_id: a.cbeta_id, + title_zh: a.title_zh, + juan_num: a.juan_num, + parallels: cap_per_lang(a.parallels, top), + }) + .collect() } /// List a text's aligned groups by Taishō id (case-insensitive), optionally diff --git a/tests/query.rs b/tests/query.rs index 91be13c..a73adbb 100644 --- a/tests/query.rs +++ b/tests/query.rs @@ -117,6 +117,117 @@ fn empty_query_returns_empty() { assert!(search(&conn, "", None, 3).unwrap().is_empty()); } +#[test] +fn exact_sentence_outranks_longer_higher_confidence_match() { + let conn = Connection::open_in_memory().unwrap(); + init_schema(&conn).unwrap(); + for (zt, zn, lang, f, c, cb, ti, j) in [ + ( + "舍利子色不异空色即是空空即是色", + "舍利子色不异空色即是空空即是色", + "sa", + "long-high", + 1.0, + "T0002", + "長句", + 1, + ), + ( + "色即是空", + "色即是空", + "sa", + "exact-lower", + 0.8, + "T0001", + "短句", + 1, + ), + ] { + conn.execute( + "INSERT INTO parallels(zh_text,zh_norm,foreign_lang,foreign_text,confidence,cbeta_id,title_zh,juan_num) + VALUES (?1,?2,?3,?4,?5,?6,?7,?8)", + params![zt, zn, lang, f, c, cb, ti, j], + ) + .unwrap(); + } + + let groups = search(&conn, "色即是空", None, 3).unwrap(); + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].zh_text, "色即是空"); +} + +#[test] +fn shorter_containing_sentence_outranks_longer_peer() { + let conn = Connection::open_in_memory().unwrap(); + init_schema(&conn).unwrap(); + for (zt, zn, lang, f, c, cb, ti, j) in [ + ( + "觀自在菩薩行深般若波羅蜜多時照見五蘊皆空色即是空", + "观自在菩萨行深般若波罗蜜多时照见五蕴皆空色即是空", + "sa", + "long-peer", + 1.0, + "T0002", + "長句", + 1, + ), + ( + "五蘊皆空色即是空", + "五蕴皆空色即是空", + "sa", + "short-peer", + 1.0, + "T0003", + "短句", + 1, + ), + ] { + conn.execute( + "INSERT INTO parallels(zh_text,zh_norm,foreign_lang,foreign_text,confidence,cbeta_id,title_zh,juan_num) + VALUES (?1,?2,?3,?4,?5,?6,?7,?8)", + params![zt, zn, lang, f, c, cb, ti, j], + ) + .unwrap(); + } + + let groups = search(&conn, "色即是空", None, 3).unwrap(); + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].zh_text, "五蘊皆空色即是空"); +} + +#[test] +fn relevance_ties_use_stable_source_order() { + let conn = Connection::open_in_memory().unwrap(); + init_schema(&conn).unwrap(); + for (zt, zn, lang, f, cb) in [ + ( + "甲色即是空乙", + "甲色即是空乙", + "sa", + "first-inserted", + "T0002", + ), + ( + "丙色即是空丁", + "丙色即是空丁", + "sa", + "second-inserted", + "T0001", + ), + ] { + conn.execute( + "INSERT INTO parallels(zh_text,zh_norm,foreign_lang,foreign_text,confidence,cbeta_id,title_zh,juan_num) + VALUES (?1,?2,?3,?4,1.0,?5,'同分',1)", + params![zt, zn, lang, f, cb], + ) + .unwrap(); + } + + let groups = search(&conn, "色即是空", None, 3).unwrap(); + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].cbeta_id.as_deref(), Some("T0001")); +} + fn cite_fixture() -> Connection { let conn = Connection::open_in_memory().unwrap(); init_schema(&conn).unwrap(); From c8dfefc3123138051b183b44b641f1bfe4dcfebe Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:07:31 +0800 Subject: [PATCH 2/3] test: pin relevance tie ordering --- tests/query.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/query.rs b/tests/query.rs index a73adbb..a057d99 100644 --- a/tests/query.rs +++ b/tests/query.rs @@ -228,6 +228,34 @@ fn relevance_ties_use_stable_source_order() { assert_eq!(groups[0].cbeta_id.as_deref(), Some("T0001")); } +#[test] +fn relevance_ties_order_by_juan_then_zh_text() { + let conn = Connection::open_in_memory().unwrap(); + init_schema(&conn).unwrap(); + for (zt, j) in [("色即是空C", 2), ("色即是空B", 1), ("色即是空A", 1)] { + conn.execute( + "INSERT INTO parallels(zh_text,zh_norm,foreign_lang,foreign_text,confidence,cbeta_id,title_zh,juan_num) + VALUES (?1,?1,'sa',?1,1.0,'T0001','同分',?2)", + params![zt, j], + ) + .unwrap(); + } + + let groups = search(&conn, "色即是空", None, 3).unwrap(); + let order: Vec<_> = groups + .iter() + .map(|group| (group.juan_num, group.zh_text.as_str())) + .collect(); + assert_eq!( + order, + [ + (Some(1), "色即是空A"), + (Some(1), "色即是空B"), + (Some(2), "色即是空C"), + ] + ); +} + fn cite_fixture() -> Connection { let conn = Connection::open_in_memory().unwrap(); init_schema(&conn).unwrap(); From 52d3fc04da908a8c0d90e7845e22349901302c79 Mon Sep 17 00:00:00 2001 From: xr843 <137012659+xr843@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:11:43 +0800 Subject: [PATCH 3/3] test: decouple relevance tie assertions --- tests/query.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/query.rs b/tests/query.rs index a057d99..ace605f 100644 --- a/tests/query.rs +++ b/tests/query.rs @@ -232,7 +232,7 @@ fn relevance_ties_use_stable_source_order() { fn relevance_ties_order_by_juan_then_zh_text() { let conn = Connection::open_in_memory().unwrap(); init_schema(&conn).unwrap(); - for (zt, j) in [("色即是空C", 2), ("色即是空B", 1), ("色即是空A", 1)] { + for (zt, j) in [("色即是空A", 2), ("色即是空C", 1), ("色即是空B", 1)] { conn.execute( "INSERT INTO parallels(zh_text,zh_norm,foreign_lang,foreign_text,confidence,cbeta_id,title_zh,juan_num) VALUES (?1,?1,'sa',?1,1.0,'T0001','同分',?2)", @@ -249,9 +249,9 @@ fn relevance_ties_order_by_juan_then_zh_text() { assert_eq!( order, [ - (Some(1), "色即是空A"), (Some(1), "色即是空B"), - (Some(2), "色即是空C"), + (Some(1), "色即是空C"), + (Some(2), "色即是空A"), ] ); }