From 0697213892e374697fd10913f61b00c4b40bbd0f Mon Sep 17 00:00:00 2001 From: ajianaz Date: Fri, 17 Jul 2026 13:52:17 +0700 Subject: [PATCH 1/2] feat: Jaccard token similarity as post-RRF reranking signal (#719) Add Jaccard token overlap as an orthogonal recall signal (#719): - New jaccard.rs module with tokenize() and jaccard_similarity() - Post-RRF additive boost when jaccard_weight > 0.0 - CLI config [recall].jaccard_weight (default 0.0 = off) - Setter API: Uteke::set_jaccard_weight() - Tag tokens included in content token set for richer overlap Jaccard catches different cases than BM25 (IDF-weighted) and vector cosine (semantic). When enabled (recommended 0.10-0.15), it boosts results with high token overlap regardless of term rarity or embedding distance. Closes #719 --- crates/uteke-cli/src/config.rs | 9 +++ crates/uteke-cli/src/main.rs | 6 +- crates/uteke-core/src/jaccard.rs | 110 ++++++++++++++++++++++++++++ crates/uteke-core/src/lib.rs | 13 ++++ crates/uteke-core/src/operations.rs | 37 ++++++++-- 5 files changed, 167 insertions(+), 8 deletions(-) create mode 100644 crates/uteke-core/src/jaccard.rs diff --git a/crates/uteke-cli/src/config.rs b/crates/uteke-cli/src/config.rs index 9475f443..ddf42fbe 100644 --- a/crates/uteke-cli/src/config.rs +++ b/crates/uteke-cli/src/config.rs @@ -240,6 +240,10 @@ pub struct RecallConfig { /// Weight for the recency boost applied when the `--recency` flag is /// passed to `recall` (#352). 0.0 disables; 0.15 is the default. pub recency_weight: f32, + /// Weight for the Jaccard token reranking boost (#719). + /// Applied post-RRF as an additive signal based on query-content token + /// overlap. 0.0 disables (default); 0.10-0.15 recommended. + pub jaccard_weight: f32, } impl Default for RecallConfig { @@ -254,6 +258,7 @@ impl Default for RecallConfig { graph_rerank_enabled: true, salience_weight: 0.15, recency_weight: 0.15, + jaccard_weight: 0.0, } } } @@ -515,6 +520,9 @@ impl Config { if recall.contains_key("graph_rerank_enabled") { self.recall.graph_rerank_enabled = overlay.recall.graph_rerank_enabled; } + if recall.contains_key("jaccard_weight") { + self.recall.jaccard_weight = overlay.recall.jaccard_weight; + } } // Merge server section @@ -769,6 +777,7 @@ impl Config { # graph_density_weight = 0.1 # graph_authority_weight = 0.1 # graph_rerank_enabled = true +# jaccard_weight = 0.0 # Post-RRF token overlap boost (#719). 0=off, 0.10-0.15 recommended [server] # enabled = false diff --git a/crates/uteke-cli/src/main.rs b/crates/uteke-cli/src/main.rs index e6f332e1..d8b7d698 100644 --- a/crates/uteke-cli/src/main.rs +++ b/crates/uteke-cli/src/main.rs @@ -121,7 +121,11 @@ fn main() { enabled: config.recall.graph_rerank_enabled, }, ) { - Ok(u) => u, + Ok(mut u) => { + // #719: apply Jaccard weight from config + u.set_jaccard_weight(config.recall.jaccard_weight); + u + } Err(e) => { eprintln!("Error: Failed to open memory store: {e}"); std::process::exit(1); diff --git a/crates/uteke-core/src/jaccard.rs b/crates/uteke-core/src/jaccard.rs new file mode 100644 index 00000000..e552995a --- /dev/null +++ b/crates/uteke-core/src/jaccard.rs @@ -0,0 +1,110 @@ +//! Jaccard token similarity reranking signal (#719). +//! +//! Pure set-overlap signal that catches different cases from FTS5 BM25 +//! (which uses IDF-weighted scoring). Jaccard measures the ratio of shared +//! tokens between query and content, independent of term rarity. +//! +//! Adopted from Hermes holographic memory analysis — Hermes uses Jaccard as +//! a 0.3-weighted reranking signal in its hybrid fusion. + +use std::collections::HashSet; + +/// Compute Jaccard similarity between two token sets. +/// +/// Returns `|A ∩ B| / |A ∪ B|` in `[0.0, 1.0]`. +/// Returns 0.0 if both sets are empty. +pub fn jaccard_similarity(query: &HashSet, content: &HashSet) -> f32 { + if query.is_empty() && content.is_empty() { + return 0.0; + } + let intersection = query.intersection(content).count(); + let union = query.union(content).count(); + if union == 0 { + return 0.0; + } + intersection as f32 / union as f32 +} + +/// Tokenize text into a lowercase HashSet. +/// +/// Splits on whitespace and strips punctuation from each token. +/// Empty tokens are discarded. +pub fn tokenize(text: &str) -> HashSet { + text.split_whitespace() + .map(|t| t.trim_matches(|c: char| !c.is_alphanumeric()).to_ascii_lowercase()) + .filter(|t| !t.is_empty()) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn jaccard_identical() { + let a: HashSet<_> = ["hello", "world"].iter().map(|s| s.to_string()).collect(); + assert!((jaccard_similarity(&a, &a) - 1.0).abs() < f32::EPSILON); + } + + #[test] + fn jaccard_disjoint() { + let a: HashSet<_> = ["hello", "world"].iter().map(|s| s.to_string()).collect(); + let b: HashSet<_> = ["foo", "bar"].iter().map(|s| s.to_string()).collect(); + assert!((jaccard_similarity(&a, &b) - 0.0).abs() < f32::EPSILON); + } + + #[test] + fn jaccard_partial() { + let a: HashSet<_> = ["hello", "world", "foo"].iter().map(|s| s.to_string()).collect(); + let b: HashSet<_> = ["hello", "bar"].iter().map(|s| s.to_string()).collect(); + // intersection = {"hello"} = 1, union = {"hello","world","foo","bar"} = 4 + let j = jaccard_similarity(&a, &b); + assert!((j - 0.25).abs() < 0.01, "expected ~0.25, got {j}"); + } + + #[test] + fn jaccard_empty_both() { + let a: HashSet = HashSet::new(); + let b: HashSet = HashSet::new(); + assert!((jaccard_similarity(&a, &b) - 0.0).abs() < f32::EPSILON); + } + + #[test] + fn tokenize_basic() { + let tokens = tokenize("Hello World foo"); + assert!(tokens.contains("hello")); + assert!(tokens.contains("world")); + assert!(tokens.contains("foo")); + assert_eq!(tokens.len(), 3); + } + + #[test] + fn tokenize_strips_punctuation() { + let tokens = tokenize("hello, world! (test)"); + assert!(tokens.contains("hello")); + assert!(tokens.contains("world")); + assert!(tokens.contains("test")); + } + + #[test] + fn tokenize_empty() { + let tokens = tokenize(""); + assert!(tokens.is_empty()); + } + + #[test] + fn tokenize_whitespace_only() { + let tokens = tokenize(" \t\n "); + assert!(tokens.is_empty()); + } + + #[test] + fn jaccard_with_tokenize() { + let q = tokenize("rust memory engine"); + let c = tokenize("rust is a fast memory engine for AI"); + let j = jaccard_similarity(&q, &c); + // intersection = {"rust", "memory", "engine"} = 3 + // union = {"rust", "memory", "engine", "is", "a", "fast", "for", "ai"} = 8 + assert!((j - 0.375).abs() < 0.01, "expected ~0.375, got {j}"); + } +} diff --git a/crates/uteke-core/src/lib.rs b/crates/uteke-core/src/lib.rs index 136f24d9..ac721d65 100644 --- a/crates/uteke-core/src/lib.rs +++ b/crates/uteke-core/src/lib.rs @@ -19,6 +19,7 @@ pub mod extraction; pub mod graph; pub mod graph_rerank; mod import_export; +mod jaccard; mod maintenance; pub mod memory; mod operations; @@ -324,6 +325,9 @@ pub struct Uteke { /// Salience + recency dual-axis boost config (#352). Defaults to all /// weights zero (opt-in per query via CLI flags / API params). salience_recency_config: salience_recency::SalienceRecencyConfig, + /// Jaccard token reranking weight (#719). Additive boost applied + /// post-RRF based on query-content token overlap. Default 0.0 (off). + jaccard_weight: f32, /// Recall cache — avoids redundant embedding computation for repeated queries. recall_cache: recall_cache::RecallCache, } @@ -476,6 +480,7 @@ impl Uteke { graph_rerank_config: graph_rerank_config.sanitized(), salience_recency_config: salience_recency::SalienceRecencyConfig::default(), recall_cache: recall_cache::RecallCache::new(recall_cache::RecallCacheConfig::default()), + jaccard_weight: 0.0, }) } @@ -500,6 +505,14 @@ impl Uteke { self.salience_recency_config = salience_recency::SalienceRecencyConfig::default(); } + /// Set Jaccard token reranking weight (#719). + /// + /// When > 0.0, an additive Jaccard similarity boost is applied post-RRF + /// based on query-content token overlap. Recommended: 0.10-0.15. + pub fn set_jaccard_weight(&mut self, weight: f32) { + self.jaccard_weight = weight.clamp(0.0, 1.0); + } + /// Configure cloud embedding fallback. /// /// Must be called before the first embedding operation. When configured, diff --git a/crates/uteke-core/src/operations.rs b/crates/uteke-core/src/operations.rs index 4053e8de..d7878cfd 100644 --- a/crates/uteke-core/src/operations.rs +++ b/crates/uteke-core/src/operations.rs @@ -808,11 +808,19 @@ impl crate::Uteke { .or_insert_with(|| memory.clone()); } - // Sort by RRF score descending, take top `limit` + // NOTE: min_score is NOT applied here. RRF normalized scores are + // rank-based (0..1) and not directly comparable to cosine similarity. + // Applying a cosine threshold to RRF scores would incorrectly filter + // out valid results. The caller (recall_hybrid) handles threshold + // filtering at the appropriate level. + + // #719: Jaccard token reranking boost (post-RRF, additive). + // Measures query-content token overlap as an orthogonal signal + // to BM25 (IDF-weighted) and vector cosine (semantic). let mut scored: Vec<(String, f64)> = rrf_scores.into_iter().collect(); scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); - let results: Vec = scored + let mut results: Vec = scored .into_iter() .take(limit) .map(|(id, score)| { @@ -831,11 +839,26 @@ impl crate::Uteke { }) .collect(); - // NOTE: min_score is NOT applied here. RRF normalized scores are - // rank-based (0..1) and not directly comparable to cosine similarity. - // Applying a cosine threshold to RRF scores would incorrectly filter - // out valid results. The caller (recall_hybrid) handles threshold - // filtering at the appropriate level. + if self.jaccard_weight > 0.0 { + let query_tokens = crate::jaccard::tokenize(query); + if !query_tokens.is_empty() { + for sr in &mut results { + // Tokenize content + tags for a richer overlap signal + let mut content_tokens = crate::jaccard::tokenize(&sr.memory.content); + for tag in &sr.memory.tags { + content_tokens.insert(tag.to_ascii_lowercase()); + } + let j = crate::jaccard::jaccard_similarity(&query_tokens, &content_tokens); + sr.score += j * self.jaccard_weight; + } + // Re-sort after Jaccard boost + results.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + } + } // Touch access for returned results for r in &results { From 362d2ecd851ebbbe8ca93038cdcca3f405d6b4f7 Mon Sep 17 00:00:00 2001 From: ajianaz Date: Fri, 17 Jul 2026 14:22:41 +0700 Subject: [PATCH 2/2] style: fix cargo fmt formatting (#723) --- crates/uteke-core/src/jaccard.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/uteke-core/src/jaccard.rs b/crates/uteke-core/src/jaccard.rs index e552995a..495ee9ee 100644 --- a/crates/uteke-core/src/jaccard.rs +++ b/crates/uteke-core/src/jaccard.rs @@ -31,7 +31,10 @@ pub fn jaccard_similarity(query: &HashSet, content: &HashSet) -> /// Empty tokens are discarded. pub fn tokenize(text: &str) -> HashSet { text.split_whitespace() - .map(|t| t.trim_matches(|c: char| !c.is_alphanumeric()).to_ascii_lowercase()) + .map(|t| { + t.trim_matches(|c: char| !c.is_alphanumeric()) + .to_ascii_lowercase() + }) .filter(|t| !t.is_empty()) .collect() } @@ -55,7 +58,10 @@ mod tests { #[test] fn jaccard_partial() { - let a: HashSet<_> = ["hello", "world", "foo"].iter().map(|s| s.to_string()).collect(); + let a: HashSet<_> = ["hello", "world", "foo"] + .iter() + .map(|s| s.to_string()) + .collect(); let b: HashSet<_> = ["hello", "bar"].iter().map(|s| s.to_string()).collect(); // intersection = {"hello"} = 1, union = {"hello","world","foo","bar"} = 4 let j = jaccard_similarity(&a, &b);