feat: Jaccard token similarity as post-RRF reranking signal

crates/uteke-cli/src/config.rs

@@ -240,6 +240,10 @@ pub struct RecallConfig {
     /// Weight for the recency boost applied when the `--recency` flag is
     /// passed to `recall` (#352). 0.0 disables; 0.15 is the default.
     pub recency_weight: f32,
+    /// Weight for the Jaccard token reranking boost (#719).
+    /// Applied post-RRF as an additive signal based on query-content token
+    /// overlap. 0.0 disables (default); 0.10-0.15 recommended.
+    pub jaccard_weight: f32,
 }
 
 impl Default for RecallConfig {
@@ -254,6 +258,7 @@ impl Default for RecallConfig {
             graph_rerank_enabled: true,
             salience_weight: 0.15,
             recency_weight: 0.15,
+            jaccard_weight: 0.0,
         }
     }
 }
@@ -515,6 +520,9 @@ impl Config {
             if recall.contains_key("graph_rerank_enabled") {
                 self.recall.graph_rerank_enabled = overlay.recall.graph_rerank_enabled;
             }
+            if recall.contains_key("jaccard_weight") {
+                self.recall.jaccard_weight = overlay.recall.jaccard_weight;
+            }
         }
 
         // Merge server section
@@ -769,6 +777,7 @@ impl Config {
 # graph_density_weight = 0.1
 # graph_authority_weight = 0.1
 # graph_rerank_enabled = true
+# jaccard_weight = 0.0  # Post-RRF token overlap boost (#719). 0=off, 0.10-0.15 recommended
 
 [server]
 # enabled = false

crates/uteke-cli/src/main.rs

@@ -121,7 +121,11 @@ fn main() {
             enabled: config.recall.graph_rerank_enabled,
         },
     ) {
-        Ok(u) => u,
+        Ok(mut u) => {
+            // #719: apply Jaccard weight from config
+            u.set_jaccard_weight(config.recall.jaccard_weight);
+            u
+        }
         Err(e) => {
             eprintln!("Error: Failed to open memory store: {e}");
             std::process::exit(1);

crates/uteke-core/src/jaccard.rs

@@ -0,0 +1,116 @@
+//! Jaccard token similarity reranking signal (#719).
+//!
+//! Pure set-overlap signal that catches different cases from FTS5 BM25
+//! (which uses IDF-weighted scoring). Jaccard measures the ratio of shared
+//! tokens between query and content, independent of term rarity.
+//!
+//! Adopted from Hermes holographic memory analysis — Hermes uses Jaccard as
+//! a 0.3-weighted reranking signal in its hybrid fusion.
+
+use std::collections::HashSet;
+
+/// Compute Jaccard similarity between two token sets.
+///
+/// Returns `|A ∩ B| / |A ∪ B|` in `[0.0, 1.0]`.
+/// Returns 0.0 if both sets are empty.
+pub fn jaccard_similarity(query: &HashSet<String>, content: &HashSet<String>) -> f32 {
+    if query.is_empty() && content.is_empty() {
+        return 0.0;
+    }
+    let intersection = query.intersection(content).count();
+    let union = query.union(content).count();
+    if union == 0 {
+        return 0.0;
+    }
+    intersection as f32 / union as f32
+}
+
+/// Tokenize text into a lowercase HashSet.
+///
+/// Splits on whitespace and strips punctuation from each token.
+/// Empty tokens are discarded.
+pub fn tokenize(text: &str) -> HashSet<String> {
+    text.split_whitespace()
+        .map(|t| {
+            t.trim_matches(|c: char| !c.is_alphanumeric())
+                .to_ascii_lowercase()
+        })
+        .filter(|t| !t.is_empty())
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn jaccard_identical() {
+        let a: HashSet<_> = ["hello", "world"].iter().map(|s| s.to_string()).collect();
+        assert!((jaccard_similarity(&a, &a) - 1.0).abs() < f32::EPSILON);
+    }
+
+    #[test]
+    fn jaccard_disjoint() {
+        let a: HashSet<_> = ["hello", "world"].iter().map(|s| s.to_string()).collect();
+        let b: HashSet<_> = ["foo", "bar"].iter().map(|s| s.to_string()).collect();
+        assert!((jaccard_similarity(&a, &b) - 0.0).abs() < f32::EPSILON);
+    }
+
+    #[test]
+    fn jaccard_partial() {
+        let a: HashSet<_> = ["hello", "world", "foo"]
+            .iter()
+            .map(|s| s.to_string())
+            .collect();
+        let b: HashSet<_> = ["hello", "bar"].iter().map(|s| s.to_string()).collect();
+        // intersection = {"hello"} = 1, union = {"hello","world","foo","bar"} = 4
+        let j = jaccard_similarity(&a, &b);
+        assert!((j - 0.25).abs() < 0.01, "expected ~0.25, got {j}");
+    }
+
+    #[test]
+    fn jaccard_empty_both() {
+        let a: HashSet<String> = HashSet::new();
+        let b: HashSet<String> = HashSet::new();
+        assert!((jaccard_similarity(&a, &b) - 0.0).abs() < f32::EPSILON);
+    }
+
+    #[test]
+    fn tokenize_basic() {
+        let tokens = tokenize("Hello World foo");
+        assert!(tokens.contains("hello"));
+        assert!(tokens.contains("world"));
+        assert!(tokens.contains("foo"));
+        assert_eq!(tokens.len(), 3);
+    }
+
+    #[test]
+    fn tokenize_strips_punctuation() {
+        let tokens = tokenize("hello, world! (test)");
+        assert!(tokens.contains("hello"));
+        assert!(tokens.contains("world"));
+        assert!(tokens.contains("test"));
+    }
+
+    #[test]
+    fn tokenize_empty() {
+        let tokens = tokenize("");
+        assert!(tokens.is_empty());
+    }
+
+    #[test]
+    fn tokenize_whitespace_only() {
+        let tokens = tokenize("   \t\n  ");
+        assert!(tokens.is_empty());
+    }
+
+    #[test]
+    fn jaccard_with_tokenize() {
+        let q = tokenize("rust memory engine");
+        let c = tokenize("rust is a fast memory engine for AI");
+        let j = jaccard_similarity(&q, &c);
+        // intersection = {"rust", "memory", "engine"} = 3
+        // union = {"rust", "memory", "engine", "is", "a", "fast", "for", "ai"} = 8
+        assert!((j - 0.375).abs() < 0.01, "expected ~0.375, got {j}");
+    }
+}

crates/uteke-core/src/lib.rs

@@ -19,6 +19,7 @@ pub mod extraction;
 pub mod graph;
 pub mod graph_rerank;
 mod import_export;
+mod jaccard;
 mod maintenance;
 pub mod memory;
 mod operations;
@@ -324,6 +325,9 @@ pub struct Uteke {
     /// Salience + recency dual-axis boost config (#352). Defaults to all
     /// weights zero (opt-in per query via CLI flags / API params).
     salience_recency_config: salience_recency::SalienceRecencyConfig,
+    /// Jaccard token reranking weight (#719). Additive boost applied
+    /// post-RRF based on query-content token overlap. Default 0.0 (off).
+    jaccard_weight: f32,
     /// Recall cache — avoids redundant embedding computation for repeated queries.
     recall_cache: recall_cache::RecallCache,
 }
@@ -476,6 +480,7 @@ impl Uteke {
             graph_rerank_config: graph_rerank_config.sanitized(),
             salience_recency_config: salience_recency::SalienceRecencyConfig::default(),
             recall_cache: recall_cache::RecallCache::new(recall_cache::RecallCacheConfig::default()),
+            jaccard_weight: 0.0,
         })
     }
 
@@ -500,6 +505,14 @@ impl Uteke {
         self.salience_recency_config = salience_recency::SalienceRecencyConfig::default();
     }
 
+    /// Set Jaccard token reranking weight (#719).
+    ///
+    /// When > 0.0, an additive Jaccard similarity boost is applied post-RRF
+    /// based on query-content token overlap. Recommended: 0.10-0.15.
+    pub fn set_jaccard_weight(&mut self, weight: f32) {
+        self.jaccard_weight = weight.clamp(0.0, 1.0);
+    }
+
     /// Configure cloud embedding fallback.
     ///
     /// Must be called before the first embedding operation. When configured,

crates/uteke-core/src/operations.rs

@@ -808,11 +808,19 @@ impl crate::Uteke {
                 .or_insert_with(|| memory.clone());
         }
 
-        // Sort by RRF score descending, take top `limit`
+        // NOTE: min_score is NOT applied here. RRF normalized scores are
+        // rank-based (0..1) and not directly comparable to cosine similarity.
+        // Applying a cosine threshold to RRF scores would incorrectly filter
+        // out valid results. The caller (recall_hybrid) handles threshold
+        // filtering at the appropriate level.
+
+        // #719: Jaccard token reranking boost (post-RRF, additive).
+        // Measures query-content token overlap as an orthogonal signal
+        // to BM25 (IDF-weighted) and vector cosine (semantic).
         let mut scored: Vec<(String, f64)> = rrf_scores.into_iter().collect();
         scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
 
-        let results: Vec<SearchResult> = scored
+        let mut results: Vec<SearchResult> = scored
             .into_iter()
             .take(limit)
             .map(|(id, score)| {
@@ -831,11 +839,26 @@ impl crate::Uteke {
             })
             .collect();
 
-        // NOTE: min_score is NOT applied here. RRF normalized scores are
-        // rank-based (0..1) and not directly comparable to cosine similarity.
-        // Applying a cosine threshold to RRF scores would incorrectly filter
-        // out valid results. The caller (recall_hybrid) handles threshold
-        // filtering at the appropriate level.
+        if self.jaccard_weight > 0.0 {
+            let query_tokens = crate::jaccard::tokenize(query);
+            if !query_tokens.is_empty() {
+                for sr in &mut results {
+                    // Tokenize content + tags for a richer overlap signal
+                    let mut content_tokens = crate::jaccard::tokenize(&sr.memory.content);
+                    for tag in &sr.memory.tags {
+                        content_tokens.insert(tag.to_ascii_lowercase());
+                    }
+                    let j = crate::jaccard::jaccard_similarity(&query_tokens, &content_tokens);
+                    sr.score += j * self.jaccard_weight;
+                }
+                // Re-sort after Jaccard boost
+                results.sort_by(|a, b| {
+                    b.score
+                        .partial_cmp(&a.score)
+                        .unwrap_or(std::cmp::Ordering::Equal)
+                });
+            }
+        }
 
         // Touch access for returned results
         for r in &results {