From f4e6bc710daa2a822ccbdcc48b019455052f38ba Mon Sep 17 00:00:00 2001 From: ajianaz Date: Sat, 18 Jul 2026 10:26:44 +0700 Subject: [PATCH 1/3] feat(config): configurable dream pipeline thresholds (#731) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces hardcoded dream pipeline constants with configurable values. Changes: - DreamConfig struct (uteke-core + uteke-cli config) - contradict_similarity_threshold: 0.6 (cosine > this → NOT contradiction) - contradict_tag_jaccard_min: 0.4 (up from 0.3, reduces false positives) - contradict_max_memories: 200 (O(n²) scan limit) - dedup_threshold: 0.92 (cosine > this → merge candidate) - orphan_importance_threshold: 0.15 (safer than old 0.3) - MaintenanceConfig defaults changed to safer values: - auto_aging_enabled: true → false (opt-in) - auto_aging_interval_hours: 6 → 24 (daily) - auto_dream_interval_days: 3 → 7 (weekly) - DreamConfig wired into CLI main.rs and server main.rs via set_dream_config() - dream.rs reads from self.dream_config instead of const values - DEFAULT_ORPHAN_THRESHOLD updated 0.3 → 0.15 - Server reads [dream] section from uteke.toml Config example: [dream] contradict_similarity_threshold = 0.6 contradict_tag_jaccard_min = 0.4 contradict_max_memories = 200 dedup_threshold = 0.92 orphan_importance_threshold = 0.15 Closes #731 --- crates/uteke-cli/src/config.rs | 61 +++++++++++++++++++++++++++----- crates/uteke-cli/src/main.rs | 8 +++++ crates/uteke-core/src/dream.rs | 28 +++++++-------- crates/uteke-core/src/lib.rs | 46 ++++++++++++++++++++++++ crates/uteke-core/src/orphans.rs | 2 +- crates/uteke-server/src/main.rs | 44 ++++++++++++++++++++++- 6 files changed, 164 insertions(+), 25 deletions(-) diff --git a/crates/uteke-cli/src/config.rs b/crates/uteke-cli/src/config.rs index ddf42fbe..4b8f3d83 100644 --- a/crates/uteke-cli/src/config.rs +++ b/crates/uteke-cli/src/config.rs @@ -294,21 +294,55 @@ impl Default for ServerConfig { pub struct MaintenanceConfig { /// Enable auto-aging: periodically clean up cold, stale memories. pub auto_aging_enabled: bool, - /// Auto-aging interval in hours (default: 6). + /// Auto-aging interval in hours (default: 24). pub auto_aging_interval_hours: u64, /// Enable auto-dream: periodically run dream cycle (lint → dedup → orphans). pub auto_dream_enabled: bool, - /// Auto-dream interval in days (default: 3). + /// Auto-dream interval in days (default: 7). pub auto_dream_interval_days: u64, } impl Default for MaintenanceConfig { fn default() -> Self { Self { - auto_aging_enabled: true, - auto_aging_interval_hours: 6, + auto_aging_enabled: false, + auto_aging_interval_hours: 24, auto_dream_enabled: true, - auto_dream_interval_days: 3, + auto_dream_interval_days: 7, + } + } +} + +/// Dream pipeline thresholds (#731). All phases in the dream cycle +/// (contradiction scan, dedup/consolidate, orphan detection) read from +/// these values instead of hardcoded constants. +#[derive(serde::Deserialize, Clone, Copy)] +#[serde(default)] +pub struct DreamConfig { + /// Cosine similarity threshold for contradiction scan. Memories with + /// similarity ABOVE this value are NOT contradictions. Default: 0.6. + pub contradict_similarity_threshold: f32, + /// Minimum Jaccard index for tag overlap to consider contradiction. + /// Default: 0.4 (up from original 0.3 to reduce false positives). + pub contradict_tag_jaccard_min: f32, + /// Maximum memories loaded for O(n²) contradiction scan. Default: 200. + pub contradict_max_memories: usize, + /// Cosine similarity threshold for dedup/consolidate. Memories with + /// similarity ABOVE this value are merge candidates. Default: 0.92. + pub dedup_threshold: f32, + /// Importance threshold for orphan detection. Memories below this AND + /// with no edges are flagged as orphans. Default: 0.15 (safer than 0.3). + pub orphan_importance_threshold: f64, +} + +impl Default for DreamConfig { + fn default() -> Self { + Self { + contradict_similarity_threshold: 0.6, + contradict_tag_jaccard_min: 0.4, + contradict_max_memories: 200, + dedup_threshold: 0.92, + orphan_importance_threshold: 0.15, } } } @@ -328,6 +362,7 @@ pub struct Config { pub server: ServerConfig, pub limits: LimitsConfig, pub maintenance: MaintenanceConfig, + pub dream: DreamConfig, } /// Configurable limits (#404). @@ -795,10 +830,18 @@ impl Config { [maintenance] # Auto-maintenance daemon (runs in server background) -# auto_aging_enabled = true # Clean up stale memories -# auto_aging_interval_hours = 6 # Every 6 hours -# auto_dream_enabled = true # Run dream cycle (lint → dedup → orphans) -# auto_dream_interval_days = 3 # Every 3 days +# auto_aging_enabled = false # Opt-in — auto-delete should be explicit +# auto_aging_interval_hours = 24 # Daily (not every 6h) +# auto_dream_enabled = true # Run dream cycle (lint → dedup → orphans) +# auto_dream_interval_days = 7 # Weekly (not every 3d) + +[dream] +# Dream pipeline thresholds — all phases read from these values +# contradict_similarity_threshold = 0.6 # Cosine > this → NOT contradiction +# contradict_tag_jaccard_min = 0.4 # Tag overlap ≥ this → consider contradict +# contradict_max_memories = 200 # O(n²) scan limit +# dedup_threshold = 0.92 # Cosine > this → merge candidate +# orphan_importance_threshold = 0.15 # Importance < this → orphan candidate "#; std::fs::write(&config_path, default).ok(); } diff --git a/crates/uteke-cli/src/main.rs b/crates/uteke-cli/src/main.rs index d8b7d698..21e9c009 100644 --- a/crates/uteke-cli/src/main.rs +++ b/crates/uteke-cli/src/main.rs @@ -124,6 +124,14 @@ fn main() { Ok(mut u) => { // #719: apply Jaccard weight from config u.set_jaccard_weight(config.recall.jaccard_weight); + // #731: apply dream pipeline thresholds from config + u.set_dream_config(uteke_core::DreamConfig { + contradict_similarity_threshold: config.dream.contradict_similarity_threshold, + contradict_tag_jaccard_min: config.dream.contradict_tag_jaccard_min, + contradict_max_memories: config.dream.contradict_max_memories, + dedup_threshold: config.dream.dedup_threshold, + orphan_importance_threshold: config.dream.orphan_importance_threshold, + }); u } Err(e) => { diff --git a/crates/uteke-core/src/dream.rs b/crates/uteke-core/src/dream.rs index 0e977dc4..b84bd6fa 100644 --- a/crates/uteke-core/src/dream.rs +++ b/crates/uteke-core/src/dream.rs @@ -301,8 +301,8 @@ impl crate::Uteke { } fn phase_dedup(&self, namespace: Option<&str>, dry_run: bool) -> Result { - // Threshold 0.92 = very similar; tune down for more aggressive merges. - let result = self.consolidate(namespace, 0.92, dry_run)?; + // Threshold from config (#731) — very similar; tune down for more aggressive merges. + let result = self.consolidate(namespace, self.dream_config.dedup_threshold as f64, dry_run)?; let summary = if result.merged == 0 { format!( "✓ {} duplicate pairs found, 0 merged{}", @@ -342,13 +342,13 @@ impl crate::Uteke { namespace: Option<&str>, dry_run: bool, ) -> Result { - const SIMILARITY_THRESHOLD: f32 = 0.6; - const TAG_OVERLAP_MIN: usize = 1; - const TAG_JACCARD_MIN: f32 = 0.3; - const MAX_MEMORIES: usize = 200; + let similarity_threshold = self.dream_config.contradict_similarity_threshold; + let tag_overlap_min: usize = 1; + let tag_jaccard_min = self.dream_config.contradict_tag_jaccard_min; + let max_memories = self.dream_config.contradict_max_memories; // Load top-N memories ordered by updated_at DESC - let memories = self.load_recent_memories(namespace, MAX_MEMORIES)?; + let memories = self.load_recent_memories(namespace, max_memories)?; if memories.len() < 2 { return Ok(PhaseResult { @@ -388,14 +388,14 @@ impl crate::Uteke { continue; } let intersection = tags1.intersection(tags2).count(); - if intersection < TAG_OVERLAP_MIN { + if intersection < tag_overlap_min { continue; } // Tag Jaccard must be above minimum let union = tags1.union(tags2).count(); let tag_jaccard = intersection as f32 / union as f32; - if tag_jaccard < TAG_JACCARD_MIN { + if tag_jaccard < tag_jaccard_min { continue; } @@ -408,7 +408,7 @@ impl crate::Uteke { let cosine = crate::consolidate::cosine_similarity(&m1.embedding, &m2.embedding); // Low similarity = potential contradiction - if cosine > SIMILARITY_THRESHOLD { + if cosine > similarity_threshold { continue; } @@ -536,8 +536,8 @@ impl crate::Uteke { fn phase_orphans(&self, namespace: Option<&str>) -> Result { // Detection only — never auto-delete. Inline SQL count of memories // with no edges (incoming or outgoing), access_count=0, not pinned, - // and below the default threshold (#351 provides a richer API). - const THRESHOLD: f64 = 0.3; + // and below the configured threshold (#731). + let threshold = self.dream_config.orphan_importance_threshold; // Namespace-aware query: when a namespace is specified, only count // orphans in that namespace (CodeCora #390). let count: i64 = if let Some(ns) = namespace { @@ -553,7 +553,7 @@ impl crate::Uteke { AND m.deprecated = 0 AND m.importance < ?1 AND m.namespace = ?2", - rusqlite::params![THRESHOLD, ns], + rusqlite::params![threshold, ns], |r| r.get(0), ) } else { @@ -568,7 +568,7 @@ impl crate::Uteke { AND m.pinned = 0 AND m.deprecated = 0 AND m.importance < ?1", - rusqlite::params![THRESHOLD], + rusqlite::params![threshold], |r| r.get(0), ) } diff --git a/crates/uteke-core/src/lib.rs b/crates/uteke-core/src/lib.rs index 8c964bc2..897f56cc 100644 --- a/crates/uteke-core/src/lib.rs +++ b/crates/uteke-core/src/lib.rs @@ -176,6 +176,39 @@ impl Default for RecallConfig { } } +/// Dream pipeline thresholds (#731). All phases in the dream cycle +/// (contradiction scan, dedup/consolidate, orphan detection) read from +/// these values instead of hardcoded constants. +#[derive(Clone, Copy, Debug)] +pub struct DreamConfig { + /// Cosine similarity threshold for contradiction scan. Memories with + /// similarity ABOVE this value are NOT contradictions. Default: 0.6. + pub contradict_similarity_threshold: f32, + /// Minimum Jaccard index for tag overlap to consider contradiction. + /// Default: 0.4 (up from original 0.3 to reduce false positives). + pub contradict_tag_jaccard_min: f32, + /// Maximum memories loaded for O(n²) contradiction scan. Default: 200. + pub contradict_max_memories: usize, + /// Cosine similarity threshold for dedup/consolidate. Memories with + /// similarity ABOVE this value are merge candidates. Default: 0.92. + pub dedup_threshold: f32, + /// Importance threshold for orphan detection. Memories below this AND + /// with no edges are flagged as orphans. Default: 0.15 (safer than 0.3). + pub orphan_importance_threshold: f64, +} + +impl Default for DreamConfig { + fn default() -> Self { + Self { + contradict_similarity_threshold: 0.6, + contradict_tag_jaccard_min: 0.4, + contradict_max_memories: 200, + dedup_threshold: 0.92, + orphan_importance_threshold: 0.15, + } + } +} + /// Resolve uteke data directory. /// /// Uses `UTEKE_HOME` environment variable when set, otherwise falls back to @@ -328,6 +361,9 @@ pub struct Uteke { /// Jaccard token reranking weight (#719). Additive boost applied /// post-RRF based on query-content token overlap. Default 0.0 (off). jaccard_weight: f32, + /// Dream pipeline thresholds (#731). Used by contradiction scan, + /// dedup/consolidate, and orphan detection phases. + dream_config: DreamConfig, /// Recall cache — avoids redundant embedding computation for repeated queries. recall_cache: recall_cache::RecallCache, } @@ -481,6 +517,7 @@ impl Uteke { salience_recency_config: salience_recency::SalienceRecencyConfig::default(), recall_cache: recall_cache::RecallCache::new(recall_cache::RecallCacheConfig::default()), jaccard_weight: 0.0, + dream_config: DreamConfig::default(), }) } @@ -513,6 +550,15 @@ impl Uteke { self.jaccard_weight = weight.clamp(0.0, 1.0); } + /// Set dream pipeline thresholds (#731). + /// + /// Overrides the default hardcoded values for contradiction scan, + /// dedup/consolidate, and orphan detection phases. Called once at + /// startup from CLI/server config. + pub fn set_dream_config(&mut self, config: DreamConfig) { + self.dream_config = config; + } + /// Configure cloud embedding fallback. /// /// Must be called before the first embedding operation. When configured, diff --git a/crates/uteke-core/src/orphans.rs b/crates/uteke-core/src/orphans.rs index 2acc9efa..785dd0ce 100644 --- a/crates/uteke-core/src/orphans.rs +++ b/crates/uteke-core/src/orphans.rs @@ -17,7 +17,7 @@ use serde::{Deserialize, Serialize}; /// Default importance threshold below which a memory can be considered an /// orphan candidate (#351). -pub const DEFAULT_ORPHAN_THRESHOLD: f64 = 0.3; +pub const DEFAULT_ORPHAN_THRESHOLD: f64 = 0.15; /// A memory flagged as an orphan candidate (#351). #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/crates/uteke-server/src/main.rs b/crates/uteke-server/src/main.rs index c99a5a6f..673e7b81 100644 --- a/crates/uteke-server/src/main.rs +++ b/crates/uteke-server/src/main.rs @@ -208,7 +208,38 @@ fn main() { info!("Opening store at: {db_path}"); let uteke = match Uteke::open(&db_path) { - Ok(u) => Arc::new(Mutex::new(u)), + Ok(mut u) => { + // Apply dream pipeline thresholds from config (#731) + if let Some(dream_section) = &config.dream { + u.set_dream_config(uteke_core::DreamConfig { + contradict_similarity_threshold: dream_section + .contradict_similarity_threshold + .unwrap_or_default(), + contradict_tag_jaccard_min: dream_section + .contradict_tag_jaccard_min + .unwrap_or_default(), + contradict_max_memories: dream_section + .contradict_max_memories + .unwrap_or_default(), + dedup_threshold: dream_section + .dedup_threshold + .unwrap_or_default(), + orphan_importance_threshold: dream_section + .orphan_importance_threshold + .unwrap_or_default(), + }); + info!( + "Dream config loaded: dedup={:.2}, orphan_thresh={:.2}", + dream_section + .dedup_threshold + .unwrap_or(uteke_core::DreamConfig::default().dedup_threshold), + dream_section + .orphan_importance_threshold + .unwrap_or(uteke_core::DreamConfig::default().orphan_importance_threshold), + ); + } + Arc::new(Mutex::new(u)) + } Err(e) => { error!("Failed to open store: {e}"); std::process::exit(1); @@ -419,6 +450,7 @@ struct ServerFileConfig { extraction: Option, maintenance: Option, aging: Option, + dream: Option, } #[derive(serde::Deserialize, Default, Clone)] @@ -435,6 +467,16 @@ struct MaintenanceFileSection { auto_dream_interval_days: Option, } +/// Dream pipeline thresholds from uteke.toml [dream] section (#731). +#[derive(serde::Deserialize, Default, Clone)] +struct DreamFileSection { + contradict_similarity_threshold: Option, + contradict_tag_jaccard_min: Option, + contradict_max_memories: Option, + dedup_threshold: Option, + orphan_importance_threshold: Option, +} + #[derive(serde::Deserialize, Default)] struct ServerFileSection { host: Option, From a95648ce972ec2a92406e8d96f3e837366d4dbe8 Mon Sep 17 00:00:00 2001 From: ajianaz Date: Sat, 18 Jul 2026 20:27:13 +0700 Subject: [PATCH 2/3] fix: resolve type mismatch and format issues (#731) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - consolidate() expects f32 not f64 — remove incorrect cast - Server dream config: unwrap_or_default() for Option returns 0 (wrong), use unwrap_or(DreamConfig::default().*) instead - Format: restructure server dream config block for rustfmt compliance --- crates/uteke-core/src/dream.rs | 3 ++- crates/uteke-server/src/main.rs | 32 ++++++++++++++------------------ 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/crates/uteke-core/src/dream.rs b/crates/uteke-core/src/dream.rs index b84bd6fa..fd1cf7ff 100644 --- a/crates/uteke-core/src/dream.rs +++ b/crates/uteke-core/src/dream.rs @@ -302,7 +302,8 @@ impl crate::Uteke { fn phase_dedup(&self, namespace: Option<&str>, dry_run: bool) -> Result { // Threshold from config (#731) — very similar; tune down for more aggressive merges. - let result = self.consolidate(namespace, self.dream_config.dedup_threshold as f64, dry_run)?; + let result = + self.consolidate(namespace, self.dream_config.dedup_threshold, dry_run)?; let summary = if result.merged == 0 { format!( "✓ {} duplicate pairs found, 0 merged{}", diff --git a/crates/uteke-server/src/main.rs b/crates/uteke-server/src/main.rs index 673e7b81..7ebfe236 100644 --- a/crates/uteke-server/src/main.rs +++ b/crates/uteke-server/src/main.rs @@ -207,35 +207,31 @@ fn main() { let db_path = home.join("uteke.db").to_string_lossy().to_string(); info!("Opening store at: {db_path}"); + let defaults = uteke_core::DreamConfig::default(); let uteke = match Uteke::open(&db_path) { Ok(mut u) => { // Apply dream pipeline thresholds from config (#731) - if let Some(dream_section) = &config.dream { + if let Some(ref dc) = config.dream { u.set_dream_config(uteke_core::DreamConfig { - contradict_similarity_threshold: dream_section + contradict_similarity_threshold: dc .contradict_similarity_threshold - .unwrap_or_default(), - contradict_tag_jaccard_min: dream_section + .unwrap_or(defaults.contradict_similarity_threshold), + contradict_tag_jaccard_min: dc .contradict_tag_jaccard_min - .unwrap_or_default(), - contradict_max_memories: dream_section + .unwrap_or(defaults.contradict_tag_jaccard_min), + contradict_max_memories: dc .contradict_max_memories - .unwrap_or_default(), - dedup_threshold: dream_section - .dedup_threshold - .unwrap_or_default(), - orphan_importance_threshold: dream_section + .unwrap_or(defaults.contradict_max_memories), + dedup_threshold: dc.dedup_threshold.unwrap_or(defaults.dedup_threshold), + orphan_importance_threshold: dc .orphan_importance_threshold - .unwrap_or_default(), + .unwrap_or(defaults.orphan_importance_threshold), }); info!( "Dream config loaded: dedup={:.2}, orphan_thresh={:.2}", - dream_section - .dedup_threshold - .unwrap_or(uteke_core::DreamConfig::default().dedup_threshold), - dream_section - .orphan_importance_threshold - .unwrap_or(uteke_core::DreamConfig::default().orphan_importance_threshold), + dc.dedup_threshold.unwrap_or(defaults.dedup_threshold), + dc.orphan_importance_threshold + .unwrap_or(defaults.orphan_importance_threshold), ); } Arc::new(Mutex::new(u)) From 4e172498b8727276dddb196bc37281168c03daad Mon Sep 17 00:00:00 2001 From: ajianaz Date: Sat, 18 Jul 2026 20:33:01 +0700 Subject: [PATCH 3/3] fix(format): collapse consolidate call to single line per rustfmt --- crates/uteke-core/src/dream.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/uteke-core/src/dream.rs b/crates/uteke-core/src/dream.rs index fd1cf7ff..5c0537c4 100644 --- a/crates/uteke-core/src/dream.rs +++ b/crates/uteke-core/src/dream.rs @@ -302,8 +302,7 @@ impl crate::Uteke { fn phase_dedup(&self, namespace: Option<&str>, dry_run: bool) -> Result { // Threshold from config (#731) — very similar; tune down for more aggressive merges. - let result = - self.consolidate(namespace, self.dream_config.dedup_threshold, dry_run)?; + let result = self.consolidate(namespace, self.dream_config.dedup_threshold, dry_run)?; let summary = if result.merged == 0 { format!( "✓ {} duplicate pairs found, 0 merged{}",