diff --git a/crates/uteke-cli/src/config.rs b/crates/uteke-cli/src/config.rs index ddf42fb..4b8f3d8 100644 --- a/crates/uteke-cli/src/config.rs +++ b/crates/uteke-cli/src/config.rs @@ -294,21 +294,55 @@ impl Default for ServerConfig { pub struct MaintenanceConfig { /// Enable auto-aging: periodically clean up cold, stale memories. pub auto_aging_enabled: bool, - /// Auto-aging interval in hours (default: 6). + /// Auto-aging interval in hours (default: 24). pub auto_aging_interval_hours: u64, /// Enable auto-dream: periodically run dream cycle (lint → dedup → orphans). pub auto_dream_enabled: bool, - /// Auto-dream interval in days (default: 3). + /// Auto-dream interval in days (default: 7). pub auto_dream_interval_days: u64, } impl Default for MaintenanceConfig { fn default() -> Self { Self { - auto_aging_enabled: true, - auto_aging_interval_hours: 6, + auto_aging_enabled: false, + auto_aging_interval_hours: 24, auto_dream_enabled: true, - auto_dream_interval_days: 3, + auto_dream_interval_days: 7, + } + } +} + +/// Dream pipeline thresholds (#731). All phases in the dream cycle +/// (contradiction scan, dedup/consolidate, orphan detection) read from +/// these values instead of hardcoded constants. +#[derive(serde::Deserialize, Clone, Copy)] +#[serde(default)] +pub struct DreamConfig { + /// Cosine similarity threshold for contradiction scan. Memories with + /// similarity ABOVE this value are NOT contradictions. Default: 0.6. + pub contradict_similarity_threshold: f32, + /// Minimum Jaccard index for tag overlap to consider contradiction. + /// Default: 0.4 (up from original 0.3 to reduce false positives). + pub contradict_tag_jaccard_min: f32, + /// Maximum memories loaded for O(n²) contradiction scan. Default: 200. + pub contradict_max_memories: usize, + /// Cosine similarity threshold for dedup/consolidate. Memories with + /// similarity ABOVE this value are merge candidates. Default: 0.92. + pub dedup_threshold: f32, + /// Importance threshold for orphan detection. Memories below this AND + /// with no edges are flagged as orphans. Default: 0.15 (safer than 0.3). + pub orphan_importance_threshold: f64, +} + +impl Default for DreamConfig { + fn default() -> Self { + Self { + contradict_similarity_threshold: 0.6, + contradict_tag_jaccard_min: 0.4, + contradict_max_memories: 200, + dedup_threshold: 0.92, + orphan_importance_threshold: 0.15, } } } @@ -328,6 +362,7 @@ pub struct Config { pub server: ServerConfig, pub limits: LimitsConfig, pub maintenance: MaintenanceConfig, + pub dream: DreamConfig, } /// Configurable limits (#404). @@ -795,10 +830,18 @@ impl Config { [maintenance] # Auto-maintenance daemon (runs in server background) -# auto_aging_enabled = true # Clean up stale memories -# auto_aging_interval_hours = 6 # Every 6 hours -# auto_dream_enabled = true # Run dream cycle (lint → dedup → orphans) -# auto_dream_interval_days = 3 # Every 3 days +# auto_aging_enabled = false # Opt-in — auto-delete should be explicit +# auto_aging_interval_hours = 24 # Daily (not every 6h) +# auto_dream_enabled = true # Run dream cycle (lint → dedup → orphans) +# auto_dream_interval_days = 7 # Weekly (not every 3d) + +[dream] +# Dream pipeline thresholds — all phases read from these values +# contradict_similarity_threshold = 0.6 # Cosine > this → NOT contradiction +# contradict_tag_jaccard_min = 0.4 # Tag overlap ≥ this → consider contradict +# contradict_max_memories = 200 # O(n²) scan limit +# dedup_threshold = 0.92 # Cosine > this → merge candidate +# orphan_importance_threshold = 0.15 # Importance < this → orphan candidate "#; std::fs::write(&config_path, default).ok(); } diff --git a/crates/uteke-cli/src/main.rs b/crates/uteke-cli/src/main.rs index d8b7d69..21e9c00 100644 --- a/crates/uteke-cli/src/main.rs +++ b/crates/uteke-cli/src/main.rs @@ -124,6 +124,14 @@ fn main() { Ok(mut u) => { // #719: apply Jaccard weight from config u.set_jaccard_weight(config.recall.jaccard_weight); + // #731: apply dream pipeline thresholds from config + u.set_dream_config(uteke_core::DreamConfig { + contradict_similarity_threshold: config.dream.contradict_similarity_threshold, + contradict_tag_jaccard_min: config.dream.contradict_tag_jaccard_min, + contradict_max_memories: config.dream.contradict_max_memories, + dedup_threshold: config.dream.dedup_threshold, + orphan_importance_threshold: config.dream.orphan_importance_threshold, + }); u } Err(e) => { diff --git a/crates/uteke-core/src/dream.rs b/crates/uteke-core/src/dream.rs index 0e977dc..5c0537c 100644 --- a/crates/uteke-core/src/dream.rs +++ b/crates/uteke-core/src/dream.rs @@ -301,8 +301,8 @@ impl crate::Uteke { } fn phase_dedup(&self, namespace: Option<&str>, dry_run: bool) -> Result { - // Threshold 0.92 = very similar; tune down for more aggressive merges. - let result = self.consolidate(namespace, 0.92, dry_run)?; + // Threshold from config (#731) — very similar; tune down for more aggressive merges. + let result = self.consolidate(namespace, self.dream_config.dedup_threshold, dry_run)?; let summary = if result.merged == 0 { format!( "✓ {} duplicate pairs found, 0 merged{}", @@ -342,13 +342,13 @@ impl crate::Uteke { namespace: Option<&str>, dry_run: bool, ) -> Result { - const SIMILARITY_THRESHOLD: f32 = 0.6; - const TAG_OVERLAP_MIN: usize = 1; - const TAG_JACCARD_MIN: f32 = 0.3; - const MAX_MEMORIES: usize = 200; + let similarity_threshold = self.dream_config.contradict_similarity_threshold; + let tag_overlap_min: usize = 1; + let tag_jaccard_min = self.dream_config.contradict_tag_jaccard_min; + let max_memories = self.dream_config.contradict_max_memories; // Load top-N memories ordered by updated_at DESC - let memories = self.load_recent_memories(namespace, MAX_MEMORIES)?; + let memories = self.load_recent_memories(namespace, max_memories)?; if memories.len() < 2 { return Ok(PhaseResult { @@ -388,14 +388,14 @@ impl crate::Uteke { continue; } let intersection = tags1.intersection(tags2).count(); - if intersection < TAG_OVERLAP_MIN { + if intersection < tag_overlap_min { continue; } // Tag Jaccard must be above minimum let union = tags1.union(tags2).count(); let tag_jaccard = intersection as f32 / union as f32; - if tag_jaccard < TAG_JACCARD_MIN { + if tag_jaccard < tag_jaccard_min { continue; } @@ -408,7 +408,7 @@ impl crate::Uteke { let cosine = crate::consolidate::cosine_similarity(&m1.embedding, &m2.embedding); // Low similarity = potential contradiction - if cosine > SIMILARITY_THRESHOLD { + if cosine > similarity_threshold { continue; } @@ -536,8 +536,8 @@ impl crate::Uteke { fn phase_orphans(&self, namespace: Option<&str>) -> Result { // Detection only — never auto-delete. Inline SQL count of memories // with no edges (incoming or outgoing), access_count=0, not pinned, - // and below the default threshold (#351 provides a richer API). - const THRESHOLD: f64 = 0.3; + // and below the configured threshold (#731). + let threshold = self.dream_config.orphan_importance_threshold; // Namespace-aware query: when a namespace is specified, only count // orphans in that namespace (CodeCora #390). let count: i64 = if let Some(ns) = namespace { @@ -553,7 +553,7 @@ impl crate::Uteke { AND m.deprecated = 0 AND m.importance < ?1 AND m.namespace = ?2", - rusqlite::params![THRESHOLD, ns], + rusqlite::params![threshold, ns], |r| r.get(0), ) } else { @@ -568,7 +568,7 @@ impl crate::Uteke { AND m.pinned = 0 AND m.deprecated = 0 AND m.importance < ?1", - rusqlite::params![THRESHOLD], + rusqlite::params![threshold], |r| r.get(0), ) } diff --git a/crates/uteke-core/src/lib.rs b/crates/uteke-core/src/lib.rs index 8c964bc..897f56c 100644 --- a/crates/uteke-core/src/lib.rs +++ b/crates/uteke-core/src/lib.rs @@ -176,6 +176,39 @@ impl Default for RecallConfig { } } +/// Dream pipeline thresholds (#731). All phases in the dream cycle +/// (contradiction scan, dedup/consolidate, orphan detection) read from +/// these values instead of hardcoded constants. +#[derive(Clone, Copy, Debug)] +pub struct DreamConfig { + /// Cosine similarity threshold for contradiction scan. Memories with + /// similarity ABOVE this value are NOT contradictions. Default: 0.6. + pub contradict_similarity_threshold: f32, + /// Minimum Jaccard index for tag overlap to consider contradiction. + /// Default: 0.4 (up from original 0.3 to reduce false positives). + pub contradict_tag_jaccard_min: f32, + /// Maximum memories loaded for O(n²) contradiction scan. Default: 200. + pub contradict_max_memories: usize, + /// Cosine similarity threshold for dedup/consolidate. Memories with + /// similarity ABOVE this value are merge candidates. Default: 0.92. + pub dedup_threshold: f32, + /// Importance threshold for orphan detection. Memories below this AND + /// with no edges are flagged as orphans. Default: 0.15 (safer than 0.3). + pub orphan_importance_threshold: f64, +} + +impl Default for DreamConfig { + fn default() -> Self { + Self { + contradict_similarity_threshold: 0.6, + contradict_tag_jaccard_min: 0.4, + contradict_max_memories: 200, + dedup_threshold: 0.92, + orphan_importance_threshold: 0.15, + } + } +} + /// Resolve uteke data directory. /// /// Uses `UTEKE_HOME` environment variable when set, otherwise falls back to @@ -328,6 +361,9 @@ pub struct Uteke { /// Jaccard token reranking weight (#719). Additive boost applied /// post-RRF based on query-content token overlap. Default 0.0 (off). jaccard_weight: f32, + /// Dream pipeline thresholds (#731). Used by contradiction scan, + /// dedup/consolidate, and orphan detection phases. + dream_config: DreamConfig, /// Recall cache — avoids redundant embedding computation for repeated queries. recall_cache: recall_cache::RecallCache, } @@ -481,6 +517,7 @@ impl Uteke { salience_recency_config: salience_recency::SalienceRecencyConfig::default(), recall_cache: recall_cache::RecallCache::new(recall_cache::RecallCacheConfig::default()), jaccard_weight: 0.0, + dream_config: DreamConfig::default(), }) } @@ -513,6 +550,15 @@ impl Uteke { self.jaccard_weight = weight.clamp(0.0, 1.0); } + /// Set dream pipeline thresholds (#731). + /// + /// Overrides the default hardcoded values for contradiction scan, + /// dedup/consolidate, and orphan detection phases. Called once at + /// startup from CLI/server config. + pub fn set_dream_config(&mut self, config: DreamConfig) { + self.dream_config = config; + } + /// Configure cloud embedding fallback. /// /// Must be called before the first embedding operation. When configured, diff --git a/crates/uteke-core/src/orphans.rs b/crates/uteke-core/src/orphans.rs index 2acc9ef..785dd0c 100644 --- a/crates/uteke-core/src/orphans.rs +++ b/crates/uteke-core/src/orphans.rs @@ -17,7 +17,7 @@ use serde::{Deserialize, Serialize}; /// Default importance threshold below which a memory can be considered an /// orphan candidate (#351). -pub const DEFAULT_ORPHAN_THRESHOLD: f64 = 0.3; +pub const DEFAULT_ORPHAN_THRESHOLD: f64 = 0.15; /// A memory flagged as an orphan candidate (#351). #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/crates/uteke-server/src/main.rs b/crates/uteke-server/src/main.rs index c99a5a6..7ebfe23 100644 --- a/crates/uteke-server/src/main.rs +++ b/crates/uteke-server/src/main.rs @@ -207,8 +207,35 @@ fn main() { let db_path = home.join("uteke.db").to_string_lossy().to_string(); info!("Opening store at: {db_path}"); + let defaults = uteke_core::DreamConfig::default(); let uteke = match Uteke::open(&db_path) { - Ok(u) => Arc::new(Mutex::new(u)), + Ok(mut u) => { + // Apply dream pipeline thresholds from config (#731) + if let Some(ref dc) = config.dream { + u.set_dream_config(uteke_core::DreamConfig { + contradict_similarity_threshold: dc + .contradict_similarity_threshold + .unwrap_or(defaults.contradict_similarity_threshold), + contradict_tag_jaccard_min: dc + .contradict_tag_jaccard_min + .unwrap_or(defaults.contradict_tag_jaccard_min), + contradict_max_memories: dc + .contradict_max_memories + .unwrap_or(defaults.contradict_max_memories), + dedup_threshold: dc.dedup_threshold.unwrap_or(defaults.dedup_threshold), + orphan_importance_threshold: dc + .orphan_importance_threshold + .unwrap_or(defaults.orphan_importance_threshold), + }); + info!( + "Dream config loaded: dedup={:.2}, orphan_thresh={:.2}", + dc.dedup_threshold.unwrap_or(defaults.dedup_threshold), + dc.orphan_importance_threshold + .unwrap_or(defaults.orphan_importance_threshold), + ); + } + Arc::new(Mutex::new(u)) + } Err(e) => { error!("Failed to open store: {e}"); std::process::exit(1); @@ -419,6 +446,7 @@ struct ServerFileConfig { extraction: Option, maintenance: Option, aging: Option, + dream: Option, } #[derive(serde::Deserialize, Default, Clone)] @@ -435,6 +463,16 @@ struct MaintenanceFileSection { auto_dream_interval_days: Option, } +/// Dream pipeline thresholds from uteke.toml [dream] section (#731). +#[derive(serde::Deserialize, Default, Clone)] +struct DreamFileSection { + contradict_similarity_threshold: Option, + contradict_tag_jaccard_min: Option, + contradict_max_memories: Option, + dedup_threshold: Option, + orphan_importance_threshold: Option, +} + #[derive(serde::Deserialize, Default)] struct ServerFileSection { host: Option,