diff --git a/crates/ruvllm/src/claude_flow/model_router.rs b/crates/ruvllm/src/claude_flow/model_router.rs index db2da3870..d4c98b5a6 100644 --- a/crates/ruvllm/src/claude_flow/model_router.rs +++ b/crates/ruvllm/src/claude_flow/model_router.rs @@ -657,6 +657,26 @@ impl TaskComplexityAnalyzer { } } + /// Compute the signed calibration bias from feedback history. + /// + /// Returns `0.0` when fewer than 10 feedback records exist. + /// A positive value means the analyzer consistently over-predicts + /// complexity; negative means under-predicting. + pub fn calibration_bias(&self) -> f32 { + let with_feedback: Vec<_> = self + .accuracy_history + .iter() + .filter_map(|r| r.actual.map(|a| (r.predicted, a))) + .collect(); + + if with_feedback.len() < 10 { + return 0.0; + } + + let signed_error: f32 = with_feedback.iter().map(|(p, a)| p - a).sum(); + signed_error / with_feedback.len() as f32 + } + /// Get accuracy statistics pub fn accuracy_stats(&self) -> AnalyzerStats { let with_feedback: Vec<_> = self diff --git a/crates/ruvllm/src/intelligence/mod.rs b/crates/ruvllm/src/intelligence/mod.rs new file mode 100644 index 000000000..f24a51fa8 --- /dev/null +++ b/crates/ruvllm/src/intelligence/mod.rs @@ -0,0 +1,458 @@ +//! External Intelligence Providers for SONA Learning (ADR-029) +//! +//! This module defines the [`IntelligenceProvider`] trait — the extension point +//! that lets external systems feed quality signals into ruvllm's learning loops +//! without modifying ruvllm core. +//! +//! ## Quick start +//! +//! ```rust,ignore +//! use ruvllm::intelligence::{FileSignalProvider, IntelligenceProvider}; +//! use std::path::PathBuf; +//! +//! // File-based (non-Rust systems write JSON, Rust reads it) +//! let provider = FileSignalProvider::new(PathBuf::from("signals.json")); +//! let signals = provider.load_signals()?; +//! +//! // Custom provider +//! struct CiPipelineProvider; +//! impl IntelligenceProvider for CiPipelineProvider { +//! fn name(&self) -> &str { "ci-pipeline" } +//! fn load_signals(&self) -> ruvllm::Result> { +//! Ok(vec![]) +//! } +//! } +//! ``` + +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::error::Result; +use crate::RuvLLMError; + +// ============================================================================ +// Signal types +// ============================================================================ + +/// A quality signal from an external system. +/// +/// Represents one completed task with quality assessment data that can feed +/// into SONA trajectories, the embedding classifier, and model router +/// calibration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QualitySignal { + /// Unique identifier for this signal. + pub id: String, + /// Human-readable task description (used for embedding generation). + pub task_description: String, + /// Execution outcome: `"success"`, `"partial_success"`, or `"failure"`. + pub outcome: String, + /// Composite quality score (0.0–1.0). + pub quality_score: f32, + /// Optional human verdict: `"approved"`, `"rejected"`, or `None`. + #[serde(default)] + pub human_verdict: Option, + /// Optional structured quality factors for detailed analysis. + #[serde(default)] + pub quality_factors: Option, + /// ISO 8601 timestamp of task completion. + pub completed_at: String, +} + +/// Granular quality factor breakdown. +/// +/// Not all providers will have all factors. Fields default to `None`, +/// meaning "not assessed" (distinct from `Some(0.0)`, which means +/// "assessed as zero"). +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct QualityFactors { + pub acceptance_criteria_met: Option, + pub tests_passing: Option, + pub no_regressions: Option, + pub lint_clean: Option, + pub type_check_clean: Option, + pub follows_patterns: Option, + pub context_relevance: Option, + pub reasoning_coherence: Option, + pub execution_efficiency: Option, +} + +/// Quality weight overrides from a provider. +/// +/// Weights influence how the intelligence loader computes composite quality +/// for that provider's signals. They must sum to approximately 1.0. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QualityWeights { + pub task_completion: f32, + pub code_quality: f32, + pub process: f32, +} + +impl Default for QualityWeights { + fn default() -> Self { + Self { + task_completion: 0.5, + code_quality: 0.3, + process: 0.2, + } + } +} + +// ============================================================================ +// Provider trait +// ============================================================================ + +/// Extension point for external systems that supply quality signals to ruvllm. +/// +/// Implementations are registered with an intelligence loader and called +/// during signal ingestion. The loader handles mapping signals to SONA +/// trajectories, classifier entries, and router calibration data. +/// +/// # Design +/// +/// Follows the same trait pattern as [`LlmBackend`](crate::backends::LlmBackend) +/// and [`Tokenizer`](crate::backends::Tokenizer) — a trait object behind +/// `Box`. +pub trait IntelligenceProvider: Send + Sync { + /// Provider identity, used in logging and diagnostics. + fn name(&self) -> &str; + + /// Load quality signals from this provider's data source. + /// + /// Called once per ingestion cycle. Returns an empty `Vec` when no + /// signals are available (this is normal, not an error). + fn load_signals(&self) -> Result>; + + /// Optionally provide quality weight overrides. + /// + /// If `Some`, these weights are used when computing composite quality + /// for this provider's signals. If `None`, default weights apply. + fn quality_weights(&self) -> Option { + None + } +} + +// ============================================================================ +// Built-in file provider +// ============================================================================ + +/// Reads quality signals from a JSON file. +/// +/// This is the default provider for systems that write a signal file to +/// `.claude/intelligence/data/`. Non-Rust integrations (TypeScript, Python, +/// etc.) typically use this path. +/// +/// The expected JSON format is either: +/// - A JSON array of [`QualitySignal`] objects, or +/// - A JSON object with a `"signals"` key containing such an array. +pub struct FileSignalProvider { + path: PathBuf, +} + +impl FileSignalProvider { + pub fn new(path: PathBuf) -> Self { + Self { path } + } +} + +impl IntelligenceProvider for FileSignalProvider { + fn name(&self) -> &str { + "file-signals" + } + + fn load_signals(&self) -> Result> { + if !self.path.exists() { + return Ok(vec![]); // No file = no signals, not an error + } + parse_signal_file(&self.path) + } + + fn quality_weights(&self) -> Option { + let config_path = self + .path + .parent() + .unwrap_or(Path::new(".")) + .join("quality-weights.json"); + read_weight_config(&config_path).ok() + } +} + +// ============================================================================ +// Intelligence loader (provider registry) +// ============================================================================ + +/// Registry that collects signals from multiple [`IntelligenceProvider`]s. +/// +/// ```rust,ignore +/// let mut loader = IntelligenceProviderLoader::new(); +/// loader.register(Box::new(FileSignalProvider::new(path))); +/// let (signals, stats) = loader.load_all(); +/// ``` +pub struct IntelligenceProviderLoader { + providers: Vec>, +} + +/// Statistics from a `load_all()` run. +#[derive(Debug, Clone, Default)] +pub struct ProviderLoadStats { + /// Total signals successfully loaded across all providers. + pub total_signals: usize, + /// Per-provider signal counts (name, count). + pub per_provider: Vec<(String, usize)>, + /// Providers that errored (name, error message). + pub errors: Vec<(String, String)>, +} + +impl IntelligenceProviderLoader { + pub fn new() -> Self { + Self { + providers: Vec::new(), + } + } + + /// Register an external intelligence provider. + /// + /// Providers are called in registration order during [`load_all()`]. + pub fn register(&mut self, provider: Box) { + self.providers.push(provider); + } + + /// Number of registered providers. + pub fn provider_count(&self) -> usize { + self.providers.len() + } + + /// Load signals from all registered providers. + /// + /// Provider failures are non-fatal — they are recorded in + /// [`ProviderLoadStats::errors`] and the remaining providers continue. + pub fn load_all(&self) -> (Vec, ProviderLoadStats) { + let mut all_signals = Vec::new(); + let mut stats = ProviderLoadStats::default(); + + for provider in &self.providers { + match provider.load_signals() { + Ok(signals) => { + let count = signals.len(); + stats + .per_provider + .push((provider.name().to_string(), count)); + stats.total_signals += count; + all_signals.extend(signals); + tracing::info!("Provider '{}': loaded {} signals", provider.name(), count); + } + Err(e) => { + let msg = e.to_string(); + tracing::warn!("Provider '{}' failed: {}", provider.name(), msg); + stats + .errors + .push((provider.name().to_string(), msg)); + } + } + } + + (all_signals, stats) + } +} + +impl Default for IntelligenceProviderLoader { + fn default() -> Self { + Self::new() + } +} + +// ============================================================================ +// File parsing helpers +// ============================================================================ + +/// Wrapper for JSON files that may be a bare array or `{ "signals": [...] }`. +#[derive(Deserialize)] +#[serde(untagged)] +enum SignalFileFormat { + Array(Vec), + Wrapped { signals: Vec }, +} + +fn parse_signal_file(path: &Path) -> Result> { + let data = std::fs::read_to_string(path).map_err(|e| { + RuvLLMError::Config(format!("Failed to read signal file {}: {}", path.display(), e)) + })?; + let parsed: SignalFileFormat = serde_json::from_str(&data).map_err(|e| { + RuvLLMError::Config(format!( + "Failed to parse signal file {}: {}", + path.display(), + e + )) + })?; + Ok(match parsed { + SignalFileFormat::Array(v) => v, + SignalFileFormat::Wrapped { signals } => signals, + }) +} + +fn read_weight_config(path: &Path) -> Result { + let data = std::fs::read_to_string(path).map_err(|e| { + RuvLLMError::Config(format!( + "Failed to read weight config {}: {}", + path.display(), + e + )) + })?; + serde_json::from_str(&data).map_err(|e| { + RuvLLMError::Config(format!( + "Failed to parse weight config {}: {}", + path.display(), + e + )) + }) +} + +// ============================================================================ +// Tests +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + #[test] + fn test_file_provider_missing_file_returns_empty() { + let provider = FileSignalProvider::new(PathBuf::from("/nonexistent/signals.json")); + let signals = provider.load_signals().unwrap(); + assert!(signals.is_empty()); + } + + #[test] + fn test_file_provider_parses_array_format() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("signals.json"); + let mut f = std::fs::File::create(&path).unwrap(); + write!( + f, + r#"[{{"id":"s1","task_description":"test","outcome":"success","quality_score":0.9,"completed_at":"2026-01-01T00:00:00Z"}}]"# + ) + .unwrap(); + + let provider = FileSignalProvider::new(path); + let signals = provider.load_signals().unwrap(); + assert_eq!(signals.len(), 1); + assert_eq!(signals[0].id, "s1"); + assert!((signals[0].quality_score - 0.9).abs() < f32::EPSILON); + } + + #[test] + fn test_file_provider_parses_wrapped_format() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("signals.json"); + let mut f = std::fs::File::create(&path).unwrap(); + write!( + f, + r#"{{"signals":[{{"id":"s2","task_description":"lint","outcome":"failure","quality_score":0.2,"completed_at":"2026-01-01T00:00:00Z"}}]}}"# + ) + .unwrap(); + + let provider = FileSignalProvider::new(path); + let signals = provider.load_signals().unwrap(); + assert_eq!(signals.len(), 1); + assert_eq!(signals[0].outcome, "failure"); + } + + #[test] + fn test_quality_weights_default() { + let w = QualityWeights::default(); + let sum = w.task_completion + w.code_quality + w.process; + assert!((sum - 1.0).abs() < f32::EPSILON); + } + + #[test] + fn test_loader_no_providers() { + let loader = IntelligenceProviderLoader::new(); + let (signals, stats) = loader.load_all(); + assert!(signals.is_empty()); + assert_eq!(stats.total_signals, 0); + assert!(stats.errors.is_empty()); + } + + #[test] + fn test_loader_multiple_providers() { + struct MockProvider { + name: &'static str, + count: usize, + } + impl IntelligenceProvider for MockProvider { + fn name(&self) -> &str { + self.name + } + fn load_signals(&self) -> Result> { + Ok((0..self.count) + .map(|i| QualitySignal { + id: format!("{}-{}", self.name, i), + task_description: "test".into(), + outcome: "success".into(), + quality_score: 0.8, + human_verdict: None, + quality_factors: None, + completed_at: "2026-01-01T00:00:00Z".into(), + }) + .collect()) + } + } + + let mut loader = IntelligenceProviderLoader::new(); + loader.register(Box::new(MockProvider { + name: "ci", + count: 3, + })); + loader.register(Box::new(MockProvider { + name: "review", + count: 2, + })); + + let (signals, stats) = loader.load_all(); + assert_eq!(signals.len(), 5); + assert_eq!(stats.total_signals, 5); + assert_eq!(stats.per_provider.len(), 2); + } + + #[test] + fn test_loader_tolerates_provider_failure() { + struct FailProvider; + impl IntelligenceProvider for FailProvider { + fn name(&self) -> &str { + "broken" + } + fn load_signals(&self) -> Result> { + Err(RuvLLMError::Config("boom".into())) + } + } + + struct OkProvider; + impl IntelligenceProvider for OkProvider { + fn name(&self) -> &str { + "ok" + } + fn load_signals(&self) -> Result> { + Ok(vec![QualitySignal { + id: "good".into(), + task_description: "test".into(), + outcome: "success".into(), + quality_score: 1.0, + human_verdict: None, + quality_factors: None, + completed_at: "2026-01-01T00:00:00Z".into(), + }]) + } + } + + let mut loader = IntelligenceProviderLoader::new(); + loader.register(Box::new(FailProvider)); + loader.register(Box::new(OkProvider)); + + let (signals, stats) = loader.load_all(); + assert_eq!(signals.len(), 1); + assert_eq!(stats.errors.len(), 1); + assert_eq!(stats.errors[0].0, "broken"); + } +} diff --git a/crates/ruvllm/src/lib.rs b/crates/ruvllm/src/lib.rs index f6367f34e..8a1be3894 100644 --- a/crates/ruvllm/src/lib.rs +++ b/crates/ruvllm/src/lib.rs @@ -49,6 +49,7 @@ pub mod capabilities; pub mod claude_flow; pub mod context; pub mod error; +pub mod intelligence; pub mod evaluation; pub mod gguf; pub mod hub; @@ -96,6 +97,10 @@ pub use backends::{ }; #[cfg(feature = "async-runtime")] pub use backends::{AsyncTokenStream, LlmBackendAsync}; +pub use intelligence::{ + FileSignalProvider, IntelligenceProvider, IntelligenceProviderLoader, ProviderLoadStats, + QualityFactors, QualitySignal, QualityWeights as SignalQualityWeights, +}; pub use claude_flow::{ AgentContext, AgentCoordinator, diff --git a/docs/adr/ADR-029-external-intelligence-providers.md b/docs/adr/ADR-029-external-intelligence-providers.md new file mode 100644 index 000000000..f89e46da6 --- /dev/null +++ b/docs/adr/ADR-029-external-intelligence-providers.md @@ -0,0 +1,419 @@ +# ADR-029: External Intelligence Providers for SONA Learning + +**Status:** Accepted (Implemented) +**Date:** 2026-02-20 +**Authors:** @grparry +**Technical Area:** LLM Serving Runtime / External Signal Ingestion + +--- + +## Context and Problem Statement + +RuvLLM's learning loops — SONA trajectory recording, HNSW embedding classification, and model router calibration — depend on quality signals to distinguish good executions from bad ones. Today, those signals come from ruvllm's own inference pipeline: a request completes, a quality score is computed internally, and the score feeds back into the learning loops. + +This works when ruvllm is the entire system. But increasingly, ruvllm operates as one component within larger orchestration pipelines — workflow engines, CI/CD systems, coding assistants, multi-agent frameworks — where the *real* quality signal lives outside ruvllm. The external system knows whether the task actually met acceptance criteria, whether tests passed, whether the human reviewer approved or rejected the output. Ruvllm doesn't have access to any of that. + +### The gap + +ADR-002 established Ruvector as the unified memory layer and defined the Witness Log schema with `quality_score: f32`. ADR-CE-021 established that multiple systems (RuvLLM, Prime-Radiant) can contribute trajectories to a shared SONA instance. But neither ADR addresses **how external systems feed quality data in**. + +The result is that anyone integrating ruvllm into a larger system faces a choice: + +1. **Fork ruvllm** and add custom signal-loading code directly. This works but creates maintenance burden — every upstream update risks merge conflicts, and the custom code is scattered across internal modules rather than isolated behind a clean boundary. + +2. **Use hardcoded quality scores.** The default path today. Every execution gets the same assumed quality regardless of actual outcome. SONA can't distinguish great work from bad work. The embedding classifier has no quality gradient. The model router has no real calibration data. + +3. **Wait for a proper extension point.** This is what we're proposing. + +### Existing extension precedents in ruvllm + +Ruvllm already has well-designed trait-based extension points: + +| Trait | Purpose | Pattern | +|-------|---------|---------| +| `LlmBackend` | Pluggable inference backends | Trait + `SharedBackend` wrapper + factory | +| `WasmKernelPack` | Pluggable compute kernels | Trait + `KernelRegistry` | +| `Tokenizer` | Pluggable tokenization | Trait object behind `Option<&dyn Tokenizer>` | + +An intelligence provider follows the same pattern — a trait that external integrations implement, registered with the intelligence loader at startup. + +--- + +## Decision Drivers + +### Who benefits + +- **Workflow engine authors** (e.g., CI/CD pipelines, coding assistants, agentic frameworks) who orchestrate tasks through ruvllm and have rich quality data from test results, lint checks, human review +- **Multi-system deployments** where quality assessment happens in a different process or language than ruvllm (TypeScript, Python, Go services) +- **Federated setups** (per ADR-CE-021) where multiple signal sources contribute to shared SONA learning + +### Design constraints + +- **Zero overhead when unused.** If no providers are registered, ruvllm behaves exactly as it does today. No new file reads, no trait dispatch, no allocations. +- **File-based by default.** The simplest provider reads a JSON file from the existing `.claude/intelligence/data/` directory. No network calls, no database dependencies. +- **No automatic weight changes.** Providers supply signals; they do not modify routing weights or thresholds directly. Weight proposals are separate (generated by analysis tools, applied by human decision). +- **Backward compatible.** The existing `memory.json`, `trajectories.json`, and `patterns.json` loading continues unchanged. Providers are additive. + +--- + +## Considered Options + +### Option A: Direct File Convention (Status Quo Path) + +Any external system writes a JSON file to `.claude/intelligence/data/` in a specific format. The intelligence loader discovers and parses it by filename convention. + +**Pros:** +- No code changes needed in ruvllm beyond the initial file parser +- Simple for integrators to implement (just write a file) + +**Cons:** +- File format is rigid — every integrator must conform to one schema +- No way to customize how signals map to SONA trajectories, classifier labels, or router feedback +- Adding a new signal source means modifying `load_all()` in ruvllm's intelligence loader +- No validation or error reporting back to the provider + +### Option B: Trait-Based Intelligence Providers + +Define an `IntelligenceProvider` trait. External systems implement it (as a Rust crate, or via a thin adapter over a JSON file). The intelligence loader accepts registered providers and calls them during `load_all()`. + +**Pros:** +- Clean separation — providers are self-contained implementations behind a stable interface +- Providers control their own signal format, parsing, and mapping logic +- New providers can be added without modifying ruvllm core +- Follows established patterns (`LlmBackend`, `WasmKernelPack`) +- Testable in isolation + +**Cons:** +- Slightly more upfront design than a file convention +- Rust-only providers (non-Rust systems still write files, but need a Rust adapter to register) + +### Option C: gRPC / HTTP Signal Endpoint + +Ruvllm exposes an endpoint that external systems push signals to at runtime. + +**Pros:** +- Language-agnostic — any system can POST quality data +- Real-time ingestion + +**Cons:** +- Adds networking dependency to an edge-focused runtime +- Contradicts ruvllm's design philosophy (single binary, no external services) +- Security surface area (authentication, validation, DoS) +- Significantly more complex than file or trait approaches + +--- + +## Decision Outcome + +**Recommended Option: Option B — Trait-Based Intelligence Providers**, with a built-in file-based provider as the default implementation. + +This gives the extensibility of a trait interface while keeping the simplicity of file-based exchange for the common case. Non-Rust systems write a JSON file; a built-in `FileSignalProvider` reads it. Rust-native integrations can implement the trait directly for tighter control. + +### The trait + +```rust +use crate::error::Result; +use std::path::Path; + +/// A quality signal from an external system. +/// +/// Represents one completed task with quality assessment data +/// that can feed into SONA trajectories, the embedding classifier, +/// and model router calibration. +#[derive(Debug, Clone)] +pub struct QualitySignal { + /// Unique identifier for this signal + pub id: String, + /// Human-readable task description (used for embedding generation) + pub task_description: String, + /// Execution outcome: "success", "partial_success", "failure" + pub outcome: String, + /// Composite quality score (0.0 - 1.0) + pub quality_score: f32, + /// Optional human verdict: "approved", "rejected", or None + pub human_verdict: Option, + /// Optional structured quality factors for detailed analysis + pub quality_factors: Option, + /// ISO 8601 timestamp of task completion + pub completed_at: String, +} + +/// Granular quality factor breakdown. +/// +/// Not all providers will have all factors. Fields default to None, +/// meaning "not assessed" (distinct from 0.0, which means "assessed as zero"). +#[derive(Debug, Clone, Default)] +pub struct QualityFactors { + pub acceptance_criteria_met: Option, + pub tests_passing: Option, + pub no_regressions: Option, + pub lint_clean: Option, + pub type_check_clean: Option, + pub follows_patterns: Option, + pub context_relevance: Option, + pub reasoning_coherence: Option, + pub execution_efficiency: Option, +} + +/// Quality weight overrides from a provider. +/// +/// If a provider returns weights, they influence how the composite +/// quality score is computed from individual factors. Weights must +/// sum to approximately 1.0. +#[derive(Debug, Clone)] +pub struct QualityWeights { + pub task_completion: f32, + pub code_quality: f32, + pub process: f32, +} + +/// Trait for external systems that supply quality signals to ruvllm. +/// +/// Implementations are registered with `IntelligenceLoader` and called +/// during `load_all()`. The loader handles mapping signals to SONA +/// trajectories, classifier entries, and router calibration data. +/// +/// # Examples +/// +/// File-based provider (built-in): +/// ```rust +/// let provider = FileSignalProvider::new(data_dir.join("workflow-signals.json")); +/// loader.register_provider(Box::new(provider)); +/// ``` +/// +/// Custom provider: +/// ```rust +/// struct MyPipelineProvider { /* ... */ } +/// +/// impl IntelligenceProvider for MyPipelineProvider { +/// fn name(&self) -> &str { "my-pipeline" } +/// fn load_signals(&self) -> Result> { +/// // Read from your data source +/// } +/// } +/// ``` +pub trait IntelligenceProvider: Send + Sync { + /// Provider identity, used in logging and diagnostics. + fn name(&self) -> &str; + + /// Load quality signals from this provider's data source. + /// + /// Called once during `IntelligenceLoader::load_all()`. + /// Returns an empty vec if no signals are available (not an error). + fn load_signals(&self) -> Result>; + + /// Optionally provide quality weight overrides. + /// + /// If `Some`, these weights are used when the intelligence loader + /// computes composite quality for this provider's signals. + /// If `None`, the loader uses its default weights. + fn quality_weights(&self) -> Option { + None + } +} +``` + +### Built-in file provider + +```rust +/// Reads quality signals from a JSON file in the standard format. +/// +/// This is the default provider for systems that write a signal file +/// to `.claude/intelligence/data/`. Non-Rust integrations (TypeScript, +/// Python, etc.) typically use this path. +pub struct FileSignalProvider { + path: PathBuf, +} + +impl FileSignalProvider { + pub fn new(path: PathBuf) -> Self { + Self { path } + } +} + +impl IntelligenceProvider for FileSignalProvider { + fn name(&self) -> &str { + "file-signals" + } + + fn load_signals(&self) -> Result> { + if !self.path.exists() { + return Ok(vec![]); // No file = no signals, not an error + } + // Parse JSON, map to QualitySignal structs + parse_signal_file(&self.path) + } + + fn quality_weights(&self) -> Option { + // Check for quality-weights.json alongside the signal file + let config_path = self.path + .parent() + .unwrap_or(Path::new(".")) + .join("../config/quality-weights.json"); + read_weight_config(&config_path).ok() + } +} +``` + +### Integration with IntelligenceLoader + +```rust +impl IntelligenceLoader { + /// Register an external intelligence provider. + /// + /// Providers are called in registration order during `load_all()`. + pub fn register_provider(&mut self, provider: Box) { + self.providers.push(provider); + } +} +``` + +Inside `load_all()`, after the existing `memory.json` / `trajectories.json` / `patterns.json` loading: + +```rust +// Load signals from registered external providers +for provider in &self.providers { + match provider.load_signals() { + Ok(signals) => { + let weights = provider.quality_weights(); + let loaded = self.ingest_signals( + backend.sona(), + backend.embedding_classifier(), + &signals, + weights.as_ref(), + )?; + stats.provider_signals_loaded += loaded; + log::info!( + "Provider '{}': loaded {} signals", + provider.name(), + loaded + ); + } + Err(e) => { + log::warn!( + "Provider '{}' failed to load signals: {}", + provider.name(), + e + ); + // Non-fatal — other providers and built-in loading continue + } + } +} +``` + +### Model router calibration feedback + +One small addition to `TaskComplexityAnalyzer` enables providers to contribute calibration data: + +```rust +impl TaskComplexityAnalyzer { + /// Record quality feedback for calibration. + /// + /// Over time, this shifts the analyzer's predictions to match + /// observed outcomes. Positive bias = over-predicting complexity, + /// negative bias = under-predicting. + pub fn record_feedback( + &mut self, + predicted_complexity: f32, + actual_quality: f32, + model: ClaudeModel, + ) { + self.feedback_history.push_back(FeedbackEntry { + predicted: predicted_complexity, + actual: actual_quality, + model, + }); + if self.feedback_history.len() > self.max_history { + self.feedback_history.pop_front(); + } + } + + /// Returns signed calibration error. + /// Positive = systematically over-predicting, negative = under-predicting. + pub fn calibration_bias(&self) -> f32 { + if self.feedback_history.is_empty() { + return 0.0; + } + let sum: f32 = self.feedback_history + .iter() + .map(|f| f.predicted - f.actual) + .sum(); + sum / self.feedback_history.len() as f32 + } +} +``` + +This is a general-purpose improvement — useful for any feedback source, not specific to external providers. + +--- + +## Implementation Strategy + +### Phase 1: Trait and built-in provider (~80 lines) + +1. Add `QualitySignal`, `QualityFactors`, `QualityWeights` structs to a new `src/intelligence/mod.rs` module +2. Add `IntelligenceProvider` trait +3. Add `FileSignalProvider` as the built-in implementation +4. Add `register_provider()` and provider iteration to `IntelligenceLoader` +5. Add `record_feedback()` and `calibration_bias()` to `TaskComplexityAnalyzer` + +### Phase 2: Migrate existing signal loading (~30 lines) + +Refactor the current `load_workflow_signals()` method in `IntelligenceLoader` to use `FileSignalProvider` internally. This ensures the trait interface exercises the same code path as the existing implementation. + +### Phase 3: Feature flag (optional, ~5 lines) + +```toml +[features] +intelligence-providers = [] # Enables IntelligenceProvider trait and FileSignalProvider +``` + +If gated, the trait and built-in provider compile only when opted in. The rest of ruvllm is unaffected. + +--- + +## Consequences + +### Positive + +1. **Clean integration boundary.** External systems implement one trait instead of modifying ruvllm internals. Fork divergence drops to near zero. +2. **Follows established patterns.** Same approach as `LlmBackend` and `WasmKernelPack` — familiar to anyone who has extended ruvllm before. +3. **Language-agnostic in practice.** Non-Rust systems write a JSON file; `FileSignalProvider` reads it. No network calls, no IPC, no foreign function interface. +4. **Graceful when absent.** No providers registered = no behavior change. File missing = empty signal set, not an error. +5. **Testable.** Providers can be unit-tested independently. Mock providers simplify integration testing. + +### Negative + +1. **One more trait to maintain.** Though the surface area is small (two required methods, one optional). +2. **Rust adapter needed for custom providers.** Non-Rust systems must use the file path unless they write a Rust wrapper. (This is consistent with how `LlmBackend` works.) + +### What stays the same + +- **Built-in data loading.** `memory.json`, `trajectories.json`, `patterns.json` continue to load through existing code paths. Providers are additive. +- **SONA internals.** Trajectory recording, Q-learning, EWC++ — none of this changes. Providers just supply data that flows through the existing learning loops. +- **No automatic weight changes.** Providers can supply quality weights, but those weights are informational — they influence how the loader computes composite quality for that provider's signals. They do not change ruvllm's routing or threshold behavior. Weight adjustments remain a human decision. + +--- + +## Alternatives Considered But Not Proposed + +**Plugin dynamic loading (dlopen/libloading).** Maximum flexibility but adds unsafe code, platform-specific concerns, and ABI stability requirements. Too much complexity for the use case. If needed in the future, the trait interface can be extended to support it without breaking existing providers. + +**WASM-based providers.** Appealing for sandboxing, but signal loading is a startup-time operation reading trusted local files. WASM's sandboxing benefits don't apply, and the overhead isn't justified. + +--- + +## Related Decisions + +- **ADR-002**: RuvLLM Integration with Ruvector — defines the Witness Log schema with `quality_score: f32` and the learning feedback architecture +- **ADR-CE-021**: Shared SONA — establishes the precedent of multiple external systems contributing trajectories to SONA +- **ADR-004**: KV Cache Management — demonstrates the tiered, policy-driven approach that benefits from better calibration data + +--- + +## References + +1. RuvLLM `LlmBackend` trait: `crates/ruvllm/src/backends/mod.rs` +2. WASM Kernel Pack trait: ADR-002, "WASM Kernel Packs" section +3. SONA trajectory recording: `crates/ruvllm/src/sona/mod.rs` +4. Intelligence Loader: `crates/ruvllm/src/backends/intelligence_loader.rs`