diff --git a/apps/el-chat/src/main.rs b/apps/el-chat/src/main.rs index 30efb36..9ee041d 100644 --- a/apps/el-chat/src/main.rs +++ b/apps/el-chat/src/main.rs @@ -243,6 +243,13 @@ fn main() { continue; } "reset" => { + // Release the prior conversation's KV BEFORE reporting success. + // If it can't be cleared, stop rather than fail open (the + // provider drops on exit, freeing the KV via ownership). + if !release_session(&provider) { + eprintln!("(ending session to release conversation memory)"); + break; + } history = vec![ChatMessage::system(&args.system)]; eprintln!("(conversation reset)"); continue; @@ -251,10 +258,14 @@ fn main() { let new_sys = parts.next().unwrap_or("").trim(); if new_sys.is_empty() { eprintln!("(usage: /system )"); - } else { - history = vec![ChatMessage::system(new_sys)]; - eprintln!("(system prompt updated; conversation reset)"); + continue; + } + if !release_session(&provider) { + eprintln!("(ending session to release conversation memory)"); + break; } + history = vec![ChatMessage::system(new_sys)]; + eprintln!("(system prompt updated; conversation reset)"); continue; } other => { @@ -271,6 +282,10 @@ fn main() { Ok(reply) => history.push(ChatMessage::assistant(reply)), Err(e) => { eprintln!("\n(generation error: {e}; conversation reset)"); + if !release_session(&provider) { + eprintln!("(ending session to release conversation memory)"); + break; + } history = vec![ChatMessage::system(&args.system)]; } } @@ -278,6 +293,30 @@ fn main() { eprintln!("bye."); } +/// Release the provider's resident conversation memory — KV (evicted in the +/// engine), prompt, output, and buffered events — while keeping the model loaded +/// (ADR-018; PRD "KV caches … cleared on session end"). Used by `/reset`, +/// `/system`, and generation-error recovery so a discarded conversation does not +/// linger in the engine until the next turn. +/// +/// Retries once, then returns `false` if the conversation could **not** be +/// cleared. Callers must then stop rather than fail open (continue and report a +/// successful reset with user K/V possibly still resident): on `false` they break +/// the loop so `main` returns and the provider drops, freeing the KV via ownership. +fn release_session(provider: &QwenChatProvider) -> bool { + if provider.end_session().is_ok() { + return true; + } + // One retry for a transient failure. + match provider.end_session() { + Ok(()) => true, + Err(e) => { + eprintln!("(error: could not clear conversation memory: {e})"); + false + } + } +} + /// Stream one assistant reply to stdout via the SDK's `LlmProvider::chat_stream`; /// returns the accumulated text so the caller can append it to history. fn run_turn(provider: &QwenChatProvider, req: &ChatRequest) -> el_core::Result { diff --git a/crates/adapters/el-engine-candle/README.md b/crates/adapters/el-engine-candle/README.md index e984993..e8f2527 100644 --- a/crates/adapters/el-engine-candle/README.md +++ b/crates/adapters/el-engine-candle/README.md @@ -47,11 +47,14 @@ println!("{}", reply.content); # Ok::<(), el_core::EdgeError>(()) ``` -Each `chat` call builds a fresh `QwenEngine` (Candle exposes no public KV-cache -reset) and runs the standard SDK path: provenance permit → `load_prompt` -(prefill) → `generate` (grammar mask → safety steer → chunk-guard + checkpointed -rollback → greedy commit). Decoding is deterministic greedy argmax, so replies -are reproducible. +The model weights load **once** in `from_paths` and stay resident; each `chat` +reuses one persistent provenance-gated session (ADR-018) — reset (which evicts the +previous conversation's KV from Candle's cache via a position-0 forward, keeping +the weights loaded) → `load_prompt` (prefill) → `generate` (grammar mask → safety +steer → chunk-guard + checkpointed rollback → greedy commit). No per-turn reload +from disk. Decoding is deterministic greedy argmax, so replies are reproducible. +`end_session()` releases a conversation's memory while keeping the model resident; +dropping the provider frees the weights too. ### On-device safety (ADR-005 + ADR-012) @@ -98,9 +101,11 @@ signature before issuing the permit. The chat model as its own expert is a ## Benchmark instrumentation Setting `EL_BENCH=1` makes `QwenChatProvider::chat` print a per-phase breakdown -(model load / tokenize / prefill / decode / detokenize) plus per-forward -attribution (model compute vs. seam quantisation vs. runtime loop) to stderr. -It is zero-cost when unset and is a diagnostic only — not part of public behaviour. +(session setup / tokenize / prefill / decode / detokenize) plus per-forward +attribution (model compute vs. seam quantisation vs. runtime loop) to stderr. The +weights load once at startup (ADR-018), so per-turn "session setup" is near-zero — +not a per-chat model load. Zero-cost when unset; a diagnostic only, not part of +public behaviour. ## Features & dependencies diff --git a/crates/adapters/el-engine-candle/src/lib.rs b/crates/adapters/el-engine-candle/src/lib.rs index aae630c..d3f360c 100644 --- a/crates/adapters/el-engine-candle/src/lib.rs +++ b/crates/adapters/el-engine-candle/src/lib.rs @@ -179,6 +179,11 @@ impl InferenceEngine for CandleEngine { fn rollback(&mut self, _keep_committed: u32) -> Result<()> { Ok(()) } + + /// Stateless: no conversation cache to discard between turns (ADR-018). + fn reset_cache(&mut self) -> Result<()> { + Ok(()) + } } // ── LlmProvider (text-level) wrapper (ADR-010) ─────────────────────────────── @@ -261,7 +266,8 @@ impl LlmProvider for LocalLlmProvider { let max = req.max_tokens.unwrap_or(64); let mut session = self.session.lock().unwrap(); - session.reset(); + session.reset()?; + let _ = session.drain_events(); // bound buffered events across reused turns let ports = Ports::permissive(); session.load_prompt(&ports, &prompt_tokens)?; session.generate(&ports, max)?; @@ -351,8 +357,12 @@ mod bench { /// A real Qwen2 transformer `InferenceEngine`. /// /// Holds candle's stateful KV cache. Within one generation it is fed -/// incrementally (prefill, then one new token per `next_logits` call); candle -/// exposes no public cache reset, so a *fresh conversation* builds a new engine. +/// incrementally (prefill, then one new token per `next_logits` call). The engine +/// is **loaded once and reused across conversations** (ADR-018): candle exposes no +/// public cache-clear, but its attention *replaces* the cache on a forward at +/// `index_pos == 0`, so [`reset_cache`](InferenceEngine::reset_cache) evicts the +/// previous conversation's KV with a single benign position-0 forward — no engine +/// reconstruction and no reload from disk between turns. /// /// A *within-generation* safety backtrack (ADR-012) is supported via /// [`InferenceEngine::rollback`]: candle's attention discards its cache when a @@ -375,6 +385,12 @@ pub struct QwenEngine { last_logits: Vec, vocab: usize, eos: Token, + /// Whether candle's per-layer KV cache may hold conversation-derived K/V that + /// still needs clearing (ADR-018). Set by every `forward_one` (before the + /// fallible model forward) and cleared **only** after a fully successful + /// eviction forward in `reset_cache`, so a partially-failed eviction is retried + /// rather than skipped — `index_pos` alone can't carry that signal. + cache_dirty: bool, } impl QwenEngine { @@ -397,12 +413,17 @@ impl QwenEngine { last_logits: Vec::new(), vocab: 0, eos, + cache_dirty: false, }) } /// One forward over a single token at the current position; advances the KV /// cache and returns milli-logits for the next token. fn forward_one(&mut self, token: Token) -> Result> { + // Any forward may write conversation K/V into candle's cache; mark dirty + // before the fallible call so a forward that fails part-way still leaves + // the cache flagged for clearing (ADR-018). + self.cache_dirty = true; let t_total = bench::enabled().then(std::time::Instant::now); let input = Tensor::from_vec(vec![token], (1, 1), &self.device) @@ -479,6 +500,43 @@ impl InferenceEngine for QwenEngine { } Ok(()) } + + /// Release the current conversation's KV **while keeping the resident weights + /// loaded** (ADR-018) — the separation of conversation lifecycle from model + /// lifecycle, and the engine half of [`InferenceSession::close`] / `reset`. + /// + /// candle's `quantized_qwen2` owns its per-layer KV with no public clear API, + /// but its attention *ignores and replaces* the cache on a forward at + /// `index_pos == 0`. So one forward over a benign token (id 0) drops the prior + /// (user) K/V tensors — freeing that memory and clearing the user's data from + /// the cache (PRD line 131) — without touching the weights. What remains is a + /// single non-user token's KV, itself overwritten by the next prefill or freed + /// when the engine is dropped. Skipped when nothing has been cached yet + /// (`index_pos == 0`), so a pristine or already-cleared engine does no work. + /// + /// Distinct from `rollback`, which *replays* a retained prefix to rewind + /// within a single generation. Fallible (it runs a forward); on error the + /// caller (`reset`/`close`) leaves session state untouched and surfaces it. + fn reset_cache(&mut self) -> Result<()> { + if self.cache_dirty { + // Overwrite (and thereby drop) the user K/V by forwarding a benign + // token at position 0; the resulting logits are discarded. `cache_dirty` + // is cleared **only after** a fully successful forward — if candle + // fails after replacing some layers, it stays set so the next call + // re-clears (a partially-cleared cache is never reported as clean). + self.index_pos = 0; + self.forward_one(0)?; + self.cache_dirty = false; + } + self.index_pos = 0; + self.fed = 0; + // Release (not just `clear`) the conversation-derived buffers so their + // bytes aren't retained in an owned allocation (P2): `Vec::new()` drops + // the old allocation; `clear()` would keep capacity and the stale ids. + self.prompt = Vec::new(); + self.last_logits = Vec::new(); + Ok(()) + } } // ── On-device safety wiring (ADR-005 tier + ADR-012 control loop) ──────────── @@ -680,19 +738,36 @@ impl ExpertLogits for QwenExpert { } } +/// The resident model behind a [`QwenChatProvider`] (ADR-018). +/// +/// The weights are loaded **once** (in [`QwenChatProvider::from_paths`]) into +/// `Loaded`. The first `chat` promotes them into a reusable [`InferenceSession`] +/// (`Active`) — done lazily so the builder-configured safety tier / expert is +/// finalized first — and every later turn reuses that one session via +/// `reset()` + `load_prompt()`. The model is never re-read from disk per turn. +enum ChatSession { + /// Weights resident, no conversation session yet. + Loaded(QwenEngine), + /// Reusable session wrapping the resident engine. + Active(InferenceSession), + /// Transient placeholder held only during the `Loaded` → `Active` swap. + Swapping, +} + /// A real local chat backend: a Qwen2 GGUF model + its tokenizer, driven /// through [`el_runtime::InferenceSession`]. /// -/// Each `chat` call renders the whole conversation to Qwen2.5 ChatML, builds a -/// fresh [`QwenEngine`] (candle has no public KV-cache reset), then runs the -/// SDK's standard provenance-gated session: `load_prompt` (prefill) → -/// `generate` (grammar mask → safety steer → guard + checkpointed rollback → -/// greedy commit). On-device safety (ADR-005 `Lightweight` tier + the ADR-012 -/// control loop) is **on by default**; see [`with_safety`](Self::with_safety). -/// The provider holds no mutable session state, so it is `Send + Sync` without -/// locking. +/// The model weights are loaded **once** at construction and kept resident +/// (ADR-018): each `chat` renders the whole conversation to Qwen2.5 ChatML and +/// reuses one persistent provenance-gated session — `reset` (discard the prior +/// conversation + engine KV cache) → `load_prompt` (prefill) → `generate` +/// (grammar mask → safety steer → guard + checkpointed rollback → greedy commit) +/// — instead of rebuilding the engine and re-reading the GGUF every turn. +/// On-device safety (ADR-005 `Lightweight` tier + the ADR-012 control loop) is +/// **on by default**; see [`with_safety`](Self::with_safety). The resident model +/// lives behind a `Mutex`, so the provider stays `Send + Sync` and concurrent +/// `chat` calls serialize on the one conversation. pub struct QwenChatProvider { - model_path: std::path::PathBuf, tokenizer: Tokenizer, permit: LoadPermit, eos: Token, @@ -700,10 +775,14 @@ pub struct QwenChatProvider { model_label: String, safety: SafetyConfig, /// Optional safety **expert** GGUF for ADR-013 contrastive steering. `None` - /// runs the token-only `Lightweight` steerer. + /// runs the token-only `Lightweight` steerer. NOTE (ADR-018 follow-up): the + /// expert is still loaded per turn; persisting it like the base model is the + /// next increment. The default (no-expert) path is fully resident. expert_model: Option, /// Contrastive steering strength ×1000 (1000 = 1.0×). steer_alpha_milli: i32, + /// Resident model + reusable session (ADR-018). + session: std::sync::Mutex, } impl QwenChatProvider { @@ -731,8 +810,13 @@ impl QwenChatProvider { let safety = SafetyConfig::lightweight(&tokenizer); let permit = local_load_permit(&model_path)?; + + // ADR-018: load the weights ONCE here and keep them resident, instead of + // re-reading the GGUF on every `chat`. The first `chat` promotes this into + // a reusable session (see `ChatSession`). + let engine = QwenEngine::from_path(&model_path, eos)?; + Ok(Self { - model_path, tokenizer, permit, eos, @@ -741,6 +825,7 @@ impl QwenChatProvider { safety, expert_model: None, steer_alpha_milli: 1000, + session: std::sync::Mutex::new(ChatSession::Loaded(engine)), }) } @@ -792,6 +877,24 @@ impl QwenChatProvider { self } + /// End the current conversation, releasing its KV / output / prompt / buffered + /// events **while keeping the model resident** (ADR-018 separation of + /// conversation and model lifecycles; the AC-4 explicit release / PRD line 131 + /// "KV caches … cleared on session end"). The next `chat` starts a fresh + /// conversation on the same loaded weights — no reload. A no-op if no + /// conversation has started yet. To free the weights too, drop the provider + /// (Rust ownership). + pub fn end_session(&self) -> Result<()> { + let mut cell = self + .session + .lock() + .map_err(|_| EdgeError::Engine("chat session mutex poisoned"))?; + if let ChatSession::Active(session) = &mut *cell { + session.close()?; + } + Ok(()) + } + fn encode(&self, text: &str) -> Result> { let enc = self .tokenizer @@ -815,18 +918,12 @@ impl LlmProvider for QwenChatProvider { let prompt_tokens = self.encode(&prompt)?; let d_encode = t_encode.map(|t| t.elapsed()).unwrap_or_default(); - // Fresh engine + session each turn (candle KV cache has no public reset); - // the full conversation is re-prefilled. This is the standard SDK path — - // provenance permit, session lifecycle, decode loop — not a shortcut. - let t_load = bench::enabled().then(std::time::Instant::now); - let engine = QwenEngine::from_path(&self.model_path, self.eos)?; - let d_load = t_load.map(|t| t.elapsed()).unwrap_or_default(); - // Carry the active safety tier on the session config so the runtime // derives the tier-aware ADR-012 `RollbackPolicy` and records the true // mode. A supplied expert promotes the tier to `SecDecoding`, so the // runtime's `SafetyModeSelector` can gate it on device class instead of - // it masquerading as `Lightweight`. + // it masquerading as `Lightweight`. Both are deterministic from the + // builder-set config, so they are identical on every turn. let requested = requested_session_safety(self.safety.mode, self.expert_model.is_some()); let cfg = SessionConfig { safety: requested, @@ -837,7 +934,44 @@ impl LlmProvider for QwenChatProvider { // downgrades to `Lightweight` on non-accelerator devices, where the // expert is dropped — honest tier-aware behaviour). let effective = SafetyModeSelector::resolve(requested, cfg.device); - let mut session = InferenceSession::new(SessionId(1), cfg, engine, self.permit); + + // ADR-018: reuse the resident model. Lock the session cell; on first use + // promote the loaded weights into a reusable session (created with the now + // final builder config); every later turn reuses it — no disk reload. + let mut cell = self + .session + .lock() + .map_err(|_| EdgeError::Engine("chat session mutex poisoned"))?; + let t_load = bench::enabled().then(std::time::Instant::now); + if matches!(&*cell, ChatSession::Loaded(_)) { + let engine = match std::mem::replace(&mut *cell, ChatSession::Swapping) { + ChatSession::Loaded(e) => e, + _ => unreachable!("guarded by the matches! above"), + }; + *cell = ChatSession::Active(InferenceSession::new( + SessionId(1), + cfg, + engine, + self.permit, + )); + } + let d_load = t_load.map(|t| t.elapsed()).unwrap_or_default(); + let session = match &mut *cell { + ChatSession::Active(s) => s, + // `Swapping` only persists if a prior promotion panicked — in which + // case `lock()` above would already have failed on the poisoned mutex. + _ => return Err(EdgeError::Engine("chat session not initialized")), + }; + + // Discard the previous conversation and the engine's KV cache, then run + // this turn over the SAME resident weights (ADR-018) — no disk reload. + session.reset()?; + // Provider-owned turn isolation: drop any events buffered by a prior turn + // (e.g. one that errored before its end-of-turn drain) so this turn's + // safety count below cannot include another turn's stale violations. + // `reset()` deliberately preserves events (generic semantics); bounding + // them per turn is the reusing provider's job. + let _ = session.drain_events(); let mut ports = self.safety.ports(); if matches!(effective, SafetyMode::SecDecoding) { @@ -877,14 +1011,15 @@ impl LlmProvider for QwenChatProvider { let decoded = self.decode(&out)?.trim().to_string(); let d_detok = t_detok.map(|t| t.elapsed()).unwrap_or_default(); - // ADR-012: surface what the decode-time control loop did. A fail-closed - // stop (rollbacks exhausted / no safe checkpoint) returns the - // deterministic refusal rather than the truncated unsafe prefix; any - // intervention is reported on stderr so the test client can show the - // guard working without corrupting the reply on stdout. + // ADR-018: always drain — a persistent session would otherwise accumulate + // events across turns. ADR-012: surface what the decode-time control loop + // did. A fail-closed stop (rollbacks exhausted / no safe checkpoint) + // returns the deterministic refusal rather than the truncated unsafe + // prefix; any intervention is reported on stderr so the test client can + // show the guard working without corrupting the reply on stdout. + let events = session.drain_events(); let safety_active = !matches!(self.safety.mode, SafetyMode::Off); let content = if safety_active { - let events = session.drain_events(); let violations = events .iter() .filter(|e| matches!(e.event, DomainEvent::SafetyViolationDetected { .. })) @@ -994,7 +1129,7 @@ fn report_breakdown( eprintln!("│ prompt_tokens={prompt_tokens} completion_tokens={completion_tokens}"); eprintln!("│ phase wall(ms) %total throughput"); eprintln!( - "│ model load {:>9.1} {:>6.1}% (read+dequantize GGUF)", + "│ session setup {:>9.1} {:>6.1}% (weights loaded once at startup — ADR-018)", ms(d_load), pct(d_load) ); diff --git a/crates/el-grammar/src/lib.rs b/crates/el-grammar/src/lib.rs index 2a081ab..0afb1fe 100644 --- a/crates/el-grammar/src/lib.rs +++ b/crates/el-grammar/src/lib.rs @@ -165,6 +165,9 @@ mod tests { fn rollback(&mut self, _keep: u32) -> el_core::Result<()> { Ok(()) // stateless } + fn reset_cache(&mut self) -> el_core::Result<()> { + Ok(()) // stateless + } } struct OkVerifier; diff --git a/crates/el-runtime/src/defaults.rs b/crates/el-runtime/src/defaults.rs index c1bc075..8586d92 100644 --- a/crates/el-runtime/src/defaults.rs +++ b/crates/el-runtime/src/defaults.rs @@ -57,4 +57,9 @@ impl InferenceEngine for NullEngine { fn rollback(&mut self, _keep_committed: u32) -> Result<()> { Ok(()) } + + /// Stateless: no conversation cache to discard. + fn reset_cache(&mut self) -> Result<()> { + Ok(()) + } } diff --git a/crates/el-runtime/src/ports.rs b/crates/el-runtime/src/ports.rs index 6646b22..99a1b22 100644 --- a/crates/el-runtime/src/ports.rs +++ b/crates/el-runtime/src/ports.rs @@ -37,6 +37,23 @@ pub trait InferenceEngine { /// make the choice explicit. A stateless engine whose `next_logits` /// recomputes purely from `committed` implements it as `Ok(())`. fn rollback(&mut self, keep_committed: u32) -> Result<()>; + + /// Return the engine to its **pristine, pre-prefill** state so the same + /// loaded weights can serve a *new conversation* without being reloaded + /// (ADR-018). After `Ok(())`, the next [`prefill`](Self::prefill) must build a + /// KV cache from scratch as if the engine had just been constructed. + /// + /// This is distinct from [`rollback`](Self::rollback): `rollback(keep)` rewinds + /// *within* a generation to a safe prefix of length `keep` (ADR-012); + /// `reset_cache` discards the whole conversation. It is what lets a provider + /// hold one resident model and reuse it across turns instead of re-reading the + /// weights from disk every call. + /// + /// Like `rollback`, it is **required with no default**: a stateful adapter that + /// forgot to override would otherwise carry a stale cache into the next + /// conversation. A stateless engine whose `next_logits` recomputes purely from + /// `committed` implements it as `Ok(())`. + fn reset_cache(&mut self) -> Result<()>; } /// Prompt Compression port (LLMLingua-2 — context 2). diff --git a/crates/el-runtime/src/session.rs b/crates/el-runtime/src/session.rs index 5770f15..a56ae2d 100644 --- a/crates/el-runtime/src/session.rs +++ b/crates/el-runtime/src/session.rs @@ -451,14 +451,62 @@ impl InferenceSession { } } - /// Clear KV/output for a fresh conversation (volatile memory only). - pub fn reset(&mut self) { + /// Reset for a fresh conversation on the **same resident weights** — the seam + /// that turns a provider from "rebuild the engine every turn" into "load once, + /// reuse" (ADR-018). Clears the session's KV descriptors / output / prompt and + /// resets the engine's *logical* cache; distinct from a safety + /// [`rollback`](InferenceEngine::rollback) (which rewinds *within* a + /// generation). + /// + /// Returns the engine's `reset_cache` error rather than swallowing it: an + /// engine that fails to discard its cache must not be reported as a clean + /// fresh session. On error the session state is left **untouched** (so an empty + /// session can never desync from a stale engine cache) and the caller is + /// notified — it should drop/rebuild rather than reuse. + /// + /// Buffered events are **preserved** (generic semantics): a telemetry consumer + /// may still [`drain_events`](Self::drain_events) after a reset. Turn-level + /// event isolation is the caller's concern — a provider that reuses one session + /// across turns should drain between turns (see the adapter providers). + /// + /// `reset_cache` releases the *previous conversation's* KV while keeping the + /// resident weights loaded — that separation of conversation lifecycle from + /// model lifecycle is the point of ADR-018. + pub fn reset(&mut self) -> Result<()> { + self.engine.reset_cache()?; self.kv = KvRegion::new(); self.prompt.clear(); self.output.clear(); self.step = 0; self.phase = Phase::Initialized; self.emit(DomainEvent::SessionReset); + Ok(()) + } + + /// End the current conversation: release its volatile memory — engine KV + /// (via [`reset_cache`](InferenceEngine::reset_cache)), KV descriptors, + /// committed output, retained prompt, and buffered events — while keeping the + /// **resident model loaded** so the session can serve a new conversation + /// without reloading weights (ADR-018; PRD line 131 "KV caches … cleared on + /// session end"; the AC-4 explicit release). + /// + /// Takes `&mut self`, not `self`: dropping the session would also drop the + /// (expensive) weights, which is the opposite of "load once, reuse." Instead + /// the engine releases the conversation's KV in place — for candle's + /// `quantized_qwen2`, a position-0 forward overwrites and frees the prior K/V + /// tensors (see its `reset_cache`). Unlike [`reset`](Self::reset), `close` also + /// frees the buffers' *capacity* and discards buffered events, minimizing the + /// idle footprint. Propagates a `reset_cache` failure (state untouched on + /// error). To free the weights too, drop the session/provider (ownership). + pub fn close(&mut self) -> Result<()> { + self.engine.reset_cache()?; + self.kv = KvRegion::new(); + self.prompt = Vec::new(); + self.output = Vec::new(); + self.events = Vec::new(); + self.step = 0; + self.phase = Phase::Initialized; + Ok(()) } /// Consult the opt-in LAN relay. Hard-fails with [`EdgeError::AirGapViolation`] @@ -538,6 +586,9 @@ mod tests { fn rollback(&mut self, _keep: u32) -> Result<()> { Ok(()) // stateless } + fn reset_cache(&mut self) -> Result<()> { + Ok(()) // stateless + } } // Grammar masker that disallows specific token ids. @@ -567,7 +618,7 @@ mod tests { assert_eq!(s.output(), &[3]); assert_eq!(s.phase(), Phase::Completed); - s.reset(); + s.reset().unwrap(); assert_eq!(s.phase(), Phase::Initialized); assert!(s.output().is_empty()); } @@ -745,6 +796,9 @@ mod tests { fn rollback(&mut self, _keep: u32) -> Result<()> { Ok(()) // stateless: logits depend only on the passed ctx } + fn reset_cache(&mut self) -> Result<()> { + Ok(()) // stateless + } } /// `guard_every` larger than any completion here, so the *cadence* check @@ -872,6 +926,10 @@ mod tests { self.last_keep.set(keep_committed); Ok(()) } + fn reset_cache(&mut self) -> Result<()> { + self.fed = 0; // pristine: a fresh conversation re-feeds from scratch + Ok(()) + } } #[test] @@ -1195,4 +1253,346 @@ mod tests { } ))); } + + // ----- ADR-018 persistent model / stateful session reuse ----- + + /// Counts how often it is prefilled and cache-reset, and emits EOS on the + /// first decode step so generation terminates immediately. Mirrors a *resident* + /// engine: one instance reused across conversations, never reconstructed. + struct CountingEngine { + eos: Token, + vocab: usize, + prefills: std::rc::Rc>, + resets: std::rc::Rc>, + } + impl InferenceEngine for CountingEngine { + fn prefill(&mut self, t: &[Token]) -> Result { + self.prefills.set(self.prefills.get() + 1); + Ok(t.len() as u32) + } + fn next_logits(&mut self, _committed: &[Token]) -> Vec { + let mut v = vec![0i32; self.vocab]; + if let Some(s) = v.get_mut(self.eos as usize) { + *s = 1; + } + v + } + fn eos_token(&self) -> Token { + self.eos + } + fn rollback(&mut self, _keep: u32) -> Result<()> { + Ok(()) + } + fn reset_cache(&mut self) -> Result<()> { + self.resets.set(self.resets.get() + 1); + Ok(()) + } + } + + fn counting_engine() -> ( + CountingEngine, + std::rc::Rc>, + std::rc::Rc>, + ) { + let prefills = std::rc::Rc::new(std::cell::Cell::new(0u32)); + let resets = std::rc::Rc::new(std::cell::Cell::new(0u32)); + let engine = CountingEngine { + eos: 1, + vocab: 4, + prefills: prefills.clone(), + resets: resets.clone(), + }; + (engine, prefills, resets) + } + + #[test] + fn reset_resets_the_engine_cache() { + // AC: reset() must discard the engine's conversation cache, not just the + // session's descriptors — otherwise a reused resident engine would carry a + // stale KV cache into the next conversation. + let (engine, _prefills, resets) = counting_engine(); + let mut s = + InferenceSession::new(SessionId(40), SessionConfig::default(), engine, permit()); + assert_eq!(resets.get(), 0); + s.reset().unwrap(); + assert_eq!( + resets.get(), + 1, + "reset() must reset the engine cache (ADR-018)" + ); + } + + #[test] + fn one_engine_serves_multiple_conversations_via_reset() { + // AC-1/AC-2: the same resident engine is reused across conversations — it + // is prefilled again per turn but never reconstructed, and its cache is + // reset between turns. + let (engine, prefills, resets) = counting_engine(); + let mut s = + InferenceSession::new(SessionId(41), SessionConfig::default(), engine, permit()); + let ports = Ports::permissive(); + + s.load_prompt(&ports, &[1, 2]).unwrap(); + s.generate(&ports, 8).unwrap(); + assert_eq!(prefills.get(), 1); + + // Reuse for a second conversation — no new engine, no reload. + s.reset().unwrap(); + s.load_prompt(&ports, &[3, 4, 5]).unwrap(); + s.generate(&ports, 8).unwrap(); + + assert_eq!( + prefills.get(), + 2, + "the one resident engine was prefilled again, not rebuilt" + ); + assert!( + resets.get() >= 1, + "the engine cache was reset between conversations" + ); + } + + /// Counts both cache resets and drops, so a test can prove `close()` releases + /// the conversation's KV (reset_cache) **without** dropping the resident engine. + struct ResidentEngine { + eos: Token, + vocab: usize, + resets: std::rc::Rc>, + dropped: std::rc::Rc>, + } + impl Drop for ResidentEngine { + fn drop(&mut self) { + self.dropped.set(self.dropped.get() + 1); + } + } + impl InferenceEngine for ResidentEngine { + fn prefill(&mut self, t: &[Token]) -> Result { + Ok(t.len() as u32) + } + fn next_logits(&mut self, _committed: &[Token]) -> Vec { + let mut v = vec![0i32; self.vocab]; + if let Some(s) = v.get_mut(self.eos as usize) { + *s = 1; + } + v + } + fn eos_token(&self) -> Token { + self.eos + } + fn rollback(&mut self, _keep: u32) -> Result<()> { + Ok(()) + } + fn reset_cache(&mut self) -> Result<()> { + self.resets.set(self.resets.get() + 1); + Ok(()) + } + } + + #[test] + fn close_releases_conversation_but_keeps_engine_resident() { + // AC-4 / ADR-018: `close` must release the conversation's memory (KV via + // the engine's `reset_cache`) while keeping the resident weights loaded — + // and the session must remain reusable for the next conversation. Dropping + // the engine (and its weights) is NOT close's job; that is provider drop. + let resets = std::rc::Rc::new(std::cell::Cell::new(0u32)); + let dropped = std::rc::Rc::new(std::cell::Cell::new(0u32)); + let engine = ResidentEngine { + eos: 1, + vocab: 4, + resets: resets.clone(), + dropped: dropped.clone(), + }; + let mut s = + InferenceSession::new(SessionId(42), SessionConfig::default(), engine, permit()); + let ports = Ports::permissive(); + s.load_prompt(&ports, &[1, 2, 3]).unwrap(); + s.generate(&ports, 8).unwrap(); + assert!(s.kv_len() > 0, "conversation memory is measurable (AC-4)"); + + s.close().unwrap(); + + assert_eq!(s.kv_len(), 0, "close frees the session's KV descriptors"); + assert!(s.output().is_empty(), "close frees committed output"); + assert!( + resets.get() >= 1, + "close releases the engine's conversation KV (keeps weights)" + ); + assert_eq!( + dropped.get(), + 0, + "the resident engine/weights are NOT dropped by close" + ); + + // Model still resident → the session serves a new conversation, no reload. + s.load_prompt(&ports, &[4, 5]).unwrap(); + s.generate(&ports, 8).unwrap(); + assert!( + s.kv_len() > 0, + "same engine serves a new conversation after close" + ); + assert_eq!(dropped.get(), 0); + } + + #[test] + fn reset_preserves_undrained_events() { + // Generic semantics: reset() must NOT silently drop a consumer's undrained + // telemetry. Turn-level isolation is the provider's job (drain between + // turns), not reset()'s. + let (engine, _prefills, _resets) = counting_engine(); + let mut s = + InferenceSession::new(SessionId(44), SessionConfig::default(), engine, permit()); + let ports = Ports::permissive(); + s.load_prompt(&ports, &[1, 2]).unwrap(); + s.generate(&ports, 8).unwrap(); // emits events the caller has not drained + + s.reset().unwrap(); + + let evs = s.drain_events(); + assert!( + evs.iter() + .any(|e| matches!(e.event, DomainEvent::TokenCommitted { .. })), + "pre-reset events are preserved for the consumer" + ); + assert!( + evs.iter() + .any(|e| matches!(e.event, DomainEvent::SessionReset)), + "and the reset itself is recorded" + ); + } + + /// Reset is fallible: this engine refuses to discard its cache. + struct FailResetEngine { + eos: Token, + vocab: usize, + } + impl InferenceEngine for FailResetEngine { + fn prefill(&mut self, t: &[Token]) -> Result { + Ok(t.len() as u32) + } + fn next_logits(&mut self, _committed: &[Token]) -> Vec { + let mut v = vec![0i32; self.vocab]; + if let Some(s) = v.get_mut(self.eos as usize) { + *s = 1; + } + v + } + fn eos_token(&self) -> Token { + self.eos + } + fn rollback(&mut self, _keep: u32) -> Result<()> { + Ok(()) + } + fn reset_cache(&mut self) -> Result<()> { + Err(EdgeError::Engine("reset_cache refused")) + } + } + + #[test] + fn reset_propagates_engine_cache_failure_and_leaves_state() { + // A fallible engine that cannot discard its cache must NOT be reported as a + // clean fresh session: the error surfaces and the session descriptors stay + // untouched, so an empty session can never desync from a stale engine cache. + let mut s = InferenceSession::new( + SessionId(45), + SessionConfig::default(), + FailResetEngine { eos: 1, vocab: 4 }, + permit(), + ); + let ports = Ports::permissive(); + s.load_prompt(&ports, &[1, 2, 3]).unwrap(); + s.generate(&ports, 8).unwrap(); + let kv_before = s.kv_len(); + assert!(kv_before > 0); + + let err = s.reset().unwrap_err(); + assert!(matches!(err, EdgeError::Engine(_))); + assert_eq!( + s.kv_len(), + kv_before, + "descriptors must be left intact when the engine reset fails" + ); + assert_eq!( + s.phase(), + Phase::Completed, + "phase is unchanged on a failed reset" + ); + } + + /// Marks itself dirty when it caches, fails its first `reset_cache` (a + /// simulated partial eviction), then succeeds — recording `cleared` only after + /// a *fully successful* attempt. Mirrors the `QwenEngine` dirty-flag contract. + struct RetryClearEngine { + eos: Token, + vocab: usize, + dirty: bool, + fails_left: std::rc::Rc>, + cleared: std::rc::Rc>, + } + impl InferenceEngine for RetryClearEngine { + fn prefill(&mut self, t: &[Token]) -> Result { + self.dirty = true; + Ok(t.len() as u32) + } + fn next_logits(&mut self, _committed: &[Token]) -> Vec { + self.dirty = true; + let mut v = vec![0i32; self.vocab]; + if let Some(s) = v.get_mut(self.eos as usize) { + *s = 1; + } + v + } + fn eos_token(&self) -> Token { + self.eos + } + fn rollback(&mut self, _keep: u32) -> Result<()> { + Ok(()) + } + fn reset_cache(&mut self) -> Result<()> { + if self.dirty { + if self.fails_left.get() > 0 { + self.fails_left.set(self.fails_left.get() - 1); + return Err(EdgeError::Engine("partial eviction")); // stays dirty + } + self.dirty = false; + self.cleared.set(true); + } + Ok(()) + } + } + + #[test] + fn reset_retries_clearing_after_a_failed_attempt() { + // Regression (P1): a `reset_cache` that fails after partially clearing must + // NOT be skipped on the next attempt. Dirtiness is tracked independently of + // any position cursor, so the retry re-clears rather than falsely reporting + // a clean cache while user K/V is still resident. + let fails_left = std::rc::Rc::new(std::cell::Cell::new(1u32)); + let cleared = std::rc::Rc::new(std::cell::Cell::new(false)); + let engine = RetryClearEngine { + eos: 1, + vocab: 4, + dirty: false, + fails_left: fails_left.clone(), + cleared: cleared.clone(), + }; + let mut s = + InferenceSession::new(SessionId(47), SessionConfig::default(), engine, permit()); + let ports = Ports::permissive(); + s.load_prompt(&ports, &[1, 2]).unwrap(); + s.generate(&ports, 8).unwrap(); + + // First reset fails (simulated partial eviction) and is surfaced. + assert!(s.reset().is_err()); + assert!( + !cleared.get(), + "a failed reset must not report the cache cleared" + ); + + // The retry actually re-clears — it is not skipped. + s.reset().unwrap(); + assert!( + cleared.get(), + "the next reset re-clears after a failed attempt" + ); + } } diff --git a/docs/adr/ADR-018-persistent-model-instances-and-stateful-sessions.md b/docs/adr/ADR-018-persistent-model-instances-and-stateful-sessions.md new file mode 100644 index 0000000..cd19ee8 --- /dev/null +++ b/docs/adr/ADR-018-persistent-model-instances-and-stateful-sessions.md @@ -0,0 +1,224 @@ +# ADR-018: Persistent model instances and stateful inference sessions + +- **Status**: proposed +- **Date**: 2026-06-22 +- **Deciders**: +- **Tags**: runtime, performance, on-device, follow-up, P0 + +## Context + +The improvements plan +([docs/research/improvements-plan.md](../research/improvements-plan.md) §P0.1, +roadmap Phase 1, EPIC-1) identifies model-and-session lifecycle as the single +highest-value foundation: weights should load **once**, and conversation state +(committed tokens + KV) should persist and be **reused across turns** instead of +re-prefilling the whole history. + +What the codebase does today contradicts that on the real backend: + +- `QwenChatProvider::chat` builds a **fresh `QwenEngine` every turn** and + **reloads the GGUF from disk each call** (`QwenEngine::from_path(&self.model_path, …)`), + then re-prefills the entire rendered conversation. The inline comment is + explicit: *"Fresh engine + session each turn (candle KV cache has no public + reset); the full conversation is re-prefilled."* +- `QwenEngine` holds candle's stateful KV cache (`index_pos`, `fed`) but + `candle-transformers`' `quantized_qwen2` exposes no public cache reset/truncate, + so a *new conversation* can only be served by constructing a new engine. +- `LocalLlmProvider` keeps one `InferenceSession` behind a `Mutex` but calls + `session.reset()` at the start of every `chat`, clearing KV — so even there, + prefix reuse never happens. +- `InferenceSession` (the ADR-001 aggregate root) already owns the session state + machine (`Phase`), the `KvRegion`, the retained `prompt`, and a `reset()` that + clears volatile memory — but it conflates *model lifetime* (the `engine`, moved + in by value) with *conversation lifetime*. + +Net effect: per-turn latency pays a full model load **and** a full prefill of the +growing history — exactly the orchestration cost the plan calls out first. + +## Decision + +Separate **model lifecycle** from **conversation lifecycle**, keeping +`el-runtime` as the session authority (the plan's explicit constraint — ruvLLM's +`RuvLLMEngine` is a reference, not an adopted dependency). + +1. **Persistent model handle.** Introduce a loaded-model handle (e.g. + `LoadedModel` / `StatefulLlmProvider`) that owns the weights and the + `LoadPermit`, constructed once. Provider `chat`/`chat_stream` calls borrow it; + weights are never re-read from disk per turn. (Mmap loading — ADR-021 — makes + that one load cheap and shareable.) + +2. **Multiple isolated sessions over one model.** A session is a conversation: + its own `KvRegion`, committed output, `prompt`, `Phase`, and event buffer, + bound to the shared immutable model. `SessionConfig` stays immutable for the + session's life (ADR-001). + +3. **Engine cache ownership.** Because the per-conversation KV cache must be + resettable/clonable without reloading weights, the `InferenceEngine` seam gains + an explicit conversation-reset (e.g. `reset_cache()` / a per-session cache + handle) distinct from the ADR-012 `rollback`. The candle `quantized_qwen2` + limitation (no public cache reset) is recorded under Consequences and is the + reason this is an engine-seam change, not a pure runtime change. + +4. **Prefix reuse across turns.** On a follow-up turn, prefill only the **new** + suffix (new user message + assistant turn opener), reusing the KV for the + unchanged prefix, rather than re-prefilling the whole transcript. + +5. **Explicit lifecycle ops.** Every session exposes `reset`, `close`, and + eviction; session memory is measurable and explicitly releasable (the telemetry + to prove this comes from ADR-023). + +## Consequences + +### Positive +- Eliminates per-turn model reload and whole-history re-prefill — the largest + measured hot-path cost named in the plan. +- Multi-turn conversations scale with *new* tokens per turn, not total transcript + length. +- Clean separation lets one resident model back many lightweight sessions + (foundation for the later policy/adapter work, P1+). + +### Negative +- Requires an engine-seam change: candle's `quantized_qwen2` has no public + KV-cache reset, so faithful per-conversation reuse needs either a forked/custom + attention cache or an engine wrapper that manages cache lifetime — the same + engine-internals constraint ADR-012 hit for `rollback`. +- Resident weights held for the model's lifetime raise steady-state memory vs. + load-per-call (mitigated by mmap, ADR-021, and the ADR-003 budget). +- Prefix reuse interacts with the ADR-012 rollback/checkpoint invariants — a + retained prefix must remain a guard-verified-safe prefix. + +### Neutral +- `InferenceSession::reset()` and the `Phase` machine remain; this ADR re-homes + the engine from "owned by the session" to "shared by sessions," which is + additive to the existing lifecycle. +- The cloud `LlmProvider` (ADR-010) is unaffected — it is already stateless per + call. + +## Implementation status + +Landed via a SPARC pass (scope: AC-1/AC-2/AC-4 + the engine seam; AC-3 deferred): + +- **Engine seam** — added required `InferenceEngine::reset_cache(&mut self) -> + Result<()>` (`el-runtime::ports`), distinct from `rollback`: it returns the + engine to a pristine pre-prefill state so resident weights serve a new + conversation. Implemented for every engine (`NullEngine`, `CandleEngine` + stateless `Ok(())`; `QwenEngine` resets `index_pos`/`fed`/`prompt`/`last_logits`) + and all test engines. No default — a forgotten override can't silently carry a + stale cache. +- **Verified enabler** — candle `quantized_qwen2` attention discards its KV cache + on a forward at `index_pos == 0`, so a re-`prefill` rebuilds it; the engine is + reusable without reloading weights. The prior "candle has no public cache reset" + caveat is satisfied this way. +- **Session lifecycle (conversation vs. model)** — `InferenceEngine::reset_cache` + *actually releases the current conversation's KV while keeping weights loaded*: + for `QwenEngine` it runs one position-0 forward over a benign token, which + candle's attention uses to overwrite (and thus drop) the prior user K/V tensors + — freeing that memory and clearing user data — without touching the weights. + `reset()` (reuse) and `close(&mut self)` (end-of-conversation: also frees buffer + capacity + discards events) both build on it; both keep the model resident and + the session reusable, and both propagate a `reset_cache` failure (state untouched + on error). `reset()` preserves undrained events (generic semantics). `close` + takes `&mut self`, **not** `self`: consuming would also drop the expensive + weights — the opposite of "load once, reuse." To free the weights too, drop the + session/provider (ownership). `kv_len()` exposes the measurable footprint (AC-4). +- **Resident model + explicit release (AC-1/AC-2/AC-4)** — `QwenChatProvider` loads + the `QwenEngine` **once** in `from_paths` and holds it in a `Mutex` + (`Loaded` → `Active`, promoted lazily on the first `chat` so builder config is + final), reusing one session per turn via `reset()` + `load_prompt()` + + `generate()`. The per-turn `QwenEngine::from_path` disk reload is gone. + `QwenChatProvider::end_session()` exposes the explicit conversation release + (releases KV/output/prompt/events, keeps the model resident). Turn-level event + isolation lives in the providers (drain/discard at turn start), not in generic + `reset()`. +- **Tests** — runtime tests prove `reset()` resets the engine and preserves + undrained events; one engine serves multiple conversations without + reconstruction; `close()` releases the conversation, keeps the engine resident + (Drop-counter shows zero drops), and leaves the session reusable; and a fallible + `reset_cache` is propagated, not swallowed. Full workspace suite green; + `cargo fmt --all -- --check` and `clippy -D warnings` clean. + +Still **deferred** (own increment, per the spec's scope decision): + +- **AC-3 — cross-turn prefix reuse / incremental prefill.** Each turn still + re-prefills the whole conversation (no weight reload). Reusing the unchanged + prefix's KV needs an engine "extend-without-reset" contract, a session "continue" + transition, and care around the ADR-012 safe-prefix invariant and tokenizer + round-trip stability. The persistent-session architecture here is its foundation. +- **Expert persistence.** With `--expert-model`, the safety expert GGUF is still + loaded per turn; persisting it like the base model is a follow-up. The default + (no-expert) path is fully resident. +- **Concurrent multi-session weight sharing.** candle couples weights + KV cache, + so this realizes *serial* conversation reuse; true N-sessions-over-one-model + needs an engine that separates weights from cache (same root constraint as + ADR-022). + +### Review fixes (post-implementation hardening) + +**Round 1.** (1) `reset()`/`close()` silently ignored `reset_cache()` errors → they +now return `Result<()>` and propagate, leaving session state untouched on failure +so an empty session can't desync from a stale engine cache; (2) stray empty root +artifacts from a botched shell redirect were removed. + +**Round 2** (correcting round 1's overreach + remaining gaps). (1) **KV not freed +on session end (PRD line 131).** Documenting candle's limitation was insufficient — +the PRD requires user K/V cleared on session end via ownership. `close` now takes +`self` **by value**, so ending a session drops the engine and frees its KV +tensors; round 1's `&mut self` reset could not. Proven by a Drop-counting engine +test. (2) **`reset()` overreached by clearing events.** Round 1 fixed the +provider's event leak by clearing events in generic `reset()`, which silently drops +a consumer's undrained telemetry. Reverted — `reset()` preserves events; turn-level +isolation now lives in the providers (drain/discard at turn start). (3) **Release +gate.** `cargo fmt --all -- --check` now passes (it was failing in `el-runtime` / +`el-engine-candle`). (4) A further stray root artifact (`0)`) was removed. + +**Round 3** (correcting round 2's over-correction). Making `close(self)` consume +the session freed the KV but **also dropped the resident weights** — defeating the +ADR's separation of conversation lifecycle from model lifecycle (a caller could not +release conversation memory while keeping the model loaded), and `QwenChatProvider` +exposed no release at all. Fixes: (1) `reset_cache` now genuinely frees the +conversation's KV in place (the position-0 benign-forward overwrite for candle), so +KV is released **without** dropping weights; (2) `close` is back to `&mut self` — +releases the conversation, keeps the model resident, session reusable; (3) +`QwenChatProvider::end_session()` exposes that release to provider callers; (4) +further stray artifacts (`0`, `1`) from shell-redirect mishaps removed, and that +redirect pattern abandoned. Regression test: +`close_releases_conversation_but_keeps_engine_resident` (asserts KV freed, engine +**not** dropped, session reusable). Full gate green: tests, +`cargo fmt --all -- --check`, `clippy -D warnings`. + +**Round 4.** (1) **Partial-eviction failure could later report success.** +`reset_cache` keyed its skip on `index_pos`, which it zeroed *before* the fallible +forward — so a forward that failed after candle replaced some layers left +`index_pos == 0`, and the next `reset_cache` skipped clearing while user K/V was +still resident. Now `QwenEngine` tracks a `cache_dirty` flag set by every +`forward_one` (before the fallible call) and cleared **only after** a fully +successful eviction forward; a failed eviction stays dirty and is retried. +Regression test `reset_retries_clearing_after_a_failed_attempt`. (2) **App reset +paths didn't release.** `el-chat`'s `/reset`, `/system`, and generation-error +recovery now call `QwenChatProvider::end_session()` (via a `release_session` +helper) instead of only discarding chat history, so a discarded conversation +doesn't linger in the engine until the next turn. (3) **Buffers retained +allocations.** `reset_cache` used `Vec::clear()` (keeps capacity + stale token +bytes); it now assigns `Vec::new()` to **release** the prompt and logits +allocations. (Guaranteed zeroize-before-free would need the `zeroize` crate — +out of scope; the owned allocation is dropped.) + +**Round 5.** (1) **CLI cleanup failed open.** `release_session` logged an +`end_session` failure but the caller still cleared history and reported a +successful reset, so user KV could remain resident silently. It now **retries +once** and returns `false` on failure; `/reset`, `/system`, and error recovery +release *before* reporting success and, on unrecoverable failure, **stop** (break +the loop → `main` returns → the provider drops → KV freed via ownership) instead of +failing open. (2) **Stale public docs.** The adapter `README` and the `QwenEngine` +doc still described the old "fresh engine per `chat` (Candle can't reset its +cache)" behavior; both now describe the ADR-018 resident model + in-place +`reset_cache` eviction, and the `EL_BENCH` section reflects the once-at-startup +load ("session setup", not a per-chat "model load"). + +## Links +- Source: [docs/research/improvements-plan.md](../research/improvements-plan.md) §P0.1, EPIC-1. +- Builds on: [ADR-001](./ADR-001-adopt-webassembly-as-cross-platform-sdk-runtime.md) (session immutability), [ADR-010](./ADR-010-unified-llm-provider-trait-with-opt-in-frontier-egress.md) (`LlmProvider` seam), [ADR-002](./ADR-002-candle-as-rust-native-inference-engine.md) (engine). +- Constrained by: [ADR-003](./ADR-003-static-memory-planning-with-zero-allocation-arena.md) (memory budget for resident weights), [ADR-012](./ADR-012-layered-decode-time-safety-control-loop-with-checkpointed-rollback.md) (retained prefix must stay a safe prefix; cache-reset is distinct from `rollback`). +- Enables: [ADR-021](./ADR-021-memory-mapped-verified-gguf-loading.md) (cheap shared load), [ADR-019](./ADR-019-in-loop-incremental-decoding-and-token-streaming.md) (streaming over a persistent session), [ADR-022](./ADR-022-two-tier-quantized-kv-cache-with-attention-aware-eviction.md) (per-session KV budgets). +- Implementation seams: `crates/el-runtime` (`InferenceSession`, `ports::InferenceEngine`), `crates/adapters/el-engine-candle` (`QwenEngine`, `QwenChatProvider`, `LocalLlmProvider`). +- Measured by: [ADR-023](./ADR-023-baseline-performance-instrumentation.md) (warm-turn latency, prefill-tokens-saved). diff --git a/docs/adr/ADR-019-in-loop-incremental-decoding-and-token-streaming.md b/docs/adr/ADR-019-in-loop-incremental-decoding-and-token-streaming.md new file mode 100644 index 0000000..32dd9bf --- /dev/null +++ b/docs/adr/ADR-019-in-loop-incremental-decoding-and-token-streaming.md @@ -0,0 +1,100 @@ +# ADR-019: In-loop incremental decoding and real token streaming + +- **Status**: proposed +- **Date**: 2026-06-22 +- **Deciders**: +- **Tags**: runtime, performance, bindings, follow-up, P0 + +## Context + +The improvements plan +([docs/research/improvements-plan.md](../research/improvements-plan.md) §P0.2, +roadmap Phase 1, EPIC-2) calls for **true in-loop streaming**: the first token +should be emitted after model load + prefill + **one** decode step, not after the +whole response is finished. + +The `LlmProvider` trait (ADR-010) already advertises a streaming entry point — +`chat_stream(req, on_token: &mut dyn FnMut(ChatToken))` — and `ChatToken { text, +is_final }` exists. But the implementations only *simulate* streaming: + +- `QwenChatProvider::chat_stream` and `LocalLlmProvider::chat_stream` both call + `self.chat(req)?` to completion, then re-emit the finished string character by + character. The comment is explicit: *"The runtime decode loop runs to completion + internally (no per-token hook), so … we stream the finished reply out character + by character."* +- `InferenceSession::generate_with_policy` is the decode loop, but it commits + tokens into `self.output` and returns only a `StopReason` — there is **no + per-token emit hook**. +- Consequently time-to-first-token equals total generation time, and + `TelemetrySnapshot.ttft_ms` is never populated with a real measurement. + +A real-streaming design must respect the ADR-012 control loop: committed tokens +can be **rolled back** by the chunk guard. Emitting a token the instant it is +sampled would risk surfacing an unsafe span that the guard later rewinds — a +fail-*open* leak. So "stream each token immediately" is not safe as-is; emission +must be gated on the safety loop. + +## Decision + +Thread a per-token emit callback through the decode loop and drive real streaming +from it, with emission gated by the safety control loop. + +1. **Emit hook in the loop.** Add an optional sink to + `generate_with_policy` (e.g. `on_token: Option<&mut dyn FnMut(Token)>`, or an + `emit` closure on a small context) called when a token becomes **eligible to + surface** — see §3. The invariant decode order is unchanged: grammar mask → + safety adjust → sample → commit, then *consider emit*. + +2. **Provider streams from the hook.** `QwenChatProvider`/`LocalLlmProvider` + `chat_stream` detokenize emitted token ids incrementally and call `on_token`, + ending with `ChatToken { is_final: true }`. The character-replay shim is + removed. + +3. **Safety-gated emission (the key constraint).** A token is surfaced only once + it is **guard-verified safe** — i.e. at the next checkpoint/guard-pass boundary + (ADR-012), not at the instant of commit. Tokens still inside the in-flight, + not-yet-scored window are buffered; a rollback discards the buffer instead of + un-saying already-streamed text. This trades a little TTFT granularity (first + emit lands at the first guard-verified boundary, not literally token 1) for + never streaming a span the guard will rewind. With `SafetyMode::Off` or no + guard wired, tokens surface immediately after commit. + +4. **Cancellation.** The sink can request stop; the loop must halt promptly, + release temporary buffers, and leave the session in a consistent `Phase`. + +5. **Binding parity.** Because the seam is a plain `FnMut` callback (no async + runtime in `el-core`), each binding wraps it natively: FRB → Dart `Stream`, + uniffi → async callback, wasm-bindgen → `ReadableStream`. The `el-ffi` adapter + carries this through to all surfaces. + +## Consequences + +### Positive +- TTFT drops from "whole response" to "first guard-verified boundary," the + headline UX win the plan targets, and becomes a *measurable* number (ADR-023). +- One streaming path through the real decode loop replaces the per-provider + character-replay shims. +- Streaming composes with persistent sessions (ADR-018): tokens stream as the + reused-prefix continuation is generated. + +### Negative +- Emission gated on guard boundaries means first-token latency is bounded by + `guard_every` under an active guard — slightly coarser than per-token; documented + as the safety/latency trade-off. +- The loop grows an emit/cancel path and a small surface buffer; more state than a + fire-and-forget loop. +- Detokenization becomes incremental (partial multi-byte / multi-token glyphs must + not be split mid-emit) — the tokenizer-owning adapter handles boundary buffering. + +### Neutral +- `LlmProvider::chat` (non-streaming) is unchanged; it can be expressed as + `chat_stream` drained to a string. +- The ADR-007 content-free telemetry guarantee is untouched: streamed text goes to + the host callback, never onto a `DomainEvent`. + +## Links +- Source: [docs/research/improvements-plan.md](../research/improvements-plan.md) §P0.2, EPIC-2. +- Builds on: [ADR-010](./ADR-010-unified-llm-provider-trait-with-opt-in-frontier-egress.md) (`chat_stream`/`ChatToken` seam), [ADR-018](./ADR-018-persistent-model-instances-and-stateful-sessions.md) (persistent session to stream over), [ADR-009](./ADR-009-flutter-rust-bridge-for-dart-bindings.md) (Dart stream binding). +- Constrained by: [ADR-012](./ADR-012-layered-decode-time-safety-control-loop-with-checkpointed-rollback.md) (emit only guard-verified tokens — no streaming a rolled-back span), [ADR-005](./ADR-005-on-device-only-tiered-decoder-time-safety.md) (decode order invariant). +- Implementation seams: `crates/el-runtime` (`generate_with_policy` emit hook), `crates/adapters/el-engine-candle` (`chat_stream` incremental detokenize), `crates/adapters/el-ffi` (per-binding stream wrappers). +- Measured by: [ADR-023](./ADR-023-baseline-performance-instrumentation.md) (real `ttft_ms`). diff --git a/docs/adr/ADR-020-batched-single-pass-prefill.md b/docs/adr/ADR-020-batched-single-pass-prefill.md new file mode 100644 index 0000000..654e4e0 --- /dev/null +++ b/docs/adr/ADR-020-batched-single-pass-prefill.md @@ -0,0 +1,89 @@ +# ADR-020: Batched (single-pass) prefill + +- **Status**: proposed +- **Date**: 2026-06-22 +- **Deciders**: +- **Tags**: runtime, performance, on-device, follow-up, P0 + +## Context + +The improvements plan +([docs/research/improvements-plan.md](../research/improvements-plan.md) §P1.8, +pulled into roadmap Phase 1 / EPIC-2) identifies token-by-token prefill as a +removable orchestration cost: a prompt of length `N` should be processed as a +single `(1, N)` forward, not `N` separate forwards. + +The current engine does exactly the slow thing: + +- `QwenEngine::prefill` loops `self.forward_one(t)` once **per prompt token**, + each call running a `(1, 1)` candle forward and advancing `index_pos` by one. +- `candle-transformers`' `quantized_qwen2::ModelWeights::forward(&input, index_pos)` + already accepts a multi-token `input` tensor, so the batched shape is available + at the seam — only the adapter drives it one token at a time. +- `CandleEngine::prefill` (the linear engine-seam proof) returns `tokens.len()` + without a real forward, so the cost lives entirely in the real `QwenEngine`. +- The `InferenceEngine` port defines `prefill(&mut self, tokens) -> Result` + with no batched variant; `next_logits` is the separate decode step. + +Prefill dominates first-response latency for long prompts (system prompt + RAG +context + history), so this is a direct lever on TTFT and on the per-turn cost +that ADR-018 reduces by reuse. + +## Decision + +Process the prompt in a single batched forward, keeping prefill and decode +distinct, and preserving exact equivalence with sequential prefill. + +1. **Batched prefill at the seam.** Either add `InferenceEngine::prefill_batch` + or make `prefill` itself issue one `(1, N)` forward over the whole (compressed) + prompt. `decode_step`/`next_logits` stays a separate single-token path. + +2. **Equivalence requirement.** Batched prefill must leave the engine in the + **same final KV state** and yield the **same** post-prefill logits as the + current sequential loop (a regression test asserts batched == sequential on the + toy and Qwen engines). Determinism (ADR-002) is preserved. + +3. **Chunked prefill for memory-bound devices.** Devices that cannot fit a full + `(1, N)` activation batch process the prompt in chunks of size drawn from the + ADR-003 memory plan, accumulating KV across chunks to the identical end state. + Chunk size is a planned quantity, not a magic constant. + +4. **Honest prefill telemetry.** `load_prompt` currently emits + `PrefillCompleted { prefill_tps: 0 }` — a placeholder. With batched prefill, + emit a **measured** `prefill_tps` (wired through ADR-023), so the speedup is + observable in CI rather than asserted. + +5. **No safety/grammar change.** Prefill does not sample tokens, so the ADR-005 + decode-order invariant and the ADR-012/013 guard/steer/ingress logic are + untouched; ingress triage still scores the prompt before generation. + +## Consequences + +### Positive +- Prompt-processing latency falls from `N` forwards to one batched forward (plus + chunking only where memory forces it) — the plan's named win. +- Compounds with ADR-018: a persistent session re-prefills only the new suffix, + and that suffix is itself batched. +- Turns the `prefill_tps: 0` placeholder into a real, CI-gated metric. + +### Negative +- A `(1, N)` forward needs a larger transient activation buffer than `(1, 1)`; + on tight devices this forces chunking, partially eroding the gain — bounded by + the ADR-003 plan. +- `prefill_batch` widens the `InferenceEngine` contract; every engine adapter must + provide it (a sequential fallback satisfies the contract for engines that can't + batch). + +### Neutral +- The candle path already supports multi-token `forward`, so this is an adapter + change, not an upstream dependency change. +- Speculative decoding (P1, §9; events `DraftProposed`/`DraftVerified` already + exist) reuses the same prefill/decode split but is out of scope for this + milestone. + +## Links +- Source: [docs/research/improvements-plan.md](../research/improvements-plan.md) §P1.8 (Phase 1 / EPIC-2). +- Builds on: [ADR-002](./ADR-002-candle-as-rust-native-inference-engine.md) (candle multi-token forward), [ADR-018](./ADR-018-persistent-model-instances-and-stateful-sessions.md) (suffix-only prefill on reuse). +- Constrained by: [ADR-003](./ADR-003-static-memory-planning-with-zero-allocation-arena.md) (chunk size from the memory plan), [ADR-005](./ADR-005-on-device-only-tiered-decoder-time-safety.md)/[ADR-012](./ADR-012-layered-decode-time-safety-control-loop-with-checkpointed-rollback.md) (decode/guard semantics unchanged). +- Implementation seams: `crates/el-runtime` (`ports::InferenceEngine::prefill`/`prefill_batch`, `InferenceSession::load_prompt`), `crates/adapters/el-engine-candle` (`QwenEngine::prefill`). +- Measured by: [ADR-023](./ADR-023-baseline-performance-instrumentation.md) (measured `prefill_tps`, prompt latency). diff --git a/docs/adr/ADR-021-memory-mapped-verified-gguf-loading.md b/docs/adr/ADR-021-memory-mapped-verified-gguf-loading.md new file mode 100644 index 0000000..ba37069 --- /dev/null +++ b/docs/adr/ADR-021-memory-mapped-verified-gguf-loading.md @@ -0,0 +1,97 @@ +# ADR-021: Memory-mapped verified GGUF loading + +- **Status**: proposed +- **Date**: 2026-06-22 +- **Deciders**: +- **Tags**: runtime, performance, provenance, on-device, follow-up, P0 + +## Context + +The improvements plan +([docs/research/improvements-plan.md](../research/improvements-plan.md) §P0.5, +roadmap Phase 1, EPIC-3) wants memory-mapped GGUF loading to cut startup latency, +avoid copying the whole model into anonymous heap, and let the OS page unused +weights — **without** weakening the provenance gate. + +Today loading is fully buffered and re-done per turn: + +- `QwenEngine::from_path` opens the file, reads the GGUF header with + `gguf_file::Content::read`, then calls `Qwen2Weights::from_gguf(content, &mut + file, &device)`, materializing weights into resident tensors on the CPU device. + `CandleEngine` additionally `dequantize`s `token_embd.weight` / `output.weight` + into full `f32` tensors. +- `from_bytes` is documented as the "WASM / memory-mapped scenario," but it is a + `std::io::Cursor` over an in-memory `&[u8]` — **not** an actual `mmap`. +- Because `QwenChatProvider::chat` rebuilds the engine each turn (ADR-018), this + buffered load is paid **per call**. +- Provenance ordering matters: `local_load_permit` issues a `LoadPermit` through + the ADR-006 gate, but it is currently a *trust-the-local-file* verifier (no + cryptographic check of the GGUF bytes — see ADR-013 §Implementation status). + The load order must remain: file → signature verification → verified permit → + tensor/mmap construction → session. + +## Decision + +Add a memory-mapped model source beneath `el-engine-candle` that maps the GGUF +file and constructs tensor views over the mapping, with verification strictly +**before** any view is built. + +1. **`MmapModelSource`.** A loader that `mmap`s the GGUF and hands candle tensor + views backed by the mapping instead of heap copies. The mapping is kept alive + for the model's entire lifetime (tensor views must not outlive it). + +2. **Verify-before-map ordering (ADR-006).** Provenance verification reads/digests + the file and issues the `LoadPermit` **before** executable tensor views are + constructed. A corrupt or altered file fails at verification, never at first + inference. This preserves the type-level gate: a session still cannot be built + without a `LoadPermit`. (Replacing the trust-the-file verifier with real + whole-artifact signature verification is tracked under ADR-006/ADR-013 and is a + precondition for production signed assets, not introduced here.) + +3. **Quantized stays mapped.** Where the GGUF is already quantized, prefer mapping + the quantized bytes and dequantizing **on demand** (pairs with ADR-022's + tiered KV / on-demand dequant) rather than eagerly materializing full-precision + tensors — that is where the resident-memory win comes from. The realized saving + is engine-dependent and recorded as a measured result, not assumed. + +4. **Buffered fallback.** Targets without usable `mmap` (notably wasm32) fall back + to the existing buffered/`from_bytes` path. The public engine API is unchanged; + only the load strategy is selected per target. + +5. **Per-OS validation.** Android, iOS, Linux, macOS, and Windows are tested + separately — `mmap` semantics, file-locking, and page behavior differ across + them. + +## Consequences + +### Positive +- Lower cold-start latency and lower peak RSS: the OS pages weights on demand + instead of a full read-into-heap (+ dequantize) up front. +- Combined with ADR-018 (load once, share across sessions), the per-turn load cost + disappears and the mapped pages are shared. +- Provenance is preserved exactly: verification still gates session construction. + +### Negative +- `mmap` + tensor-views-over-mapping needs careful lifetime handling; the mapping + must outlive every view (a use-after-unmap would be a soundness bug — note the + crate is `#![forbid(unsafe_code)]`, so the mapping must come from a safe `mmap` + abstraction). +- Real integrity (vs. trust-the-file) means hashing/verifying the whole artifact + before mapping, which costs a one-time read — it trims the *copy*, not the + *verify*. Honest accounting required. +- Platform variance (esp. wasm has no `mmap`; iOS file protections) means two code + paths to maintain and test. + +### Neutral +- The buffered loader remains the reference and the wasm path; this is an additive + strategy selection. +- Tensor numerics are unchanged — mapping changes *where bytes live*, not the + forward result; determinism (ADR-002) holds. + +## Links +- Source: [docs/research/improvements-plan.md](../research/improvements-plan.md) §P0.5, EPIC-3. +- Builds on: [ADR-002](./ADR-002-candle-as-rust-native-inference-engine.md) (GGUF/candle), [ADR-018](./ADR-018-persistent-model-instances-and-stateful-sessions.md) (load once, share). +- Constrained by: [ADR-006](./ADR-006-mandatory-ed25519-model-signature-verification-load-gate.md) (verify → permit → map ordering; real whole-artifact verification still owed), [ADR-008](./ADR-008-implement-the-sdk-in-rust-instead-of-c-cpp.md) (no `unsafe` — safe mmap abstraction), [ADR-001](./ADR-001-adopt-webassembly-as-cross-platform-sdk-runtime.md) (wasm has no mmap → fallback). +- Pairs with: [ADR-022](./ADR-022-two-tier-quantized-kv-cache-with-attention-aware-eviction.md) (map quantized bytes, dequantize on demand). +- Implementation seams: `crates/adapters/el-engine-candle` (`QwenEngine`/`CandleEngine` load path, new `MmapModelSource`), `crates/el-provenance` (verify-before-map ordering). +- Measured by: [ADR-023](./ADR-023-baseline-performance-instrumentation.md) (cold start, peak RSS vs. buffered). diff --git a/docs/adr/ADR-022-two-tier-quantized-kv-cache-with-attention-aware-eviction.md b/docs/adr/ADR-022-two-tier-quantized-kv-cache-with-attention-aware-eviction.md new file mode 100644 index 0000000..3ab83a1 --- /dev/null +++ b/docs/adr/ADR-022-two-tier-quantized-kv-cache-with-attention-aware-eviction.md @@ -0,0 +1,110 @@ +# ADR-022: Two-tier quantized KV cache with attention-aware eviction + +- **Status**: proposed +- **Date**: 2026-06-22 +- **Deciders**: +- **Tags**: memory, runtime, performance, on-device, follow-up, P0 + +## Context + +The improvements plan +([docs/research/improvements-plan.md](../research/improvements-plan.md) §P0.3 + +§P0.4, roadmap Phase 2 opener / EPIC-4 — the stated first-milestone cutoff) +combines two KV concerns: a **two-tier quantized KV cache** (recent tokens at +high precision, older tokens quantized to Q8→Q4→Q3) and **attention-aware +eviction** (sliding-window first, then H2O/PyramidKV-style retention) with pinned +ranges for system prompts and tool definitions. KV memory grows with context and +is the dominant *mutable* cost after weights, so this is the milestone's +memory-reduction payload. + +The codebase has KV in two layers, neither of which quantizes or evicts: + +- **Session layer.** `el-memory::KvRegion` is **descriptor-only**: `KvSlot { + token_index, offset, valid }` — it tracks *where* KV lives, not the K/V tensors + themselves (ADR-003). It offers `push`, `truncate` (the ADR-012 rollback + primitive), `mark_pruned`, and `compact` (for rejected drafts) — but no + precision tiers and no eviction policy. `compact` emits `KvCacheCompacted`. +- **Engine layer.** The real K/V tensors live inside candle's `quantized_qwen2`, + which is **append-only with no public truncation** (the same limitation ADR-012 + hit for rollback, forcing prompt replay). +- `SessionConfig.memory_budget_bytes` exists (ADR-003), and + `MemoryBudgetExceeded` is defined, but nothing enforces a per-session KV ceiling + or migrates old KV to a cheaper tier. + +So there is a budget concept and descriptor bookkeeping, but no policy that bounds +or compresses the actual KV growth. + +## Decision + +Introduce a pluggable KV policy in `el-memory` and an eviction strategy layered on +it, degrading deterministically under the ADR-003 budget. + +1. **`KvCachePolicy` port.** Extend `el-memory` from descriptors to a policy over + real KV storage (the plan's interface): + ```rust + pub trait KvCachePolicy { + fn append(&mut self, layer: usize, key: TensorView, value: TensorView) -> Result<(), MemoryError>; + fn view_for_attention(&self, layer: usize, range: TokenRange) -> Result, MemoryError>; + fn memory_usage(&self) -> KvMemoryStats; + } + ``` + Recent tokens stay FP16/BF16; older blocks quantize to **Q8 first**, then Q4/Q3 + as evaluated. Blocks are **dequantized only when attention needs them**. The + uncompressed policy remains the reference implementation. + +2. **Per-session KV budget.** The policy enforces a hard per-session KV ceiling + derived from `memory_budget_bytes`; exceeding it triggers tiering/eviction, and + `MemoryBudgetExceeded` / `KvCacheCompacted` make the action observable + (content-free, ADR-007). + +3. **Eviction, staged.** Start with **deterministic sliding-window** eviction; add + **pinned ranges** (system prompt, tool definitions) that are never evicted; put + **H2O/PyramidKV** attention-scored retention behind an experimental feature + flag. Eviction must never remove tokens required by the grammar/tool-call state + (ADR / grammar constraint) — those are implicitly pinned. + +4. **Engine-capability honesty.** True per-block KV quantization/eviction requires + an engine that exposes its KV cache. candle's `quantized_qwen2` does not, so on + the **current** engine this ADR lands as: (a) the `KvCachePolicy` port + + reference impl + budget accounting at the session/descriptor layer, and (b) + deterministic sliding-window/pinning where the descriptor layer can express it. + Full tiered-tensor quantization is realized only with an engine that owns a + truncatable/inspectable cache (a custom attention kernel — see the deferred + flash/sparse-attention work, plan §P1.7/§P2.17). This contingency is recorded + exactly as ADR-012 recorded the rollback-replay contingency. + +5. **Determinism.** In deterministic mode, eviction decisions are reproducible + (sliding-window and pinning are pure functions of token positions; the + experimental attention-scored path is excluded from deterministic builds). + +## Consequences + +### Positive +- Bounds the dominant mutable memory cost: longer conversations or smaller budgets + on phones/embedded targets without blunt whole-history truncation. +- Pinned system/tool ranges preserve instruction-following while trimming + mid-history — better long-context quality than FIFO. +- Pluggable policy keeps an uncompressed reference for A/B quality regression. + +### Negative +- Real tensor-level tiering needs engine KV access that candle's + `quantized_qwen2` doesn't expose — full realization waits on a custom kernel; + until then the win is partial (budget + descriptor-layer eviction). +- Quantization adds dequant overhead on the attention read path; if it erases the + decode-time gain it is not worth shipping — gated by the ADR-023 benchmark. +- Q4/Q3 tiers risk quality regression; each tier must pass deterministic quality + tests before promotion past Q8. + +### Neutral +- `KvRegion`'s descriptor model, `truncate` (ADR-012 rollback), and `compact` + remain; this ADR adds a policy *above* them, not a replacement. +- Interacts with ADR-021: mapping quantized weights and on-demand dequant share + the same "decompress only what's needed" discipline. + +## Links +- Source: [docs/research/improvements-plan.md](../research/improvements-plan.md) §P0.3 + §P0.4, EPIC-4 (milestone cutoff). +- Builds on: [ADR-003](./ADR-003-static-memory-planning-with-zero-allocation-arena.md) (memory budget, `KvRegion` descriptors), [ADR-018](./ADR-018-persistent-model-instances-and-stateful-sessions.md) (per-session KV lifecycle). +- Constrained by: [ADR-002](./ADR-002-candle-as-rust-native-inference-engine.md) (engine KV access — candle has no cache truncation), [ADR-012](./ADR-012-layered-decode-time-safety-control-loop-with-checkpointed-rollback.md) (rollback `truncate` must keep working under any policy), [ADR-007](./ADR-007-content-free-domain-events-privacy-by-construction-telemetry.md) (eviction events stay content-free). +- Pairs with: [ADR-021](./ADR-021-memory-mapped-verified-gguf-loading.md) (on-demand dequant discipline). +- Implementation seams: `crates/el-memory` (`KvRegion`, new `KvCachePolicy`/`KvView`/`KvMemoryStats`), `crates/adapters/el-engine-candle` (`QwenEngine` cache access — contingent on a truncatable cache), `crates/el-core` (`MemoryBudgetExceeded`/`KvCacheCompacted` events). +- Measured by: [ADR-023](./ADR-023-baseline-performance-instrumentation.md) (peak KV bytes, decode tok/s with vs. without tiering, long-context quality). diff --git a/docs/adr/ADR-023-baseline-performance-instrumentation.md b/docs/adr/ADR-023-baseline-performance-instrumentation.md new file mode 100644 index 0000000..d87019c --- /dev/null +++ b/docs/adr/ADR-023-baseline-performance-instrumentation.md @@ -0,0 +1,100 @@ +# ADR-023: Baseline performance instrumentation + +- **Status**: proposed +- **Date**: 2026-06-22 +- **Deciders**: +- **Tags**: telemetry, performance, testing, follow-up, P0 + +## Context + +The improvements plan +([docs/research/improvements-plan.md](../research/improvements-plan.md) roadmap +Phase 1 §6, drawing on §P2.13 evaluation/ablation) makes baseline performance +instrumentation a Phase-1 deliverable: every optimization in this milestone +(ADR-018..022) must ship with a **measured** before/after, or the speedups are +unfalsifiable. The plan warns the ruvLLM performance figures are *claims* to be +independently benchmarked inside EdgeIntelligence before they become acceptance +criteria. + +Instrumentation today is split and largely unpopulated: + +- **`el-telemetry`** has the right shape — `MetricsCollector` folds content-free + `DomainEvent`s into `TelemetrySnapshot { prefill_tps, decode_tps, ttft_ms, + peak_bytes, tokens_generated, … }` (ADR-007). But the **runtime never feeds it + real numbers**: `load_prompt` emits `PrefillCompleted { prefill_tps: 0 }` (a + placeholder), and `MetricsSampled { decode_tps, ttft_ms, peak_bytes }` is + defined but **never emitted** by `generate_with_policy`. `ttft_ms` only moves in + unit tests. +- **`el-engine-candle`** has a *second*, disjoint path: an `EL_BENCH`-gated `bench` + module that prints a per-phase wall-clock breakdown (load / tokenize / prefill / + decode / detok, plus model-vs-seam-vs-loop forward attribution and tok/s) to + **stderr**. It is useful but ad hoc, adapter-local, and not machine-readable. +- **`apps/el-bench`** exists as the harness but has no standardized, CI-consumable + metric output to gate regressions. + +So there are two measurement systems, neither of which produces a structured +baseline a CI gate can read. + +## Decision + +Make `el-telemetry` the single instrumentation authority, populate the existing +events with real measurements, and emit a machine-readable benchmark record from +`apps/el-bench`. + +1. **Populate the existing event schema.** Compute and emit a **measured** + `prefill_tps` from `load_prompt` (real once ADR-020 batches prefill), and emit + `MetricsSampled { decode_tps, ttft_ms, peak_bytes }` from the decode loop — + `ttft_ms` taken at the first **emitted** token (ADR-019). No new content on + events; the ADR-007 `Copy`-only guarantee is preserved. + +2. **One measurement path.** Fold the `EL_BENCH` stderr breakdown into a *view* + over the same `TelemetrySnapshot` / event stream rather than a parallel timing + system — the pretty stderr report becomes a renderer of the canonical numbers, + not an independent source of them. + +3. **Machine-readable benchmark output.** `apps/el-bench` emits a structured + (JSON) record per run covering the plan's metric list where feasible on the + host: binary size, cold/warm startup, peak RSS, time-to-first-token, decode + tokens/s, prefill tokens/s, quality/safety scores, determinism, storage growth. + Each record is tagged with **device, model, build flags, and context size** so + numbers are never quoted context-free. + +4. **Ablation per optimization.** Each P0 optimization ships behind a switch and + with a baseline-vs-enabled comparison captured through this path: persistent + sessions (ADR-018), streaming TTFT (ADR-019), batched prefill (ADR-020), mmap + load (ADR-021), quantized/bounded KV (ADR-022). + +5. **Regression gates.** Safety/quality regression thresholds (reusing the + clinical-safety benchmark) and key performance thresholds **block merges** in + CI; benchmark output is the artifact those gates read. + +## Consequences + +### Positive +- Establishes the measured baseline the whole milestone is judged against; ruvLLM + claims become locally verified numbers or get rejected. +- Collapses two instrumentation paths into one content-free, CI-readable source of + truth. +- Makes every P0 optimization independently switchable and provable — no "it feels + faster." + +### Negative +- Taking timings/RSS on the hot path has a (small) cost; sampling must stay cheap + and off the per-token critical section where possible. +- Cross-platform metrics (energy, peak RSS) are not uniformly available; the + record marks unavailable fields rather than faking them. +- A real CI perf gate needs stable hardware or tolerance bands to avoid flakiness. + +### Neutral +- This is logically the *first* thing to land (instrument before optimizing), even + though it is numbered last in this batch — ADR-018..022 each depend on it for + their acceptance evidence. +- No change to the content-free telemetry guarantee (ADR-007); only numeric/enum + fields are populated. + +## Links +- Source: [docs/research/improvements-plan.md](../research/improvements-plan.md) Phase 1 §6 (and §P2.13 evaluation/ablation). +- Builds on: [ADR-007](./ADR-007-content-free-domain-events-privacy-by-construction-telemetry.md) (`DomainEvent`/`MetricsCollector`/`TelemetrySnapshot`). +- Measures: [ADR-018](./ADR-018-persistent-model-instances-and-stateful-sessions.md), [ADR-019](./ADR-019-in-loop-incremental-decoding-and-token-streaming.md), [ADR-020](./ADR-020-batched-single-pass-prefill.md), [ADR-021](./ADR-021-memory-mapped-verified-gguf-loading.md), [ADR-022](./ADR-022-two-tier-quantized-kv-cache-with-attention-aware-eviction.md). +- Implementation seams: `crates/el-telemetry` (`MetricsCollector`/`TelemetrySnapshot`), `crates/el-runtime` (emit measured `prefill_tps` + `MetricsSampled`), `crates/adapters/el-engine-candle` (fold `EL_BENCH` into the event view), `apps/el-bench` (machine-readable record + CI gate). +- Related: [ADR-012](./ADR-012-layered-decode-time-safety-control-loop-with-checkpointed-rollback.md)/[ADR-013](./ADR-013-model-backed-steering-layers-for-the-hybrid-safety-control-loop.md) (safety/quality regression gates reuse the clinical-safety benchmark). diff --git a/docs/adr/README.md b/docs/adr/README.md index 6c3c352..05b75ad 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -21,6 +21,12 @@ the `adr-patterns` namespace. | [011](./ADR-011-multi-registry-release-ci-crates-io-npm-pub-dev.md) | Multi-Registry Release CI (crates.io, npm, pub.dev) | accepted | ci, release, crates.io, npm, pub.dev, packaging | | [012](./ADR-012-layered-decode-time-safety-control-loop-with-checkpointed-rollback.md) | Layered decode-time safety control loop with checkpointed rollback | accepted | safety, security, on-device, runtime, supporting | | [013](./ADR-013-model-backed-steering-layers-for-the-hybrid-safety-control-loop.md) | Model-backed steering layers for the hybrid safety control loop | proposed | safety, security, on-device, runtime, follow-up | +| [018](./ADR-018-persistent-model-instances-and-stateful-sessions.md) | Persistent model instances and stateful inference sessions | proposed | runtime, performance, on-device, follow-up, P0 | +| [019](./ADR-019-in-loop-incremental-decoding-and-token-streaming.md) | In-loop incremental decoding and real token streaming | proposed | runtime, performance, bindings, follow-up, P0 | +| [020](./ADR-020-batched-single-pass-prefill.md) | Batched (single-pass) prefill | proposed | runtime, performance, on-device, follow-up, P0 | +| [021](./ADR-021-memory-mapped-verified-gguf-loading.md) | Memory-mapped verified GGUF loading | proposed | runtime, performance, provenance, on-device, follow-up, P0 | +| [022](./ADR-022-two-tier-quantized-kv-cache-with-attention-aware-eviction.md) | Two-tier quantized KV cache with attention-aware eviction | proposed | memory, runtime, performance, on-device, follow-up, P0 | +| [023](./ADR-023-baseline-performance-instrumentation.md) | Baseline performance instrumentation | proposed | telemetry, performance, testing, follow-up, P0 | ## Decision relationships @@ -42,6 +48,22 @@ flowchart LR A012 --> A013[013 Model-backed steering layers] A003 -. budget triggers degradation .-> A012 A006 -. signs safety weights .-> A013 + + %% P0 performance milestone (docs/research/improvements-plan.md, EPIC 1-4) + A010 --> A018[018 Persistent model + stateful sessions] + A018 --> A019[019 In-loop streaming] + A012 -. emit only guard-verified tokens .-> A019 + A018 --> A020[020 Batched prefill] + A018 --> A021[021 Mmap'd verified load] + A006 -. verify before map .-> A021 + A018 --> A022[022 Two-tier quantized KV + eviction] + A003 -. KV budget / degradation .-> A022 + A007 --> A023[023 Baseline instrumentation] + A023 -. measures .-> A018 + A023 -. measures .-> A019 + A023 -. measures .-> A020 + A023 -. measures .-> A021 + A023 -. measures .-> A022 ``` > **ADR-008 is foundational** (the language decision) and drives the revisions to diff --git a/docs/benchmarks/2026-06-14-qwen-chat-bottleneck.md b/docs/benchmarks/2026-06-14-qwen-chat-bottleneck.md index 79b1547..7b3be61 100644 --- a/docs/benchmarks/2026-06-14-qwen-chat-bottleneck.md +++ b/docs/benchmarks/2026-06-14-qwen-chat-bottleneck.md @@ -5,6 +5,10 @@ `el-chat` test client (`el_core::LlmProvider` → `el_engine_candle::QwenChatProvider` → `el_runtime::InferenceSession`). **Question:** Where does the time go, and what is the dominant bottleneck? +**Status:** ⭐ **Performance baseline** — the canonical *pre-ADR-018* reference for +the local inference path. Later performance runs are compared against this report. +See [2026-06-22 — after ADR-018 (full suite)](./2026-06-22-adr-018-resident-model-full-suite.md), +which retires bottleneck #2 and re-measures #1/#3. --- diff --git a/docs/benchmarks/2026-06-22-adr-018-resident-model-full-suite.md b/docs/benchmarks/2026-06-22-adr-018-resident-model-full-suite.md new file mode 100644 index 0000000..4b85b01 --- /dev/null +++ b/docs/benchmarks/2026-06-22-adr-018-resident-model-full-suite.md @@ -0,0 +1,221 @@ +# SDK Benchmark — Full-suite performance after ADR-018 (el-bench / Qwen2.5-0.5B) + +**Date:** 2026-06-22 +**Subject:** End-to-end performance of the local inference path exercised by the +`el-bench` harness (`el_core::LlmProvider` → `el_engine_candle::QwenChatProvider` +→ `el_runtime::InferenceSession`), across the **full** normalized benchmark suite +(CounselBench, MindEval, VERA-MH). +**Question:** After [ADR-018](../adr/ADR-018-persistent-model-instances-and-stateful-sessions.md) +(persistent model + stateful sessions) landed, where does the time go now — and +which of the [2026-06-14 bottlenecks](./2026-06-14-qwen-chat-bottleneck.md) remain? + +--- + +## 1. Executive summary + +Measured on the **same host** as the 2026-06-14 bottleneck report (Intel i5-14500, +14C/20T, 32 GB; release build), driving the real `el-bench` binary over **all 66 +tasks → 97 model replies, 0 errors, in 53.7 min (3221 s)**, safety `Off` (raw-model +baseline — the SDK safety layer is benched separately). + +The picture has shifted exactly as the prior report predicted: + +| 2026-06-14 bottleneck | Status now | Evidence | +|---|---|---| +| **#2 Full model reload every turn** (~1.2 s/turn) | **FIXED by ADR-018** | `session setup` is `0.0 ms` on all 97 replies — the 491 MB GGUF loads **once** at startup and the resident session is reused; no per-turn reload. | +| **#1 Prefill is not batched** | **Now the #1 cost, confirmed at scale** | Prefill is **56.8%** of all compute (1827 s), running token-by-token at **14.9 tok/s** — *the same rate as decode*. | +| **#3 Decode compute floor** (~15 tok/s) | **Unchanged** (kernel-level) | Decode 43.2% (1388 s) at **14.2 tok/s**; 99.3% candle forward. | + +**Headline:** ADR-018 removed the per-turn weight-reload tax (bottleneck #2). With +that gone, **prefill — still un-batched — is now the single largest cost (56.8% of +compute)**, making [ADR-020 (batched prefill)](../adr/ADR-020-batched-single-pass-prefill.md) +the highest-value next step. The SDK's own per-token glue remains negligible +(< 1% of compute; 99.3% is raw model forward). + +The `mindeval` suite (multi-turn) isolates the second remaining structural cost — +**re-prefilling the whole conversation each turn**: 5 tasks generated 30 replies +that prefilled **16,491 prompt tokens to produce only 4,258 completion tokens +(~4× more prefill than generation)**, dragging its end-to-end rate to 2.7 tok/s. +That is the case for ADR-018 **AC-3** (cross-turn prefix reuse / incremental +prefill), deferred from this increment. + +### 1.1 Versus the [baseline](./2026-06-14-qwen-chat-bottleneck.md) + +The two runs use **different harnesses and inputs**: the baseline is `el-chat` +micro-runs (hand-picked prompts, 4–128 max-tokens) chosen to *isolate* each +bottleneck; this is the full `el-bench` suite (66 tasks, 256 max-tokens). So **raw +totals are not comparable run-for-run** — but the prompt-independent quantities +(per-phase rates, per-token cost, SDK overhead, bottleneck status) are, and that is +the fair comparison. Same host both times (i5-14500, release). + +| Metric (comparable across harnesses) | Baseline (2026-06-14) | After ADR-018 (2026-06-22) | Change | +|---|---|---|---| +| **Per-turn model reload** | ~1.2 s warm / 1.5 s cold, **every turn** | **0.0 ms** (loaded once) | **eliminated** ✅ | +| Prefill throughput | ~15 tok/s (14.6–16.4) | 14.9 tok/s | unchanged — ADR-020 not yet done | +| Decode throughput | ~15 tok/s (13.3–17.4) | 14.2 tok/s | unchanged — kernel floor (#3) | +| Per-token decode (model) | 68.5 ms | ~65–81 ms | unchanged | +| SDK glue overhead | < 1.2% of decode | ~0.7% of compute | unchanged (negligible) | +| Dominant structural cost | #1 prefill **+** #2 reload | #1 prefill **alone** (56.8%) | #2 removed | + +**Reading:** ADR-018 targeted exactly one of the three baseline bottlenecks — **#2, +the per-turn weight reload** — and eliminated it (a flat ~1.2 s/turn tax → 0). By +design it does **not** touch prefill batching or the decode kernel, so the +per-phase *rates* are unchanged; what changed is that the flat reload tax is gone +and prefill (#1) now stands alone as the dominant cost. The two remaining baseline +bottlenecks are unchanged: prefill batching ([ADR-020](../adr/ADR-020-batched-single-pass-prefill.md)) +and the decode floor (#5). The baseline's projected "≈3× faster second turn" assumed +#1+#2+#3 together; this increment banked the #2 portion, and the §3 data shows #1 is +now the larger remaining prize. + +--- + +## 2. Method + +### 2.1 Harness + +The benchmark drives the **actual** `el-bench` release binary — no synthetic +harness — so every number reflects the real public SDK path +(`LlmProvider::chat` → `QwenChatProvider` → `QwenEngine` + `InferenceSession`), +including the ADR-018 resident-session reuse landed in this branch. + +Per-phase timing comes from the same opt-in, env-gated instrumentation +(`EL_BENCH=1`) used in the 2026-06-14 report (`crates/adapters/el-engine-candle/src/lib.rs`): +each `chat()` is split into **session setup → tokenize → prefill → decode → +detokenize**, and each `forward_one` into **model** (candle transformer forward) +vs **seam** (tensor build + `to_vec1` + float→milli-logit), with the remainder +attributed to the runtime **loop**. Zero-cost when unset. + +Per-reply latency and token counts are also recorded in the transcript JSONL +(`prompt_tokens`, `completion_tokens`, `ms` per exchange); the per-suite table +below is computed from those, the prefill/decode split from the `EL_BENCH` log. + +### 2.2 Configuration + +- **Model:** `qwen2.5-0.5b-instruct-q4_k_m.gguf` (491 MB) + `tokenizer.json`, local `models/` (air-gapped, ADR-004). +- **Build:** `cargo build --release -p el-bench` (`opt-level=3`, LTO). +- **Safety:** `SafetyMode::Off` — el-bench benches the raw model; the ADR-005/012 safety loop is evaluated in the clinical-safety report. +- **Decoding:** deterministic greedy argmax → reproducible. +- **Tasks:** `benchmarks/tasks/{counselbench-adv,counselbench-eval,mindeval,veramh}.jsonl`, default `--max-tokens 256`, no `--limit` (full suite). +- **CPU:** Intel i5-14500 (6 P + 8 E, 20 threads); candle `gemm`/rayon CPU backend; no GPU. + +--- + +## 3. Raw results + +### 3.1 Per-suite (end-to-end, from the transcript) + +`avg ms/reply` is wall time per `chat()` (prefill + decode); `e2e tok/s` is +`completion_tokens / total_time` — an output rate that *includes* prefill, so it +falls as prefill grows. + +| Suite | tasks | replies | prompt tok | compl tok | avg ms/reply | e2e tok/s | +|-------|------:|--------:|-----------:|----------:|-------------:|----------:| +| counselbench-adv | 24 | 24 | 3,168 | 5,717 | 23,935 | 10.0 | +| counselbench-eval | 20 | 20 | 3,126 | 5,112 | 25,841 | 9.9 | +| mindeval | 5 | 30 | 16,491 | 4,258 | 52,225 | 2.7 | +| veramh | 17 | 23 | 4,363 | 4,706 | 24,481 | 8.4 | +| **TOTAL** | 66 | 97 | 27,148 | 19,793 | 33,207 | 6.1 | + +### 3.2 Prefill vs decode (summed over all 97 replies, from `EL_BENCH`) + +| Phase | wall | share | tokens | throughput | +|-------|-----:|------:|-------:|-----------:| +| **prefill** | 1827.2 s | **56.8%** | 27,148 | **14.9 tok/s** | +| **decode** | 1387.9 s | 43.2% | 19,696 | **14.2 tok/s** | + +**Compute attribution:** the candle model forward is **99.3%** of prefill+decode +wall; seam quantization + the runtime decode loop are ~0.7% combined. +Representative per-token decode: ~65–82 ms = model ~65–81 + seam ~0.4 + loop ~0.3. + +--- + +## 4. Analysis + +### 4.1 Bottleneck #2 (model reload) is gone — ADR-018 + +The 2026-06-14 report measured a **~1.2 s warm / ~1.5 s cold weight reload on every +`chat()`**, because the engine was rebuilt per turn (candle couples weights + KV +cache in one `ModelWeights`, and there was no public cache reset). ADR-018 made +`QwenChatProvider` load the `QwenEngine` once into a resident `Mutex` +and reuse one session per turn, evicting only the conversation's KV via a +position-0 forward (`reset_cache`). The effect is visible on **every** reply: + +``` +│ session setup 0.0 ms (weights loaded once at startup — ADR-018) +``` + +Across 97 replies that removes ~97 × ~1.2 s ≈ **~2 min** of pure reload tax plus 97 +redundant 491 MB reads/dequantizations — and, more importantly, removes the flat +per-turn tax that dominated short turns in the prior report. + +### 4.2 Bottleneck #1 (prefill not batched) is now the largest cost + +With the reload gone, prefill is the biggest line item: **56.8% of all compute**, +at **14.9 tok/s — identical to decode's 14.2 tok/s**. That equality is the +signature of no prefill batching: the prompt is fed as `prompt_len` independent +single-token forwards instead of one batched `(1, prompt_len)` forward (root cause +unchanged from the prior report, `QwenEngine::prefill`). candle already accepts the +batched shape, so this is a pure SDK fix — +[ADR-020](../adr/ADR-020-batched-single-pass-prefill.md) — with an expected +order-of-magnitude prefill win. **It is now the highest-value optimization in the +codebase.** + +### 4.3 Multi-turn re-prefill — the `mindeval` signal (ADR-018 AC-3) + +`mindeval`'s 5 multi-turn tasks produced 30 replies that prefilled **16,491 prompt +tokens for only 4,258 generated** — because each turn re-prefills the entire +growing conversation from scratch (the system prompt + every prior user/assistant +turn). Its end-to-end rate (2.7 tok/s) is ~4× worse than the largely single-turn +suites (8–10 tok/s). ADR-018 deferred **AC-3** (reuse the unchanged-prefix KV and +prefill only the new suffix); this is the data motivating it. ADR-020 (batch) and +AC-3 (don't re-prefill) are complementary: batch makes the necessary prefill fast; +AC-3 removes the unnecessary prefill. + +### 4.4 Bottleneck #3 (decode floor ~14 tok/s) — unchanged + +Decode is still **99.3% candle q4_k_m forward** at ~14 tok/s — a +memory-bandwidth-bound batch-1 decode floor, an engine/kernel concern, not SDK +orchestration. Unchanged by ADR-018 and out of scope for the P0 milestone. + +--- + +## 5. Recommendations (prioritized) + +1. **Batch the prefill — [ADR-020](../adr/ADR-020-batched-single-pass-prefill.md).** + Now the single largest cost (56.8%). Replace the per-token loop in + `QwenEngine::prefill` with one `(1, prompt_len)` forward. Biggest win, pure SDK. +2. **Cross-turn prefix reuse — ADR-018 AC-3.** Prefill only new suffix tokens; + removes the redundant re-prefill that makes `mindeval`-style multi-turn ~4× + slower. Now unblocked by the resident session this PR added. +3. **Real token streaming — [ADR-019](../adr/ADR-019-in-loop-incremental-decoding-and-token-streaming.md).** TTFT is still full-generation time (`chat_stream` replays a finished reply); a per-token emit hook drops it to `prefill + 1 token`. UX, separate from throughput. +4. **Baseline instrumentation as a CI gate — [ADR-023](../adr/ADR-023-baseline-performance-instrumentation.md).** Emit measured `prefill_tps`/`MetricsSampled` and capture this run as the machine-readable baseline so ADR-020's win is regression-gated. +5. **Faster quantized decode kernel** *(largest absolute ceiling, hardest)* — the ~14 tok/s floor is candle's CPU matmul; revisit after 1–3. + +### Rough projected impact + +A representative multi-turn second turn today is prefill-bound (≈83–95% of wall in +the bounded run). Batching prefill (#1) should cut that to low single-digit seconds; +adding AC-3 (#2) removes most of it entirely. The `mindeval` aggregate (16.5 k +prefill tok → 4.3 k generated) is where the combined win is largest. Decode (#5) +remains the residual floor. + +--- + +## 6. Reproduce + +```sh +cargo build --release -p el-bench + +# Full suite (this run): all tasks, default 256 max-tokens, EL_BENCH timing. +EL_BENCH=1 ./target/release/el-bench.exe \ + --out benchmarks/out/transcripts.jsonl + +# Bounded smoke (fast, representative): 3 veramh tasks, shorter replies. +EL_BENCH=1 ./target/release/el-bench.exe \ + --suite veramh --limit 3 --max-tokens 128 --out /tmp/perf.jsonl +``` + +`EL_BENCH` prints the per-phase table + forward attribution to stderr; the +transcript JSONL records per-reply `prompt_tokens` / `completion_tokens` / `ms`. +Per-suite figures are aggregated from the transcript, the prefill/decode split +from the `EL_BENCH` stderr log. diff --git a/docs/research/improvements-plan.md b/docs/research/improvements-plan.md new file mode 100644 index 0000000..4b3d75f --- /dev/null +++ b/docs/research/improvements-plan.md @@ -0,0 +1,663 @@ +Below is a prioritized backlog based on **value to EdgeIntelligence’s edge-device goals**, not ruvLLM’s marketing order. I weighted memory reduction, startup time, latency, mobile usefulness, architectural fit, and implementation risk. + +The performance figures are ruvLLM project claims and should be independently benchmarked inside EdgeIntelligence before becoming acceptance criteria. + +## P0 — Highest-value foundation + +### 1. Persistent model instances and stateful inference sessions + +**Description:** Keep model weights loaded across requests and maintain a session-specific inference state, including conversation tokens and KV-cache references. `RuvLLMEngine` includes session lifecycle management, persistent session indexing, and KV-cache references for multi-turn interaction. ([GitHub][1]) + +**Value to EdgeIntelligence:** This directly addresses EdgeIntelligence’s largest identified orchestration issue: it currently needs to load weights once, reset only the KV cache when appropriate, and reuse KV state across conversation turns instead of re-prefilling the entire history. ([GitHub][2]) + +**Implementation direction:** + +* Add a `StatefulLlmProvider` or `InferenceSessionHandle`. +* Separate model lifecycle from conversation lifecycle. +* Maintain one shared model instance with multiple isolated sessions. +* Give every session explicit `reset`, `close`, and eviction operations. +* Keep `el-runtime` as the session authority; use ruvLLM’s session structures as an implementation reference or optional adapter. + +**First acceptance criteria:** + +* Model weights load once per provider instance. +* A second turn does not reload the model. +* Unchanged conversation prefixes are not re-prefilled. +* Session memory can be measured and explicitly released. + +--- + +### 2. Incremental decoding and real token streaming + +**Description:** ruvLLM exposes incremental token-generation concepts, token streams, stream events, and a `decode_step()` design for advancing generation one token at a time. Its Node interface also exposes generation as an asynchronous token stream. ([GitHub][3]) + +**Value to EdgeIntelligence:** EdgeIntelligence currently identifies true in-loop streaming as a major opportunity: the first token should be emitted after model load, prefill, and one decode step—not after the full response has completed. ([GitHub][2]) + +**Implementation direction:** + +```rust +pub trait StreamingLlmProvider { + type Stream: Iterator>; + + fn chat_stream( + &self, + session: &SessionId, + request: ChatRequest, + ) -> Result; +} +``` + +The decode pipeline should remain: + +```text +model logits +→ grammar mask +→ safety adjustment +→ sampling +→ KV commit +→ emit token +``` + +**First acceptance criteria:** + +* Time-to-first-token is independently measured. +* Cancellation stops generation and releases temporary buffers. +* Streaming works through Rust, UniFFI, Flutter, and Web bindings. +* Grammar and safety still run before every emitted token. + +--- + +### 3. Two-tier and quantized KV-cache storage + +**Description:** ruvLLM implements a two-tier KV cache in which recent tokens remain at higher precision while older tokens migrate to a quantized tier. It also contains TurboQuant 2–4-bit KV compression and reports configurations for 2-, 3-, 4-, and 8-bit storage. ([GitHub][4]) + +**Value to EdgeIntelligence:** KV memory grows with context length and often becomes the main mutable memory cost after model weights. Quantizing old KV entries could enable longer conversations or smaller memory budgets on phones and embedded devices. + +**Implementation direction:** + +* Extend `el-memory` from KV descriptors to an actual pluggable KV storage policy. +* Preserve recent tokens in FP16/BF16. +* Quantize older blocks to Q8 first, then evaluate Q4 and Q3. +* Dequantize only blocks required by attention. +* Add per-session KV memory limits. + +Suggested interface: + +```rust +pub trait KvCachePolicy { + fn append(&mut self, layer: usize, key: TensorView, value: TensorView) + -> Result<(), MemoryError>; + + fn view_for_attention( + &self, + layer: usize, + range: TokenRange, + ) -> Result, MemoryError>; + + fn memory_usage(&self) -> KvMemoryStats; +} +``` + +**First acceptance criteria:** + +* Q8 cache passes deterministic quality regression tests. +* Peak KV memory decreases by a defined percentage. +* Quantization overhead does not erase decode-time gains. +* The uncompressed implementation remains available as a reference. + +--- + +### 4. Intelligent KV eviction for long contexts + +**Description:** ruvLLM includes H2O and PyramidKV-style eviction approaches that retain tokens considered important based on attention behavior rather than discarding tokens solely by age. It also supports sliding-window and tiered cache configurations. ([GitHub][4]) + +**Value to EdgeIntelligence:** A fixed-size context currently forces a choice between high memory use and blunt truncation. Attention-aware eviction can preserve system instructions, important facts, and salient earlier messages while removing less valuable cache entries. + +**Implementation direction:** + +* Start with deterministic sliding-window eviction. +* Add pinned ranges for system prompts and tool definitions. +* Implement H2O-style scoring behind an experimental feature. +* Expose eviction events as content-free telemetry. +* Do not allow eviction to remove tokens required by grammar or tool-call state. + +**First acceptance criteria:** + +* Hard memory budgets are never exceeded. +* System prompt tokens can be pinned. +* Long-context benchmark quality exceeds simple FIFO truncation. +* Eviction decisions are reproducible in deterministic mode. + +--- + +### 5. Memory-mapped GGUF model loading + +**Description:** ruvLLM supports GGUF loading with memory mapping, quantized model formats, metadata inspection, and optional checksum validation. Its documentation presents memory mapping as a way to improve loading speed and reduce resident memory pressure. ([GitHub][4]) + +**Value to EdgeIntelligence:** This can reduce model startup latency, avoid copying the entire model into anonymous heap memory, and allow the operating system to page unused weights. + +**Implementation direction:** + +* Add an `MmapModelSource` beneath `el-engine-candle`. +* Preserve the existing signature verification gate. +* Verify the file before constructing executable tensor views. +* Keep the mapped file alive for the model’s full lifetime. +* Add fallback buffered loading for unsupported targets. + +The load order must remain: + +```text +model file +→ signature verification +→ verified load permit +→ mmap/tensor construction +→ inference session +``` + +EdgeIntelligence requires verified provenance before a session may be constructed. ([GitHub][2]) + +**First acceptance criteria:** + +* No mmap occurs before provenance validation. +* Startup time and peak RSS are compared with buffered loading. +* Corrupt or altered model files fail before inference. +* Android, iOS, Linux, macOS, and Windows behavior is tested separately. + +--- + +## P1 — Major performance and product capabilities + +### 6. Hardware capability detection and backend selection + +**Description:** ruvLLM contains platform and capability-detection types for CPU features, GPU backends, compute capabilities, and inference configuration. It offers CPU, Metal, CUDA, Apple Neural Engine/Core ML, and hybrid execution paths. ([GitHub][1]) + +**Value to EdgeIntelligence:** A single static backend configuration is unlikely to be optimal across iPhone, Apple Silicon, Android ARM, desktop CUDA, and WebAssembly. Capability detection allows the SDK to select the best supported path while keeping one public API. + +**Implementation direction:** + +```rust +pub struct DeviceCapabilities { + pub cpu_threads: usize, + pub simd: SimdLevel, + pub available_memory_bytes: u64, + pub metal: bool, + pub core_ml: bool, + pub cuda: bool, + pub wasm_simd: bool, +} +``` + +Then derive an explicit execution plan: + +```rust +pub struct ExecutionPlan { + pub backend: BackendKind, + pub model_quantization: QuantizationKind, + pub kv_policy: KvPolicyKind, + pub thread_count: usize, + pub context_limit: usize, +} +``` + +**First acceptance criteria:** + +* Selection is deterministic and inspectable. +* Applications can override every automatic decision. +* Unsupported accelerators fail over cleanly. +* No runtime network access is introduced. + +--- + +### 7. Flash Attention and memory-efficient attention kernels + +**Description:** ruvLLM includes Flash Attention 2 concepts with block sizing and online softmax, as well as optimized Metal kernels. The project describes Flash Attention as reducing attention memory complexity and improving throughput. ([GitHub][4]) + +**Value to EdgeIntelligence:** Attention cost becomes significant as prompt length grows. A memory-efficient attention implementation can reduce temporary allocations and improve prefill speed, particularly for longer prompts. + +**Implementation direction:** + +* Add an attention-kernel port beneath `InferenceEngine`. +* Keep a simple reference implementation. +* Add tiled CPU/SIMD implementation. +* Add Metal implementation for supported Apple targets. +* Validate numerical differences against the reference kernel. + +**First acceptance criteria:** + +* No per-token heap allocation in the optimized path. +* Output error remains within an agreed tolerance. +* Prefill latency improves for representative context lengths. +* Kernel selection remains target- and feature-gated. + +--- + +### 8. Batched prefill + +**Description:** ruvLLM’s serving components support batched requests and separate prefill/decode scheduling. Although continuous batching mainly targets concurrent serving, the same separation can be used to process a single prompt as a matrix rather than one forward pass per input token. ([GitHub][4]) + +**Value to EdgeIntelligence:** EdgeIntelligence’s own benchmark identifies token-by-token prefill as a removable orchestration cost and recommends feeding the prompt in one `(1, prompt_len)` forward operation. ([GitHub][2]) + +**Implementation direction:** + +* Extend `InferenceEngine` with `prefill_batch`. +* Keep `decode_step` separate from prefill. +* Produce the same final KV state as sequential prefill. +* Add chunked prefill for devices that cannot fit the full prompt batch. + +**First acceptance criteria:** + +* Batched and sequential prefill produce equivalent logits/KV state. +* Prompt processing latency falls substantially. +* Chunk size can be selected from the memory plan. +* No change to grammar or safety semantics during decoding. + +--- + +### 9. Speculative decoding + +**Description:** ruvLLM includes speculative decoding in which a smaller draft model proposes several tokens and the target model verifies them together. Its API records acceptance rate and realized speedup. ([GitHub][4]) + +**Value to EdgeIntelligence:** This can materially improve tokens per second when a small draft model closely predicts the target model. It is particularly useful when target-model compute dominates runtime. + +**Implementation direction:** + +* Add `DraftEngine` and `SpeculativeDecoder` abstractions. +* Verify candidate tokens through the existing grammar and safety pipeline. +* Load the draft model only where memory permits. +* Support self-speculation or early-exit speculation later. + +**Important limitation:** Two model instances may increase footprint enough to make this unsuitable for smaller mobile devices. + +**First acceptance criteria:** + +* Generated output distribution remains correct. +* Safety and grammar checks cannot be skipped for accepted draft tokens. +* Report acceptance rate, extra memory, and actual speedup. +* Automatically disable speculation when acceptance is poor. + +--- + +### 10. LoRA adapter loading and hot swapping + +**Description:** ruvLLM contains an `AdapterManager`, LoRA adapter registry/pool, task-specific adapters, hot swapping, and several adapter-composition strategies. ([GitHub][4]) + +**Value to EdgeIntelligence:** One base model could support different products or tasks without shipping several full models. Examples include medical terminology, coding, customer service, language specialization, or organization-specific behavior. + +**Implementation direction:** + +* Create an `AdapterProvider` port. +* Associate adapters with verified manifests. +* Apply adapters at session creation or request boundaries. +* Cache only a small number of active adapters. +* Keep the base model immutable. + +Suggested API: + +```rust +pub trait AdapterProvider { + fn load_verified( + &self, + permit: AdapterLoadPermit, + ) -> Result; + + fn activate( + &self, + session: &SessionId, + adapter: &AdapterId, + ) -> Result<(), AdapterError>; +} +``` + +**First acceptance criteria:** + +* Every adapter is signed and tied to a compatible base-model hash. +* Adapter switching does not reload the base model. +* Adapter memory is bounded and observable. +* Deterministic mode pins an exact adapter version. + +--- + +### 11. Semantic runtime-policy selection + +**Description:** `RuvLLMEngine` includes a RuVector-backed policy store. It can semantically retrieve policies such as quantization settings and routing rules based on a request-context embedding. ([GitHub][1]) + +**Value to EdgeIntelligence:** Different workloads may need different execution strategies. A short classification request might use a tiny model and aggressive quantization, while a structured generation task might use stronger grammar constraints and a larger context allocation. + +Potential policy decisions: + +* Model selection +* KV compression level +* Context limit +* Adapter selection +* Local backend selection +* Speculative decoding enablement +* Safety tier +* Cloud fallback eligibility + +**Implementation direction:** + +* Begin with static rule-based policies. +* Define a serializable, versioned `ExecutionPolicy`. +* Add semantic retrieval only after deterministic rules are stable. +* Require applications to supply or explicitly enable an embedding provider. +* Record the selected policy ID in content-free telemetry. + +**First acceptance criteria:** + +* Policies cannot bypass safety, provenance, or air-gap requirements. +* Deterministic mode pins policy versions. +* Rules have a fallback when embeddings are unavailable. +* Policy lookup overhead is measured separately. + +--- + +### 12. Structured witness records and searchable audit history + +**Description:** ruvLLM includes a witness log for recording latency breakdowns, routing decisions, and quality scores, with optional HNSW semantic search over records. ([GitHub][1]) + +**Value to EdgeIntelligence:** This can improve diagnosis of poor performance and routing decisions, especially across heterogeneous devices. EdgeIntelligence already has content-free telemetry, so the useful part is a richer structured record—not necessarily prompt indexing. + +**Implementation direction:** + +* Extend `el-telemetry` with an optional local execution journal. +* Store IDs, numeric metrics, configuration hashes, and outcomes. +* Do not store prompts, responses, embeddings, or user identifiers by default. +* Make retention and deletion explicit. +* Add semantic search only in a separate opt-in privacy mode. + +**First acceptance criteria:** + +* Default witness events remain content-free. +* Storage has a hard size and retention limit. +* Records can be fully deleted. +* Sensitive modes require explicit host configuration. + +--- + +## P2 — Differentiating intelligence features + +### 13. Evaluation harness and ablation testing + +**Description:** ruvLLM includes an evaluation harness with quality metrics and multiple ablation modes, allowing comparison of baseline inference, retrieval, adapters, and full adaptive configurations. ([GitHub][3]) + +**Value to EdgeIntelligence:** This is essential for determining whether optimizations actually help. EdgeIntelligence already has runtime and clinical/safety benchmark harnesses; adding standardized ablation support would make feature decisions evidence-based. ([GitHub][2]) + +**Implementation direction:** + +Define feature profiles: + +```text +baseline ++ persistent sessions ++ batched prefill ++ quantized KV ++ speculative decoding ++ adapters ++ semantic policy routing ++ adaptive learning +``` + +Measure: + +* Binary size +* Cold startup +* Warm startup +* Peak RSS +* Time to first token +* Decode tokens per second +* Energy consumption +* Quality/safety scores +* Determinism +* Storage growth + +**First acceptance criteria:** + +* Every major optimization ships with an ablation benchmark. +* Performance claims include device, model, build flags, and context size. +* Safety/quality regression thresholds block merges. +* Benchmark outputs are machine-readable for CI. + +--- + +### 14. SONA three-tier adaptive learning + +**Description:** ruvLLM’s SONA system provides instant, background, and deeper learning loops. The instant path uses per-request adaptation, while later loops consolidate accumulated feedback. ([GitHub][4]) + +**Value to EdgeIntelligence:** This could allow on-device personalization and automatic optimization based on user feedback or measured outcomes, without sending private data to a server. + +Potential uses: + +* Learn preferred response style +* Select the best adapter for a task +* Tune routing thresholds +* Adjust generation settings +* Improve repeated workflows + +**Implementation direction:** + +* Do not begin with weight adaptation. +* Start by learning bounded policy values. +* Require explicit feedback rather than treating every response as successful. +* Keep learning state separate from signed model assets. +* Add export, reset, and deletion APIs. +* Disable learning in deterministic and regulated modes. + +**First acceptance criteria:** + +* Learning is opt-in. +* Users can inspect and delete learned state. +* Adaptation cannot modify safety boundaries. +* Regression tests prove that deterministic mode remains unchanged. +* Learned policy rollback is supported. + +--- + +### 15. MicroLoRA per-request adaptation + +**Description:** ruvLLM exposes low-rank per-request adaptation through MicroLoRA and related configuration and feedback types. The project positions it as lightweight personalization without full model retraining. ([GitHub][4]) + +**Value to EdgeIntelligence:** It could personalize a shared base model to a user or temporary task while keeping the base weights unchanged. + +**Implementation direction:** + +* Treat MicroLoRA as a later experimental extension of the adapter system. +* Apply updates only from trusted feedback. +* Bound rank, memory, update magnitude, and lifetime. +* Separate temporary session adapters from persistent user adapters. +* Sign or integrity-protect persisted adapter state. + +**First acceptance criteria:** + +* Maximum memory and latency overhead are enforced. +* Bad feedback cannot permanently corrupt the base behavior. +* Adaptation can be rolled back atomically. +* Safety evaluation runs before promoting an adapted state. + +--- + +### 16. Quantized semantic-memory and embedding storage + +**Description:** ruvLLM includes a TurboQuant embedding store that keeps vectors in compressed form and performs similarity scoring without fully decompressing every stored vector. ([GitHub][4]) + +**Value to EdgeIntelligence:** This could support compact local RAG, semantic session recall, policy lookup, or tool-result caching without requiring a large full-precision vector database. + +**Implementation direction:** + +* Introduce a separate optional `el-semantic-memory` crate. +* Do not make vector storage part of the core inference binary. +* Encrypt persisted stores where platform facilities allow. +* Use namespace and retention controls. +* Benchmark recall degradation at each quantization level. + +**First acceptance criteria:** + +* Feature is fully optional. +* No embedding model is silently loaded. +* Memory savings and retrieval recall are reported together. +* Data can be partitioned and deleted by application/user namespace. + +--- + +### 17. Sparse attention for very long contexts + +**Description:** ruvLLM includes a sparse-attention kernel based on local windows, logarithmic strides, and landmarks, together with incremental decoding support. It is presented as a small, dependency-light component for embedded and WebAssembly targets. ([GitHub][4]) + +**Value to EdgeIntelligence:** Sparse attention could allow much longer contexts on constrained hardware, especially where dense attention is impractical. + +**Implementation direction:** + +* Integrate as a separate attention-kernel experiment. +* Limit initial support to compatible architectures. +* Compare against sliding-window attention. +* Require model-specific validation because changing the attention pattern may reduce quality if the model was not trained for it. + +**First acceptance criteria:** + +* Context-length gains are demonstrated on edge hardware. +* Quality is compared against dense attention. +* Kernel remains optional and has no effect on unsupported models. +* Incremental decoding preserves bounded memory use. + +--- + +## P3 — Valuable mainly for servers or specialized products + +### 18. PagedAttention + +**Description:** ruvLLM provides page-table-based KV management and describes a mistral-rs backend using PagedAttention and prefix caching for higher concurrency and GPU-memory utilization. ([GitHub][4]) + +**Value to EdgeIntelligence:** This is highly useful when one process serves many simultaneous sessions. It is less valuable for a single-user phone application, where its additional machinery may outweigh the gains. + +**Implementation direction:** + +* Target desktop, gateway, or local-network server deployments. +* Place it behind a separate `server` feature. +* Reuse the same `LlmProvider` API. +* Do not include it in default mobile artifacts. + +--- + +### 19. Continuous batching + +**Description:** ruvLLM includes dynamic request scheduling, token budgets, prefill/decode task separation, preemption, and batch-utilization statistics. ([GitHub][4]) + +**Value to EdgeIntelligence:** It improves aggregate throughput when multiple clients share one model. It offers limited benefit in a normal single-user mobile application. + +**Implementation direction:** + +* Add only to a server-oriented provider. +* Keep request isolation and cancellation explicit. +* Ensure batching never mixes grammar, safety, or adapter state. +* Measure tail latency as well as aggregate throughput. + +--- + +### 20. Hugging Face model acquisition and distribution + +**Description:** ruvLLM includes model download/upload and model-registry components for Hugging Face Hub integration. ([GitHub][4]) + +**Value to EdgeIntelligence:** It could simplify development, model provisioning, and cache management. + +**Why it ranks last:** EdgeIntelligence is air-gapped by default, uses explicit opt-in egress, and requires signed model provenance. Automatic downloads conflict with these defaults unless carefully isolated. ([GitHub][2]) + +**Implementation direction:** + +* Build it as a separate development/provisioning CLI. +* Never include network download code in the default runtime. +* Download to a staging path. +* Verify signature and compatibility before moving into the trusted model store. +* Keep production devices capable of operating entirely offline. + +--- + +# Recommended implementation roadmap + +## Phase 1 — Fix the existing hot path + +Implement these without introducing the full `RuvLLMEngine`: + +1. Persistent model instance +2. Stateful sessions +3. Batched prefill +4. Real token streaming +5. Memory-mapped model loading +6. Baseline performance instrumentation + +This phase directly addresses EdgeIntelligence’s measured bottlenecks and carries the lowest architectural risk. + +## Phase 2 — Reduce runtime memory + +1. Q8 two-tier KV cache +2. Q4/Q3 experimental KV cache +3. Sliding-window eviction +4. H2O/PyramidKV-style eviction +5. Hardware capability detection +6. Flash/memory-efficient attention + +This phase should establish device-specific memory budgets and quality thresholds. + +## Phase 3 — Increase generation speed + +1. Optimized CPU/SIMD kernels +2. Metal acceleration +3. Batched/chunked prefill tuning +4. Speculative decoding +5. Sparse attention experiments + +Every item should be independently switchable for ablation testing. + +## Phase 4 — Add specialization + +1. Signed LoRA adapter format +2. Adapter loading and lifecycle +3. Hot swapping +4. Adapter composition +5. Task-based adapter selection + +This introduces strong product value without making the base model adaptive. + +## Phase 5 — Add the ruvLLM intelligence layer + +1. Versioned execution-policy type +2. Static policy engine +3. Optional semantic policy store +4. Privacy-safe witness journal +5. Quantized semantic-memory store +6. Session indexing + +At this point, wrapping selected parts of `RuvLLMEngine` becomes more appropriate. + +## Phase 6 — Adaptive learning + +1. Explicit feedback API +2. Learned routing thresholds +3. Temporary MicroLoRA +4. Persistent user adapters +5. SONA background consolidation +6. Deep optimization only in controlled environments + +Adaptive behavior should remain opt-in and excluded from deterministic builds. + +# Recommended initial GitHub epics + +```text +EPIC-1: Stateful local inference +EPIC-2: Streaming and batched prefill +EPIC-3: Memory-mapped verified model loading +EPIC-4: Quantized and bounded KV cache +EPIC-5: Hardware-aware inference kernels +EPIC-6: Speculative decoding +EPIC-7: Signed LoRA adapter lifecycle +EPIC-8: Execution policy and routing +EPIC-9: Privacy-safe witness and evaluation framework +EPIC-10: Optional semantic memory +EPIC-11: Opt-in adaptive learning +EPIC-12: Multi-user serving backend +``` + +The first release milestone should stop after **EPIC-4**. Those features produce the clearest improvement to EdgeIntelligence’s latency and footprint while preserving its existing provenance, safety, privacy, and deterministic-runtime architecture. + +[1]: https://raw.githubusercontent.com/ruvnet/RuVector/main/crates/ruvllm/src/lib.rs "raw.githubusercontent.com" +[2]: https://github.com/Tovli/EdgeIntelligence "GitHub - Tovli/EdgeIntelligence · GitHub" +[3]: https://github.com/ruvnet/RuVector/tree/main/npm/packages/ruvllm "RuVector/npm/packages/ruvllm at main · ruvnet/RuVector · GitHub" +[4]: https://github.com/ruvnet/RuVector/tree/main/crates/ruvllm "RuVector/crates/ruvllm at main · ruvnet/RuVector · GitHub"