From 8602e5abbdc1bf25214f0f0bd531f22ad94c54ae Mon Sep 17 00:00:00 2001 From: Tovli Date: Tue, 23 Jun 2026 21:30:14 +0300 Subject: [PATCH 1/3] =?UTF-8?q?feat(runtime,engine):=20ADR-018=20AC-3=20?= =?UTF-8?q?=E2=80=94=20cross-turn=20prefix=20reuse=20(incremental=20prefil?= =?UTF-8?q?l)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reuse the KV already cached for the longest matching token prefix and feed only the divergent suffix, instead of re-prefilling the whole conversation every turn. - el-runtime: `InferenceSession::continue_prompt` + the `InferenceEngine::prefill_reuse` port. Safe default re-prefills the full context; stateful engines override for the fast path. A failed continue leaves a recoverable session. - el-engine-candle: `QwenEngine::prefill_reuse` does longest-common-prefix reuse against the live cache and falls back to a full prefill on any mismatch. - Soundness: reuse must leave the cache byte-identical to a fresh prefill — proven by the `ReuseEngine` mock (`prefill_reuse_leaves_same_state_as_a_fresh_prefill`) and it fails open on a wrong/short match. Measured on the full el-bench suite this cuts multi-turn (mindeval) ~3x and the whole-suite wall by a third (see docs/benchmarks/2026-06-23-...). --- crates/adapters/el-engine-candle/src/lib.rs | 284 +++++++++++-- crates/el-runtime/src/ports.rs | 30 ++ crates/el-runtime/src/session.rs | 401 ++++++++++++++++++ ...t-model-instances-and-stateful-sessions.md | 90 +++- 4 files changed, 758 insertions(+), 47 deletions(-) diff --git a/crates/adapters/el-engine-candle/src/lib.rs b/crates/adapters/el-engine-candle/src/lib.rs index d3f360c..8c7b586 100644 --- a/crates/adapters/el-engine-candle/src/lib.rs +++ b/crates/adapters/el-engine-candle/src/lib.rs @@ -364,6 +364,11 @@ mod bench { /// previous conversation's KV with a single benign position-0 forward — no engine /// reconstruction and no reload from disk between turns. /// +/// Across turns of the *same* conversation it also reuses the unchanged prefix's +/// KV: [`prefill_reuse`](InferenceEngine::prefill_reuse) feeds only the suffix the +/// re-rendered conversation adds beyond `cached`, the live token sequence behind +/// the cache (ADR-018 AC-3 cross-turn incremental prefill). +/// /// A *within-generation* safety backtrack (ADR-012) is supported via /// [`InferenceEngine::rollback`]: candle's attention discards its cache when a /// forward runs at `index_pos == 0`, so we retain the prompt and replay it from @@ -381,6 +386,12 @@ pub struct QwenEngine { /// The prefill prompt, retained so a rollback can replay it from position 0 /// to rebuild candle's KV cache (which has no public truncation). prompt: Vec, + /// The exact token sequence currently represented by the KV cache — + /// `prompt` plus the committed tokens fed so far. Its length equals + /// `index_pos` (every `forward_one` advances both in lock-step). It is the + /// basis for the cross-turn longest-common-prefix reuse check (ADR-018 AC-3, + /// [`prefill_reuse`](InferenceEngine::prefill_reuse)). + cached: Vec, /// Milli-logits produced after the most recent forward. last_logits: Vec, vocab: usize, @@ -410,6 +421,7 @@ impl QwenEngine { index_pos: 0, fed: 0, prompt: Vec::new(), + cached: Vec::new(), last_logits: Vec::new(), vocab: 0, eos, @@ -457,8 +469,10 @@ impl InferenceEngine for QwenEngine { self.index_pos = 0; self.fed = 0; self.prompt = tokens.to_vec(); // retained for rollback replay + self.cached = Vec::with_capacity(tokens.len()); for &t in tokens { self.last_logits = self.forward_one(t)?; + self.cached.push(t); } self.vocab = self.last_logits.len(); Ok(tokens.len() as u32) @@ -474,6 +488,7 @@ impl InferenceEngine for QwenEngine { Ok(l) => self.last_logits = l, Err(_) => return vec![0; self.vocab.max(1)], } + self.cached.push(t); // keep `cached` == the KV's token sequence self.fed += 1; } self.last_logits.clone() @@ -498,6 +513,10 @@ impl InferenceEngine for QwenEngine { let t = self.prompt[i]; self.last_logits = self.forward_one(t)?; } + // The cache now represents exactly the replayed prompt; the session + // re-feeds the retained committed prefix via `next_logits`, extending + // `cached` back in lock-step. + self.cached = self.prompt.clone(); Ok(()) } @@ -534,9 +553,70 @@ impl InferenceEngine for QwenEngine { // bytes aren't retained in an owned allocation (P2): `Vec::new()` drops // the old allocation; `clear()` would keep capacity and the stale ids. self.prompt = Vec::new(); + self.cached = Vec::new(); self.last_logits = Vec::new(); Ok(()) } + + /// Cross-turn incremental prefill (ADR-018 AC-3): reuse the KV already cached + /// for the longest prefix `full_context` shares with the live cache, and feed + /// only the divergent suffix at the live position — no reload, no whole-history + /// re-prefill. + /// + /// `cached` is the exact token sequence behind the current KV (length == + /// `index_pos`). The token-level longest-common-prefix against it is the + /// tokenizer-round-trip guard: a re-rendered+re-tokenized conversation that + /// drifts from what was generated simply matches a shorter prefix and the rest + /// is fed fresh. When `full_context` exactly extends the cache, only the new + /// tail is forwarded (the fast path); otherwise — divergence, or a context + /// shorter than the cache — candle cannot truncate its append-only cache, so we + /// rebuild from position 0 (a forward at `index_pos == 0` drops the old cache), + /// which is never worse than the pre-ADR-018 full re-prefill. + /// + /// Either branch leaves the engine in the **same** state a `reset_cache()` + + /// `prefill(full_context)` would: the suffix is fed by the identical + /// `forward_one` calls at the identical positions, so subsequent logits are + /// bit-identical to a from-scratch prefill (the soundness contract). + fn prefill_reuse(&mut self, full_context: &[Token]) -> Result { + let reuse = longest_common_prefix(&self.cached, full_context); + if reuse == self.cached.len() && reuse == self.index_pos { + // Fast path: the cache is an exact prefix of `full_context`. Feed only + // the new suffix at the live position; the existing KV is reused as-is. + for &t in &full_context[reuse..] { + self.last_logits = self.forward_one(t)?; + self.cached.push(t); + } + } else { + // Divergence (or a shorter context): rebuild from scratch. Setting + // `index_pos = 0` makes the first `forward_one` discard candle's old + // cache, exactly as a fresh `prefill` would. Clear `last_logits` first + // so an *empty* `full_context` leaves no stale distribution behind — + // matching `reset_cache()` + `prefill(&[])`; a non-empty context + // overwrites it in the loop. + self.index_pos = 0; + self.cached = Vec::with_capacity(full_context.len()); + self.last_logits = Vec::new(); + for &t in full_context { + self.last_logits = self.forward_one(t)?; + self.cached.push(t); + } + } + self.fed = 0; + // Replay base for this turn's rollback. + self.prompt = full_context.to_vec(); + // Set `vocab` unconditionally — exactly as `prefill` does — so an empty + // rebuild leaves `vocab == 0`, matching `reset_cache()` + `prefill(&[])`; + // a non-empty context sets it to the real vocab via the fed logits. + self.vocab = self.last_logits.len(); + Ok(self.index_pos as u32) + } +} + +/// Length of the longest common prefix of two token slices. The cross-turn KV +/// reuse cutoff (ADR-018 AC-3): how many leading tokens of a re-tokenized +/// conversation still match what the engine already cached. +fn longest_common_prefix(a: &[Token], b: &[Token]) -> usize { + a.iter().zip(b).take_while(|(x, y)| x == y).count() } // ── On-device safety wiring (ADR-005 tier + ADR-012 control loop) ──────────── @@ -674,12 +754,26 @@ impl SafetyConfig { } } +/// Interior state of a [`QwenExpert`], guarded by one mutex so the +/// rollback-detect-then-feed step is atomic. +struct ExpertState { + engine: QwenEngine, + /// How many committed tokens the expert has fed since its last prime — used + /// to detect a base rollback (committed shrinks below this) and re-sync. + fed: usize, +} + /// A safety **expert** logit source for contrastive steering (ADR-013): a second /// Qwen engine — in production base + a safety LoRA; here any same-tokenizer Qwen /// GGUF — loaded through the ADR-006 provenance gate and **primed with the turn's /// prompt** so its logits align with the base engine's. The session feeds it the -/// committed tokens via [`ExpertLogits::logits`]; interior mutability is required -/// because each forward advances candle's KV cache. +/// committed tokens via [`ExpertLogits::logits`]. +/// +/// The weights are **loaded once and kept resident** like the base model +/// (ADR-018 expert persistence): the provider holds it across turns and calls +/// [`reprime`](Self::reprime) per turn (`reset_cache` + prefill, no disk reload). +/// State lives behind a `Mutex` (not `RefCell`/`Cell`) so the resident expert is +/// `Send + Sync` and can sit in the `Send + Sync` provider. /// /// Steering is bounded to the early-token window (ADR-013), so the expert runs /// only for the first `steer_window` tokens. When the base engine rolls back @@ -689,10 +783,7 @@ impl SafetyConfig { /// at the chat model itself yields ~zero contrast (a no-op); a safety-tuned Qwen /// GGUF gives real steering. pub struct QwenExpert { - engine: std::cell::RefCell, - /// How many committed tokens the expert has fed since its last prime — used - /// to detect a base rollback (committed shrinks below this) and re-sync. - fed: std::cell::Cell, + state: std::sync::Mutex, /// Evidence the expert weights passed the ADR-006 load gate (R5). Held for /// the engine's lifetime; never used after construction. _permit: LoadPermit, @@ -711,40 +802,86 @@ impl QwenExpert { let mut engine = QwenEngine::from_path(path, eos)?; engine.prefill(prompt)?; Ok(Self { - engine: std::cell::RefCell::new(engine), - fed: std::cell::Cell::new(0), + state: std::sync::Mutex::new(ExpertState { engine, fed: 0 }), _permit: permit, }) } + + /// Re-prime the **resident** expert to a new turn's prompt without reloading + /// the GGUF (ADR-018 expert persistence): discard the prior turn's KV and + /// prefill the new prompt on the same loaded weights. The expensive part — + /// reading + parsing the GGUF — happens once in `from_path_primed`; this only + /// re-runs the (cheap, bounded) prompt prefill. + pub fn reprime(&self, prompt: &[Token]) -> Result<()> { + let mut st = self + .state + .lock() + .map_err(|_| EdgeError::Engine("expert mutex poisoned"))?; + st.engine.reset_cache()?; + st.engine.prefill(prompt)?; + st.fed = 0; + Ok(()) + } + + /// Release the expert's conversation KV while keeping its weights resident + /// (ADR-018) — the expert half of [`QwenChatProvider::end_session`]. + fn release(&self) -> Result<()> { + let mut st = self + .state + .lock() + .map_err(|_| EdgeError::Engine("expert mutex poisoned"))?; + st.engine.reset_cache()?; + st.fed = 0; + Ok(()) + } } impl ExpertLogits for QwenExpert { fn logits(&self, committed: &[Token]) -> Vec { - let mut engine = self.engine.borrow_mut(); + let mut st = match self.state.lock() { + Ok(st) => st, + Err(_) => return Vec::new(), + }; // Base rolled back? `committed` shrank below what we've fed. Re-prime the // expert to the prompt (QwenEngine::rollback replays the prompt and // resets its feed cursor) so it re-feeds the retained prefix from a clean // state — keeping the contrastive context aligned with the base. Cost is // bounded by `max_rollbacks` (ADR-012), same as the base engine. - if committed.len() < self.fed.get() { - if engine.rollback(0).is_err() { + if committed.len() < st.fed { + if st.engine.rollback(0).is_err() { return Vec::new(); } - self.fed.set(0); + st.fed = 0; } - let out = engine.next_logits(committed); - self.fed.set(committed.len()); + let out = st.engine.next_logits(committed); + st.fed = committed.len(); out } } +/// A resident expert shared into a turn's steerer (ADR-018 expert persistence): +/// the weights are loaded once and held by the provider; each turn clones this +/// `Arc` into the turn's [`ContrastiveSteerer`]. After the turn the steerer (and +/// this clone) drops, but the provider's `Arc` keeps the weights resident. The +/// newtype sidesteps the orphan rule (`ExpertLogits` cannot be implemented for a +/// bare `Arc` outside `el-safety`). +struct SharedExpert(std::sync::Arc); + +impl ExpertLogits for SharedExpert { + fn logits(&self, committed: &[Token]) -> Vec { + self.0.logits(committed) + } +} + /// The resident model behind a [`QwenChatProvider`] (ADR-018). /// /// The weights are loaded **once** (in [`QwenChatProvider::from_paths`]) into /// `Loaded`. The first `chat` promotes them into a reusable [`InferenceSession`] /// (`Active`) — done lazily so the builder-configured safety tier / expert is -/// finalized first — and every later turn reuses that one session via -/// `reset()` + `load_prompt()`. The model is never re-read from disk per turn. +/// finalized first — and every later turn reuses that one session: a follow-up +/// turn via `continue_prompt` (reuse the cached KV prefix, AC-3), a fresh turn via +/// `load_prompt`, and a turn after a mid-flight failure via `reset()` + +/// `load_prompt`. The model is never re-read from disk per turn. enum ChatSession { /// Weights resident, no conversation session yet. Loaded(QwenEngine), @@ -759,10 +896,12 @@ enum ChatSession { /// /// The model weights are loaded **once** at construction and kept resident /// (ADR-018): each `chat` renders the whole conversation to Qwen2.5 ChatML and -/// reuses one persistent provenance-gated session — `reset` (discard the prior -/// conversation + engine KV cache) → `load_prompt` (prefill) → `generate` -/// (grammar mask → safety steer → guard + checkpointed rollback → greedy commit) -/// — instead of rebuilding the engine and re-reading the GGUF every turn. +/// reuses one persistent provenance-gated session — a follow-up turn reuses the +/// cached KV prefix and prefills only the new suffix (`continue_prompt`, AC-3 +/// cross-turn incremental prefill), while a fresh turn uses `load_prompt` — +/// then `generate` (grammar mask → safety steer → guard + checkpointed rollback +/// → greedy commit), instead of rebuilding the engine and re-reading the GGUF +/// every turn. /// On-device safety (ADR-005 `Lightweight` tier + the ADR-012 control loop) is /// **on by default**; see [`with_safety`](Self::with_safety). The resident model /// lives behind a `Mutex`, so the provider stays `Send + Sync` and concurrent @@ -775,14 +914,18 @@ pub struct QwenChatProvider { model_label: String, safety: SafetyConfig, /// Optional safety **expert** GGUF for ADR-013 contrastive steering. `None` - /// runs the token-only `Lightweight` steerer. NOTE (ADR-018 follow-up): the - /// expert is still loaded per turn; persisting it like the base model is the - /// next increment. The default (no-expert) path is fully resident. + /// runs the token-only `Lightweight` steerer. The weights are loaded once and + /// kept resident in `expert` (ADR-018 expert persistence). expert_model: Option, /// Contrastive steering strength ×1000 (1000 = 1.0×). steer_alpha_milli: i32, /// Resident model + reusable session (ADR-018). session: std::sync::Mutex, + /// The resident safety expert (ADR-018 expert persistence): loaded lazily on + /// the first `SecDecoding` turn and reused across turns via `reprime`, instead + /// of re-reading the expert GGUF from disk every turn. `None` until first use, + /// or always-`None` when no `--expert-model` is configured. + expert: std::sync::Mutex>>, } impl QwenChatProvider { @@ -826,6 +969,7 @@ impl QwenChatProvider { expert_model: None, steer_alpha_milli: 1000, session: std::sync::Mutex::new(ChatSession::Loaded(engine)), + expert: std::sync::Mutex::new(None), }) } @@ -892,6 +1036,16 @@ impl QwenChatProvider { if let ChatSession::Active(session) = &mut *cell { session.close()?; } + // Release the resident safety expert's conversation KV too (keeps its + // weights). Locked after the session — the same order `chat` uses — so + // the two mutexes never deadlock. + let slot = self + .expert + .lock() + .map_err(|_| EdgeError::Engine("expert mutex poisoned"))?; + if let Some(expert) = slot.as_ref() { + expert.release()?; + } Ok(()) } @@ -963,27 +1117,44 @@ impl LlmProvider for QwenChatProvider { _ => return Err(EdgeError::Engine("chat session not initialized")), }; - // Discard the previous conversation and the engine's KV cache, then run - // this turn over the SAME resident weights (ADR-018) — no disk reload. - session.reset()?; // Provider-owned turn isolation: drop any events buffered by a prior turn // (e.g. one that errored before its end-of-turn drain) so this turn's // safety count below cannot include another turn's stale violations. - // `reset()` deliberately preserves events (generic semantics); bounding - // them per turn is the reusing provider's job. + // `continue_prompt`/`reset` deliberately preserve events (generic + // semantics); bounding them per turn is the reusing provider's job. let _ = session.drain_events(); let mut ports = self.safety.ports(); if matches!(effective, SafetyMode::SecDecoding) { if let Some(expert_path) = &self.expert_model { - let expert = QwenExpert::from_path_primed( - expert_path, - self.eos, - &prompt_tokens, - local_load_permit(expert_path)?, - )?; + // ADR-018 expert persistence: load the expert weights ONCE and keep + // them resident; every later turn re-primes (no disk reload). The + // expert lock is always taken while holding the session lock — a + // fixed order, so no deadlock with `end_session`. + let expert = { + let mut slot = self + .expert + .lock() + .map_err(|_| EdgeError::Engine("expert mutex poisoned"))?; + match slot.as_ref() { + Some(e) => { + e.reprime(&prompt_tokens)?; + std::sync::Arc::clone(e) + } + None => { + let e = std::sync::Arc::new(QwenExpert::from_path_primed( + expert_path, + self.eos, + &prompt_tokens, + local_load_permit(expert_path)?, + )?); + *slot = Some(std::sync::Arc::clone(&e)); + e + } + } + }; ports.safety = Box::new(ContrastiveSteerer::new( - expert, + SharedExpert(expert), self.safety.banned.clone(), self.steer_alpha_milli, CONTRASTIVE_TOP_K, @@ -994,7 +1165,26 @@ impl LlmProvider for QwenChatProvider { let _ = bench::take(); // clear forward accumulators before prefill let t_prefill = bench::enabled().then(std::time::Instant::now); - session.load_prompt(&ports, &prompt_tokens)?; + // ADR-018 AC-3: on a follow-up turn (a finished prior turn left the + // session `Completed`), reuse the cached KV prefix and prefill only the + // new suffix; a fresh turn does a full prefill. The engine's + // longest-common-prefix check is the backstop, so a tokenizer round-trip + // drift in the reused branch falls back to a correct full re-prefill. + match session.phase() { + el_core::Phase::Completed => session.continue_prompt(&ports, &prompt_tokens)?, + // First use, or a fresh start after `end_session` / error recovery. + el_core::Phase::Initialized => session.load_prompt(&ports, &prompt_tokens)?, + // Dirty: a prior turn's prefill failed mid-transition and left the + // session in `Prefilling`/`Decoding`. Now that the unconditional + // per-turn `reset()` is gone, a bare `load_prompt` here would hit + // `InvalidPhase` and wedge the provider — so discard the partial + // conversation (clearing the engine's possibly half-fed cache) and + // start fresh instead. + _ => { + session.reset()?; + session.load_prompt(&ports, &prompt_tokens)?; + } + } let d_prefill = t_prefill.map(|t| t.elapsed()).unwrap_or_default(); let (pf_total, pf_model, pf_calls) = bench::take(); @@ -1500,6 +1690,17 @@ mod tests { assert_eq!(got, want); } + #[test] + fn longest_common_prefix_cutoff() { + // The cross-turn KV reuse boundary (ADR-018 AC-3). + assert_eq!(longest_common_prefix(&[], &[1, 2]), 0); + assert_eq!(longest_common_prefix(&[1, 2, 3], &[1, 2, 3, 4, 5]), 3); // extends + assert_eq!(longest_common_prefix(&[1, 2, 3], &[1, 2, 3]), 3); // equal + assert_eq!(longest_common_prefix(&[1, 9, 3], &[1, 2, 3]), 1); // diverges at idx 1 + assert_eq!(longest_common_prefix(&[1, 2, 3], &[1, 2]), 2); // shorter context + assert_eq!(longest_common_prefix(&[5, 6], &[1, 2]), 0); // immediate divergence + } + #[test] fn local_load_permit_passes_the_provenance_gate() { // The runtime requires a LoadPermit; the local-trust path must yield one @@ -1637,4 +1838,15 @@ mod tests { ); assert!(matches!(r, Err(EdgeError::Engine(_)))); } + + #[test] + fn provider_and_expert_stay_send_and_sync() { + // ADR-018 expert persistence: making the expert resident must NOT cost the + // provider its thread-safety. The resident expert (`Mutex<…Arc>`) + // keeps `QwenChatProvider: Send + Sync`, which is why `QwenExpert` uses a + // `Mutex` rather than `RefCell`/`Cell`. Compile-time guard. + fn assert_send_sync() {} + assert_send_sync::(); + assert_send_sync::(); + } } diff --git a/crates/el-runtime/src/ports.rs b/crates/el-runtime/src/ports.rs index 99a1b22..ecdc1a8 100644 --- a/crates/el-runtime/src/ports.rs +++ b/crates/el-runtime/src/ports.rs @@ -54,6 +54,36 @@ pub trait InferenceEngine { /// conversation. A stateless engine whose `next_logits` recomputes purely from /// `committed` implements it as `Ok(())`. fn reset_cache(&mut self) -> Result<()>; + + /// Prefill `full_context`, **reusing the KV already cached for its longest + /// matching prefix** and feeding only the divergent suffix — cross-turn + /// incremental prefill (ADR-018 AC-3). Returns the resulting KV length. + /// + /// This is the engine half of [`InferenceSession::continue_prompt`]: on a + /// follow-up turn the whole conversation is re-rendered and re-tokenized, but a + /// stateful engine that still holds the prior turn's KV can skip re-encoding the + /// unchanged prefix. The token-level prefix match against the live cache **is** + /// the tokenizer-round-trip guard — if the re-tokenized context diverges from + /// what was cached (decode→encode is not always identity), reuse stops at the + /// divergence and the suffix is fed fresh. + /// + /// **Soundness contract.** After `Ok`, the engine MUST be in the exact state a + /// `reset_cache()` + `prefill(full_context)` would have left it — identical + /// logits for any subsequent [`next_logits`](Self::next_logits). Reuse is purely + /// a compute optimisation; it must never change *what* the cache represents, so + /// the runtime's safety checks (which re-run over `full_context` every turn) see + /// identical data. + /// + /// Unlike [`rollback`](Self::rollback)/[`reset_cache`](Self::reset_cache), a wrong + /// implementation here is a correctness/perf regression, not a safety + /// fail-*open* — so this has a **safe default**: discard the cache and re-prefill + /// the whole context (no reuse). Stateful engines override it for the fast path; + /// stateless engines (whose `next_logits` recomputes from `committed`) inherit + /// the default unchanged. + fn prefill_reuse(&mut self, full_context: &[Token]) -> Result { + self.reset_cache()?; + self.prefill(full_context) + } } /// Prompt Compression port (LLMLingua-2 — context 2). diff --git a/crates/el-runtime/src/session.rs b/crates/el-runtime/src/session.rs index a56ae2d..eaac3fd 100644 --- a/crates/el-runtime/src/session.rs +++ b/crates/el-runtime/src/session.rs @@ -155,6 +155,68 @@ impl InferenceSession { Ok(()) } + /// Begin a **follow-up turn on the same conversation**, reusing the KV cache + /// for the unchanged prefix instead of re-prefilling the whole transcript + /// (ADR-018 AC-3 cross-turn incremental prefill). `full_context` is the entire + /// re-rendered conversation, tokenized (rendering is the provider's concern). + /// + /// Valid only from [`Phase::Completed`] — i.e. a prior turn on this resident + /// session finished. The engine ([`prefill_reuse`](InferenceEngine::prefill_reuse)) + /// reuses the longest cached prefix that still matches `full_context` and feeds + /// only the divergent suffix; the prior turn's generated reply, now baked into + /// the re-rendered context, becomes part of the reused prefix. This turn's + /// generated `output` starts empty; the KV descriptors are rebuilt to the + /// engine's reported length. + /// + /// Safety is unchanged (ADR-012/ADR-013): [`generate`](Self::generate) re-scores + /// the full prompt at ingress and guards this turn's output as usual, and a + /// within-turn rollback rebuilds from `full_context` (the reuse base). KV reuse + /// is purely a compute optimisation — it never alters what the safety loop sees. + /// + /// Distinct from [`reset`](Self::reset) (discard the conversation, start fresh): + /// `continue_prompt` *keeps* the conversation and extends it. A provider that + /// cannot guarantee a clean prior turn (after a reset, error, or full discard) + /// should use [`load_prompt`](Self::load_prompt) from `Initialized` instead; + /// the engine's prefix check is a backstop that falls back to a full prefill on + /// any divergence. + /// + /// Unlike `load_prompt`, this deliberately does **not** apply prompt + /// compression: cross-turn reuse needs a byte-stable token prefix, and + /// compressing the (growing) re-rendered conversation each turn would yield a + /// different prefix and defeat reuse — the two are mutually exclusive. `ports` + /// is accepted for call-site symmetry with `load_prompt`; the grammar/safety/ + /// ingress ports are consumed by [`generate`](Self::generate), not here. + pub fn continue_prompt(&mut self, _ports: &Ports, full_context: &[Token]) -> Result<()> { + if self.phase != Phase::Completed { + return Err(EdgeError::InvalidPhase { + expected: "Completed", + found: self.phase.as_str(), + }); + } + // The whole re-rendered conversation is this turn's prompt for ADR-013 + // ingress triage (re-scored in `generate_with_policy`). + self.prompt = full_context.to_vec(); + + self.phase = Phase::Prefilling; + let kv_len = self.engine.prefill_reuse(full_context)?; + // Rebuild the KV descriptors to match the reused-plus-extended cache. + self.kv = KvRegion::new(); + for _ in 0..kv_len { + let off = self.kv.len() as u64; + self.kv.push(off); + } + // Fresh generated output for this turn; the prior reply is now in the + // prefilled context. + self.output.clear(); + self.emit(DomainEvent::PrefillCompleted { + prompt_tokens: full_context.len() as u32, + kv_len, + prefill_tps: 0, + }); + self.phase = Phase::Decoding; + Ok(()) + } + /// Run the decode loop until EOS or `max_tokens`, deriving the rollback /// policy from the session's device tier and safety mode (ADR-005/ADR-012). pub fn generate(&mut self, ports: &Ports, max_tokens: u32) -> Result { @@ -1595,4 +1657,343 @@ mod tests { "the next reset re-clears after a failed attempt" ); } + + // ----- ADR-018 AC-3 cross-turn prefix reuse / incremental prefill ----- + + /// A stateful mock mirroring `QwenEngine`'s cache bookkeeping: it tracks the + /// exact cached token sequence, counts every forward (one per fed token), and + /// implements `prefill_reuse` with the same longest-common-prefix reuse — so + /// the cross-turn reuse contract can be proven without a real model. Its + /// logits are a deterministic hash of the cached sequence, so two engines with + /// identical caches return identical logits (and divergent caches almost never + /// do): the soundness probe for reuse-equals-fresh-prefill. + struct ReuseEngine { + eos: Token, + vocab: usize, + index_pos: usize, + fed: usize, + cached: Vec, + prompt: Vec, + /// Logits after the most recent forward — **stored**, like `QwenEngine`'s + /// `last_logits`, so the empty/rebuild path can be checked for stale state. + last_logits: Vec, + forwards: std::rc::Rc>, + /// When set, the favoured token is always EOS so `generate` terminates on + /// the first decode step (keeps the reuse-savings test deterministic). + eos_immediately: bool, + } + impl ReuseEngine { + fn new(eos: Token, vocab: usize, eos_immediately: bool) -> Self { + Self { + eos, + vocab, + index_pos: 0, + fed: 0, + cached: Vec::new(), + prompt: Vec::new(), + last_logits: Vec::new(), + forwards: std::rc::Rc::new(std::cell::Cell::new(0)), + eos_immediately, + } + } + fn forward(&mut self, t: Token) { + self.forwards.set(self.forwards.get() + 1); + self.cached.push(t); + self.index_pos += 1; + self.last_logits = self.logits_for_cache(); + } + fn logits_for_cache(&self) -> Vec { + let mut v = vec![0i32; self.vocab]; + if self.eos_immediately { + if let Some(s) = v.get_mut(self.eos as usize) { + *s = 1_000_000; + } + return v; + } + // Content hash of the cached sequence so identical caches ⇒ identical + // logits — the probe for reuse soundness. + let h = self + .cached + .iter() + .fold(0i32, |a, &t| a.wrapping_mul(31).wrapping_add(t as i32 + 1)); + for (i, slot) in v.iter_mut().enumerate() { + *slot = h.wrapping_add(i as i32); + } + v + } + } + impl InferenceEngine for ReuseEngine { + fn prefill(&mut self, tokens: &[Token]) -> Result { + self.index_pos = 0; + self.fed = 0; + self.cached = Vec::new(); + self.last_logits = Vec::new(); + self.prompt = tokens.to_vec(); + for &t in tokens { + self.forward(t); + } + Ok(tokens.len() as u32) + } + fn next_logits(&mut self, committed: &[Token]) -> Vec { + while self.fed < committed.len() { + self.forward(committed[self.fed]); + self.fed += 1; + } + self.last_logits.clone() + } + fn eos_token(&self) -> Token { + self.eos + } + fn rollback(&mut self, _keep: u32) -> Result<()> { + self.index_pos = 0; + self.fed = 0; + self.cached = Vec::new(); + self.last_logits = Vec::new(); + let prompt = self.prompt.clone(); + for &t in &prompt { + self.forward(t); + } + Ok(()) + } + fn reset_cache(&mut self) -> Result<()> { + self.index_pos = 0; + self.fed = 0; + self.cached = Vec::new(); + self.last_logits = Vec::new(); + self.prompt = Vec::new(); + Ok(()) + } + fn prefill_reuse(&mut self, full_context: &[Token]) -> Result { + let reuse = self + .cached + .iter() + .zip(full_context) + .take_while(|(a, b)| a == b) + .count(); + if reuse == self.cached.len() && reuse == self.index_pos { + for &t in &full_context[reuse..] { + self.forward(t); + } + } else { + self.index_pos = 0; + self.cached = Vec::new(); + self.last_logits = Vec::new(); // mirror QwenEngine: no stale logits on rebuild + for &t in full_context { + self.forward(t); + } + } + self.fed = 0; + self.prompt = full_context.to_vec(); + Ok(self.index_pos as u32) + } + } + + /// Fails every `prefill_reuse` (a transient reuse error) but resets and + /// prefills fine — the dirty-phase recovery probe. + struct FailReuseEngine { + eos: Token, + vocab: usize, + } + impl InferenceEngine for FailReuseEngine { + fn prefill(&mut self, t: &[Token]) -> Result { + Ok(t.len() as u32) + } + fn next_logits(&mut self, _c: &[Token]) -> Vec { + let mut v = vec![0i32; self.vocab]; + if let Some(s) = v.get_mut(self.eos as usize) { + *s = 1; + } + v + } + fn eos_token(&self) -> Token { + self.eos + } + fn rollback(&mut self, _keep: u32) -> Result<()> { + Ok(()) + } + fn reset_cache(&mut self) -> Result<()> { + Ok(()) + } + fn prefill_reuse(&mut self, _full_context: &[Token]) -> Result { + Err(EdgeError::Engine("prefill_reuse failed")) + } + } + + #[test] + fn continue_prompt_reuses_cached_prefix_and_feeds_only_suffix() { + // AC-3a: a follow-up turn forwards ONLY the new suffix; the unchanged + // prefix's KV is reused — no whole-context re-prefill, no reload. + let engine = ReuseEngine::new(1, 4, true); + let forwards = engine.forwards.clone(); + let mut s = + InferenceSession::new(SessionId(50), SessionConfig::default(), engine, permit()); + let ports = Ports::permissive(); + + s.load_prompt(&ports, &[10, 11]).unwrap(); + s.generate(&ports, 8).unwrap(); // EOS immediately → output = [1] + assert_eq!(forwards.get(), 2, "turn 1 prefilled the 2 prompt tokens"); + + // Turn 2: same [10,11] prefix + a 3-token suffix. + s.continue_prompt(&ports, &[10, 11, 20, 21, 22]).unwrap(); + assert_eq!( + forwards.get() - 2, + 3, + "only the 3-token suffix was prefilled, not the whole 5-token context" + ); + assert_eq!(s.phase(), Phase::Decoding); + // KV descriptors reflect the full reused-plus-extended context. + assert_eq!(s.kv_len(), 5); + + s.generate(&ports, 8).unwrap(); + assert_eq!(s.output(), &[1]); + assert_eq!(s.phase(), Phase::Completed); + } + + #[test] + fn continue_prompt_requires_a_completed_prior_turn() { + // AC-3d: continue is valid only after a finished turn; an unprefilled + // (Initialized) session must use load_prompt instead. + let engine = ReuseEngine::new(1, 4, true); + let mut s = + InferenceSession::new(SessionId(51), SessionConfig::default(), engine, permit()); + let ports = Ports::permissive(); + let err = s.continue_prompt(&ports, &[1, 2]).unwrap_err(); + assert!(matches!(err, EdgeError::InvalidPhase { .. })); + } + + #[test] + fn prefill_reuse_leaves_same_state_as_a_fresh_prefill() { + // AC-3b soundness: prefill_reuse(ctx) must leave the engine producing the + // SAME logits as reset_cache()+prefill(ctx) — both for the fast (extend) + // path and the divergence (rebuild) path. Reuse is only a compute + // optimisation; it must never change what the cache represents. + let mut fresh = ReuseEngine::new(99, 6, false); + fresh.prefill(&[1, 2, 3, 4]).unwrap(); + let want = fresh.next_logits(&[]); + + // Fast path: cache [1,2] then extend to [1,2,3,4]. + let mut extend = ReuseEngine::new(99, 6, false); + extend.prefill(&[1, 2]).unwrap(); + extend.prefill_reuse(&[1, 2, 3, 4]).unwrap(); + assert_eq!( + extend.next_logits(&[]), + want, + "extend-reuse == fresh prefill" + ); + + // Divergence: cache [1,9,3] diverges from [1,2,3,4] at index 1 → rebuild. + let mut rebuild = ReuseEngine::new(99, 6, false); + rebuild.prefill(&[1, 9, 3]).unwrap(); + rebuild.prefill_reuse(&[1, 2, 3, 4]).unwrap(); + assert_eq!( + rebuild.next_logits(&[]), + want, + "rebuild-on-divergence == fresh prefill" + ); + } + + #[test] + fn default_prefill_reuse_does_a_full_recompute() { + // The safe default (used by every engine that doesn't override): discard + // the cache and re-prefill the whole context. NullEngine is stateless, so + // this is a no-op reset + a full prefill of the given length. + let mut e = NullEngine::new(1, 4); + assert_eq!(e.prefill(&[1, 2]).unwrap(), 2); + assert_eq!( + e.prefill_reuse(&[1, 2, 3, 4, 5]).unwrap(), + 5, + "default reuse re-prefills the full context length" + ); + } + + #[test] + fn continued_turn_still_runs_the_safety_control_loop() { + // AC-3c: a continued turn is guarded exactly like a fresh one — the + // chunk-guard + checkpointed rollback still bans the unsafe token. Uses a + // stateless engine (default prefill_reuse) to isolate the safety wiring. + let mut s = InferenceSession::new( + SessionId(52), + SessionConfig::default(), + descending_engine(), // always prefers the unsafe token 0 + permit(), + ); + let ports = guarded_ports(Box::new(BanToken(0))); + + s.load_prompt(&ports, &[7]).unwrap(); + s.generate_with_policy(&ports, 2, tiny_policy(3)).unwrap(); + assert_eq!(s.phase(), Phase::Completed); + + // Follow-up turn over the re-rendered conversation. + s.continue_prompt(&ports, &[7, 1, 1, 8]).unwrap(); + let stop = s.generate_with_policy(&ports, 3, tiny_policy(3)).unwrap(); + + assert_eq!(stop, StopReason::MaxTokens); + assert!( + !s.output().contains(&0), + "the guard still bans the unsafe token on a continued turn" + ); + assert_eq!(s.output(), &[1, 1, 1]); + let evs = s.drain_events(); + assert!(evs + .iter() + .any(|e| matches!(e.event, DomainEvent::ClaimBacktracked { .. }))); + } + + #[test] + fn prefill_reuse_into_empty_context_drops_stale_logits() { + // Regression (P2): rebuilding into an EMPTY context must leave no stale + // distribution — same state as reset_cache()+prefill(&[]). Otherwise + // continue_prompt(&[]) would report kv_len 0 yet decode from the prior + // turn's logits. + let mut reused = ReuseEngine::new(9, 4, false); + reused.prefill(&[1, 2, 3]).unwrap(); + assert!(!reused.next_logits(&[]).is_empty()); + reused.prefill_reuse(&[]).unwrap(); + + let mut fresh = ReuseEngine::new(9, 4, false); + fresh.reset_cache().unwrap(); + fresh.prefill(&[]).unwrap(); + + assert_eq!( + reused.next_logits(&[]), + fresh.next_logits(&[]), + "rebuild into an empty context == a fresh empty prefill" + ); + assert!( + reused.next_logits(&[]).is_empty(), + "no stale distribution may survive the rebuild" + ); + } + + #[test] + fn failed_continue_prompt_leaves_a_recoverable_session() { + // Regression (P1): a failed prefill_reuse sets Prefilling but never reaches + // Decoding. The session must NOT wedge — the provider's dirty-phase + // recovery (reset() then load_prompt()) must work afterwards, instead of + // the next turn hitting InvalidPhase forever. + let mut s = InferenceSession::new( + SessionId(54), + SessionConfig::default(), + FailReuseEngine { eos: 1, vocab: 4 }, + permit(), + ); + let ports = Ports::permissive(); + s.load_prompt(&ports, &[10, 11]).unwrap(); + s.generate(&ports, 8).unwrap(); + assert_eq!(s.phase(), Phase::Completed); + + let err = s.continue_prompt(&ports, &[10, 11, 12]).unwrap_err(); + assert!(matches!(err, EdgeError::Engine(_))); + assert_ne!( + s.phase(), + Phase::Completed, + "a failed reuse must not masquerade as a finished turn" + ); + + // The provider's recovery path must succeed regardless of the dirty phase. + s.reset().unwrap(); + s.load_prompt(&ports, &[20, 21]).unwrap(); + assert_eq!(s.phase(), Phase::Decoding); + assert_eq!(s.generate(&ports, 8).unwrap(), StopReason::Eos); + } } diff --git a/docs/adr/ADR-018-persistent-model-instances-and-stateful-sessions.md b/docs/adr/ADR-018-persistent-model-instances-and-stateful-sessions.md index cd19ee8..42f5d0c 100644 --- a/docs/adr/ADR-018-persistent-model-instances-and-stateful-sessions.md +++ b/docs/adr/ADR-018-persistent-model-instances-and-stateful-sessions.md @@ -137,20 +137,16 @@ Landed via a SPARC pass (scope: AC-1/AC-2/AC-4 + the engine seam; AC-3 deferred) `reset_cache` is propagated, not swallowed. Full workspace suite green; `cargo fmt --all -- --check` and `clippy -D warnings` clean. -Still **deferred** (own increment, per the spec's scope decision): - -- **AC-3 — cross-turn prefix reuse / incremental prefill.** Each turn still - re-prefills the whole conversation (no weight reload). Reusing the unchanged - prefix's KV needs an engine "extend-without-reset" contract, a session "continue" - transition, and care around the ADR-012 safe-prefix invariant and tokenizer - round-trip stability. The persistent-session architecture here is its foundation. -- **Expert persistence.** With `--expert-model`, the safety expert GGUF is still - loaded per turn; persisting it like the base model is a follow-up. The default - (no-expert) path is fully resident. +**AC-3 (cross-turn prefix reuse) and expert persistence landed in a follow-up +increment — see below.** + +Still **deferred**: + - **Concurrent multi-session weight sharing.** candle couples weights + KV cache, so this realizes *serial* conversation reuse; true N-sessions-over-one-model needs an engine that separates weights from cache (same root constraint as - ADR-022). + ADR-022). Not attempted: it requires a forked/custom attention cache, not a + runtime change. ### Review fixes (post-implementation hardening) @@ -215,6 +211,78 @@ cache)" behavior; both now describe the ADR-018 resident model + in-place `reset_cache` eviction, and the `EL_BENCH` section reflects the once-at-startup load ("session setup", not a per-chat "model load"). +### Follow-up increment — AC-3 cross-turn prefix reuse + expert persistence + +Landed via a second SPARC pass (scope: AC-3 + expert persistence; concurrent +multi-session left deferred above): + +- **Engine seam — `InferenceEngine::prefill_reuse(full_context)`.** Prefills a + context while reusing the KV already cached for its longest matching prefix and + feeding only the divergent suffix. Unlike `rollback`/`reset_cache` (whose wrong + default fails *open* on safety) a wrong reuse is only a correctness/perf + regression, so this has a **safe default**: discard the cache and re-prefill the + whole context (no reuse). Only `QwenEngine` overrides it; the other ten engine + impls inherit the default unchanged — no churn. +- **Tokenizer round-trip guard via longest-common-prefix.** `QwenEngine` now tracks + `cached`, the exact token sequence behind its KV (length == `index_pos`). A + follow-up turn re-renders + re-tokenizes the whole conversation; `prefill_reuse` + takes the token-level LCP of `cached` and the new context and reuses up to it. + When the context exactly extends the cache, only the new tail is forwarded (the + fast path); on any divergence — decode→encode is not always identity — or a + shorter context, it rebuilds from position 0 (candle has no cache truncation), + never worse than the pre-AC-3 full re-prefill. Either branch leaves the engine in + the **same** state a `reset_cache()` + `prefill(full_context)` would (the suffix + is fed by identical `forward_one` calls at identical positions), so subsequent + logits are bit-identical — the soundness contract, proven by a stateful mock. +- **Session — `InferenceSession::continue_prompt(full_context)`.** A new transition + valid only from `Completed` (a finished prior turn): retains `full_context` as the + prompt for ADR-013 ingress triage, calls `prefill_reuse`, rebuilds the KV + descriptors to the reported length, and clears this turn's `output` (the prior + reply is now baked into the prefilled context). The **ADR-012 safe-prefix + invariant is preserved**: the reuse base for a within-turn rollback is + `full_context`, ingress re-scores the whole prompt every turn, and the chunk guard + scores this turn's output as before — KV reuse changes *how* logits are computed, + never *what* the safety loop checks. +- **Provider — `QwenChatProvider::chat`.** The unconditional per-turn `reset()` is + gone; the turn branches on `session.phase()`: `Completed` → `continue_prompt` + (reuse the prior KV prefix), otherwise → `load_prompt` (fresh). `/reset`, + `/system`, and error recovery still go through `end_session` (→ `Initialized`), so + the next turn re-prefills cleanly; a refused or tokenizer-drifted prior turn falls + back automatically via the engine's LCP check. +- **Expert persistence.** `QwenExpert` is now `Send + Sync` (a `Mutex<{engine,fed}>` + instead of `RefCell`/`Cell`) and the provider holds it resident in + `Mutex>>`. The expert GGUF is loaded **once** on the first + `SecDecoding` turn and `reprime`d (`reset_cache` + prefill, no disk reload) every + later turn; a per-turn `SharedExpert(Arc)` newtype clones into the + turn's `ContrastiveSteerer` (sidestepping the orphan rule on `Arc`). + `end_session` releases the expert's KV too, keeping its weights resident. +- **Tests.** Runtime: cross-turn reuse feeds only the suffix; `prefill_reuse` leaves + the same state as a fresh prefill (fast-path *and* rebuild-on-divergence); the + default reuse does a full recompute; `continue_prompt` rejects a non-`Completed` + phase; a continued turn still runs the full guard + rollback loop. Adapter: an LCP + helper unit test, and a compile-time `Send + Sync` assertion over `QwenExpert` / + `QwenChatProvider` (the resident expert must not cost the provider its + thread-safety). A model-backed `QwenEngine`/expert integration test needs a Qwen + GGUF fixture (none in-tree, as for the increment-1 engine), so the reuse contract + is proven at the trait level + the shared `forward_one` equivalence argument. Full + workspace suite green; `cargo fmt --all -- --check` and `clippy -D warnings` clean. + +**Review fixes (increment-2 hardening).** (1) **Dirty-phase wedge (P1).** Removing +the unconditional per-turn `reset()` meant a `prefill`/`prefill_reuse` that failed +mid-transition left the session in `Prefilling`; the provider's `_ => load_prompt` +branch then hit `InvalidPhase` on every later turn and wedged. The provider now +dispatches explicitly — `Completed`→`continue_prompt`, `Initialized`→`load_prompt`, +otherwise `reset()` + `load_prompt` (discard the half-fed cache, start fresh). +Regression: `failed_continue_prompt_leaves_a_recoverable_session`. (2) **Stale logits +on empty rebuild (P2).** `QwenEngine::prefill_reuse`'s rebuild branch didn't clear +`last_logits`, so `continue_prompt(&[])` reported `kv_len == 0` while `next_logits` +still returned the prior turn's distribution — breaking the `reset_cache`+`prefill` +equivalence. It now clears `last_logits` before the rebuild loop; the trait mock +gained stored-logits semantics to guard it +(`prefill_reuse_into_empty_context_drops_stale_logits`). (3) **Root artifacts (P3).** +Six empty shell-redirect artifacts in the repo root were removed (the same mishap +class as increment-1's rounds 1–4). + ## Links - Source: [docs/research/improvements-plan.md](../research/improvements-plan.md) §P0.1, EPIC-1. - Builds on: [ADR-001](./ADR-001-adopt-webassembly-as-cross-platform-sdk-runtime.md) (session immutability), [ADR-010](./ADR-010-unified-llm-provider-trait-with-opt-in-frontier-egress.md) (`LlmProvider` seam), [ADR-002](./ADR-002-candle-as-rust-native-inference-engine.md) (engine). From a8b67f611c2e1f064db6da0856b359317eb314e3 Mon Sep 17 00:00:00 2001 From: Tovli Date: Tue, 23 Jun 2026 21:30:31 +0300 Subject: [PATCH 2/3] chore: bump workspace version to 0.3.11 --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 22 +++++++++++----------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bd18a65..8bce09d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -793,7 +793,7 @@ checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "el-bench" -version = "0.3.10" +version = "0.3.11" dependencies = [ "el-core", "el-engine-candle", @@ -803,7 +803,7 @@ dependencies = [ [[package]] name = "el-chat" -version = "0.3.10" +version = "0.3.11" dependencies = [ "el-core", "el-engine-candle", @@ -811,7 +811,7 @@ dependencies = [ [[package]] name = "el-cloud" -version = "0.3.10" +version = "0.3.11" dependencies = [ "el-core", "reqwest", @@ -822,11 +822,11 @@ dependencies = [ [[package]] name = "el-core" -version = "0.3.10" +version = "0.3.11" [[package]] name = "el-engine-candle" -version = "0.3.10" +version = "0.3.11" dependencies = [ "candle-core", "candle-transformers", @@ -838,7 +838,7 @@ dependencies = [ [[package]] name = "el-ffi" -version = "0.3.10" +version = "0.3.11" dependencies = [ "el-cloud", "el-core", @@ -852,7 +852,7 @@ dependencies = [ [[package]] name = "el-grammar" -version = "0.3.10" +version = "0.3.11" dependencies = [ "el-core", "el-provenance", @@ -861,21 +861,21 @@ dependencies = [ [[package]] name = "el-memory" -version = "0.3.10" +version = "0.3.11" dependencies = [ "el-core", ] [[package]] name = "el-provenance" -version = "0.3.10" +version = "0.3.11" dependencies = [ "el-core", ] [[package]] name = "el-provenance-ed25519" -version = "0.3.10" +version = "0.3.11" dependencies = [ "ed25519-dalek", "el-core", @@ -884,7 +884,7 @@ dependencies = [ [[package]] name = "el-runtime" -version = "0.3.10" +version = "0.3.11" dependencies = [ "el-core", "el-memory", @@ -894,14 +894,14 @@ dependencies = [ [[package]] name = "el-safety" -version = "0.3.10" +version = "0.3.11" dependencies = [ "el-core", ] [[package]] name = "el-telemetry" -version = "0.3.10" +version = "0.3.11" dependencies = [ "el-core", ] diff --git a/Cargo.toml b/Cargo.toml index 2fa495e..ab7dcf8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ exclude = [ [workspace.package] edition = "2021" -version = "0.3.10" +version = "0.3.11" license = "Apache-2.0" rust-version = "1.96" repository = "Edge-Native LLM SDK" @@ -48,16 +48,16 @@ repository = "Edge-Native LLM SDK" # This is the single source of truth for internal dep version requirements; # member crates reference them with `{ workspace = true }` and add no inline # version fields. `cargo set-version --workspace ` updates every entry. -el-core = { path = "crates/el-core", version = "0.3.10" } -el-memory = { path = "crates/el-memory", version = "0.3.10" } -el-telemetry = { path = "crates/el-telemetry", version = "0.3.10" } -el-provenance = { path = "crates/el-provenance", version = "0.3.10" } -el-safety = { path = "crates/el-safety", version = "0.3.10" } -el-runtime = { path = "crates/el-runtime", version = "0.3.10" } -el-grammar = { path = "crates/el-grammar", version = "0.3.10" } -el-provenance-ed25519 = { path = "crates/adapters/el-provenance-ed25519", version = "0.3.10" } -el-engine-candle = { path = "crates/adapters/el-engine-candle", version = "0.3.10" } -el-cloud = { path = "crates/adapters/el-cloud", version = "0.3.10" } +el-core = { path = "crates/el-core", version = "0.3.11" } +el-memory = { path = "crates/el-memory", version = "0.3.11" } +el-telemetry = { path = "crates/el-telemetry", version = "0.3.11" } +el-provenance = { path = "crates/el-provenance", version = "0.3.11" } +el-safety = { path = "crates/el-safety", version = "0.3.11" } +el-runtime = { path = "crates/el-runtime", version = "0.3.11" } +el-grammar = { path = "crates/el-grammar", version = "0.3.11" } +el-provenance-ed25519 = { path = "crates/adapters/el-provenance-ed25519", version = "0.3.11" } +el-engine-candle = { path = "crates/adapters/el-engine-candle", version = "0.3.11" } +el-cloud = { path = "crates/adapters/el-cloud", version = "0.3.11" } [profile.release] opt-level = 3 From 0565048fad17830ea60cd653eb4c5180e45ac023 Mon Sep 17 00:00:00 2001 From: Tovli Date: Tue, 23 Jun 2026 21:30:31 +0300 Subject: [PATCH 3/3] docs(benchmarks): add AC-3 full-suite perf report; summarize progression in README - New report 2026-06-23-adr-018-ac3-prefix-reuse-full-suite.md: full el-bench run (66 tasks / 97 replies, 0 errors, 35.7 min) on the same i5-14500 host as the prior reports. AC-3 cuts prefill from 56.8% to 29.5% of compute and the whole-suite wall by 33.5% vs the 2026-06-22 run; mindeval ~3x faster. Notes that the prefill *kernel* is unchanged (~15 tok/s per forward) so ADR-020 batched prefill is still pending, and decode (~14 tok/s) is now the dominant cost. - README: replace the single-report Benchmarks blurb with a progression table across the three perf reports plus a clinical-safety headline. --- README.md | 86 ++++--- ...-23-adr-018-ac3-prefix-reuse-full-suite.md | 241 ++++++++++++++++++ 2 files changed, 295 insertions(+), 32 deletions(-) create mode 100644 docs/benchmarks/2026-06-23-adr-018-ac3-prefix-reuse-full-suite.md diff --git a/README.md b/README.md index cd7d664..60da119 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,9 @@ # Edge Intelligence +[![npm package](https://img.shields.io/npm/v/edge-intelligence-sdk?style=for-the-badge&logo=npm&logoColor=white&label=npm)](https://www.npmjs.com/package/edge-intelligence-sdk) +[![pub.dev package](https://img.shields.io/pub/v/edge_intelligence?style=for-the-badge&logo=dart&logoColor=white&label=pub.dev)](https://pub.dev/packages/edge_intelligence) +[![crates.io package](https://img.shields.io/crates/v/el-core?style=for-the-badge&logo=rust&logoColor=white&label=crates.io%20el-core)](https://crates.io/crates/el-core) ![Rust](https://img.shields.io/badge/Rust-2021-f46623?style=for-the-badge&logo=rust&logoColor=white) ![Status](https://img.shields.io/badge/status-active%20prototype-2563eb?style=for-the-badge) ![Privacy](https://img.shields.io/badge/default-air--gapped-10b981?style=for-the-badge) @@ -269,38 +272,57 @@ workspace build (it pulls crates.io-only grammar dependencies); `el-cloud` and The SDK ships two reproducible benchmark harnesses. Both run inference through the public `LlmProvider` seam, so they characterize the **SDK's own behavior** and are **model-agnostic** — point them at whichever signed model your product loads. The -model is pluggable and not part of this repo, so no per-model results are published -here; run the harnesses against your own model to produce them. - -**1. Runtime performance / overhead** — -[`docs/benchmarks/2026-06-14-qwen-chat-bottleneck.md`](docs/benchmarks/2026-06-14-qwen-chat-bottleneck.md) - -A phase-level latency breakdown of the local decode path, behind opt-in, -zero-cost-when-unset instrumentation (`EL_BENCH=1`). The SDK-side conclusion: the -runtime's own per-token work — decode loop, grammar mask, full-vocab argmax, KV -commit, and content-free event emission — is **under ~1.2% of decode time**. -Latency is dominated by model compute plus two orchestration costs the SDK can -remove, independent of model choice: - -- **Batch the prefill** — feed the prompt as one `(1, prompt_len)` forward instead - of one forward per prompt token. -- **Load weights once; reset only the KV cache** — and reuse KV across turns, so a - growing conversation is not re-prefilled from scratch each turn. -- **Stream tokens for real** — emit from inside the decode loop so time-to-first- - token is `load + prefill + 1 token`, not full-generation time. - -**2. Clinical-quality & safety evaluation** — [`apps/el-bench`](apps/el-bench) · -[`benchmarks/README.md`](benchmarks/README.md) - -`el-bench` is an SDK-only test client (a sibling to `el-chat`) that replays -published mental-health benchmarks — CounselBench, MindEval, and the VERA-MH -suicide-risk safety suite — through the runtime and records transcripts for -scoring against each benchmark's rubric. Datasets and transcripts are fetched or -produced locally and are git-ignored (third-party data); only the harness and the -methodology are committed. Decoding is deterministic, so a given model + task set -yields identical transcripts — it is designed to run as a **CI safety gate**, so a -change to the model, the system prompt, or the ADR-005 safety tier can be -regression-tested against a fixed rubric. +harnesses are model-pluggable; the dated reports under +[`docs/benchmarks/`](docs/benchmarks) publish **reference results on the prototype's +reference model** (Qwen2.5-0.5B-Instruct q4_k_m, Intel i5-14500, release build) so +the trend is visible. Re-run either harness against your own model to reproduce. + +### 1. Runtime performance — measured progression + +A phase-level breakdown of the local decode path, behind opt-in, +zero-cost-when-unset instrumentation (`EL_BENCH=1`). The SDK-side conclusion is +stable across every run: the runtime's own per-token glue — decode loop, grammar +mask, full-vocab argmax, KV commit, content-free event emission — is **under ~1.2% +of decode time (~99% is raw model forward)**. The wins have therefore come from +removing *orchestration* costs, tracked across three reports: + +| Milestone | Report | Per-turn weight reload | Prefill (full suite) | Multi-turn re-prefill | Full-suite wall¹ | +|-----------|--------|------------------------|----------------------|-----------------------|------------------| +| Baseline (pre-ADR-018) | [2026-06-14](docs/benchmarks/2026-06-14-qwen-chat-bottleneck.md) | **~1.2 s every turn** | un-batched, ~15 tok/s, the #1–2 cost | whole conversation re-prefilled each turn | — (micro-bench) | +| Resident model + stateful sessions ([ADR-018](docs/adr/ADR-018-persistent-model-instances-and-stateful-sessions.md)) | [2026-06-22](docs/benchmarks/2026-06-22-adr-018-resident-model-full-suite.md) | **0 ms** (loaded once) | 56.8% of compute (now the #1 cost) | still re-prefilled | **53.7 min** | +| + Cross-turn prefix reuse (ADR-018 AC-3) | [2026-06-23](docs/benchmarks/2026-06-23-adr-018-ac3-prefix-reuse-full-suite.md) | 0 ms | **29.5%** of compute (−65% wall) | **only the new suffix is prefilled** | **35.7 min (−33.5%)** | + +¹ Same `el-bench` suite (66 tasks → 97 replies, 256 max-tokens, safety off), same +host — the 06-22 and 06-23 rows are directly comparable. The 06-14 baseline is an +`el-chat` micro-benchmark that *isolated* the bottlenecks, so its wall time is not +run-for-run comparable; the per-phase rates and bottleneck status are. + +**Net effect so far:** the per-turn 491 MB weight-reload tax is gone, and multi-turn +prefill no longer re-processes the whole conversation — cutting the full-suite run by +a third and making multi-turn (`mindeval`) **~3× faster** (52.2 → 17.5 s/reply). The +remaining levers are kernel-level: **batching the prefill of a cold/first turn** +([ADR-020](docs/adr/ADR-020-batched-single-pass-prefill.md) — the per-forward prefill +rate is still ~15 tok/s) and the **~14 tok/s quantized decode floor**, which is now +the dominant cost. + +### 2. Clinical-quality & safety evaluation + +[`apps/el-bench`](apps/el-bench) · [`benchmarks/README.md`](benchmarks/README.md) · +[2026-06-15 report](docs/benchmarks/2026-06-15-clinical-quality-safety.md) + +`el-bench` replays published mental-health benchmarks — CounselBench, MindEval, and +the VERA-MH suicide-risk safety suite (66 tasks → 97 replies) — through the runtime +and records transcripts for scoring against each benchmark's rubric. Datasets and +transcripts are fetched or produced locally and are git-ignored (third-party data); +only the harness and the methodology are committed. Decoding is deterministic, so a +given model + task set yields identical transcripts — it is designed to run as a +**CI safety gate** whenever the model, system prompt, or ADR-005 safety tier changes. + +> **Headline (reference model):** used as an *unsupervised* responder, the raw 0.5B +> model is **categorically unsafe for crisis scenarios** — 0/15 risk personas got a +> specific crisis resource and 0/17 got a direct safety question. This is the hard +> requirement behind ADR-005 decoder-time safety and a dedicated crisis-routing +> layer; see the report for the full rubric-by-rubric findings. ## Architecture decisions diff --git a/docs/benchmarks/2026-06-23-adr-018-ac3-prefix-reuse-full-suite.md b/docs/benchmarks/2026-06-23-adr-018-ac3-prefix-reuse-full-suite.md new file mode 100644 index 0000000..8fea1de --- /dev/null +++ b/docs/benchmarks/2026-06-23-adr-018-ac3-prefix-reuse-full-suite.md @@ -0,0 +1,241 @@ +# SDK Benchmark — Full-suite performance after ADR-018 AC-3 (cross-turn prefix reuse) + +**Date:** 2026-06-23 +**Subject:** End-to-end performance of the local inference path exercised by the +`el-bench` harness (`el_core::LlmProvider` → `el_engine_candle::QwenChatProvider` +→ `el_runtime::InferenceSession`) across the **full** normalized benchmark suite +(CounselBench, MindEval, VERA-MH), now with **[ADR-018](../adr/ADR-018-persistent-model-instances-and-stateful-sessions.md) +AC-3 (cross-turn prefix reuse / incremental prefill)** landed. +**Question:** The [2026-06-22 full-suite report](./2026-06-22-adr-018-resident-model-full-suite.md) +deferred AC-3 and found prefill was the #1 cost (56.8%, un-batched). With AC-3 now +implemented, where does the time go — and does the bottleneck shift? + +--- + +## 1. Executive summary + +Measured on the **same host** as the [2026-06-22](./2026-06-22-adr-018-resident-model-full-suite.md) +and [2026-06-14](./2026-06-14-qwen-chat-bottleneck.md) reports (Intel i5-14500, +14C/20T, 32 GB; release build), driving the real `el-bench` binary over **all 66 +tasks → 97 model replies, 0 errors, in 35.7 min (2141.5 s)**, safety `Off` (raw-model +baseline — the SDK safety layer is benched separately). SDK version **0.3.11**. + +AC-3 changed the picture exactly where the prior report predicted: + +| 2026-06-22 finding | Status now | Evidence | +|---|---|---| +| **#1 Prefill is the largest cost** (56.8%, 1827 s) | **Cut by 65% → 29.5% (632 s)** by AC-3 | Multi-turn replies now reuse the cached KV for the unchanged prefix and prefill only the new suffix: **9,893 prefill forwards for 27,148 presented prompt tokens** (64% of prefill positions served from cache). | +| **Multi-turn re-prefill (`mindeval`)** — re-prefilled the whole conversation each turn | **FIXED by AC-3** | `mindeval` avg time/reply **52,225 ms → 17,478 ms (~3×)**; e2e rate **2.7 → 8.1 tok/s**. | +| **#3 Decode compute floor** (~14 tok/s) | **Unchanged → now the #1 cost** | Decode 70.5% (1509 s) at **13.1 tok/s**; 99.2% candle forward. | +| Per-turn weight reload (fixed by ADR-018 core) | **Still fixed** | `session setup` is `0.0 ms` on all 97 replies. | + +**Headline:** AC-3 removed the redundant re-prefill of unchanged conversation +prefixes. Prefill dropped from the dominant cost (56.8%) to **29.5%**, cutting +**~20 min off the full run (53.7 → 35.7 min, −33.5%)** — almost entirely on the +multi-turn `mindeval` suite. With prefill shrunk, **decode is now the single largest +cost (70.5%)** — the unchanged ~14 tok/s candle q4_k_m batch-1 floor. + +**Important nuance — the prefill *kernel* did not get faster.** AC-3 makes prefill +cheaper by doing **less work**, not faster math: the per-forward prefill rate is +**15.7 tok/s (9,893 forwards / 632 s) — unchanged from the 14.9 tok/s baseline**. +Each prefill step is still **one token per forward**, so +[ADR-020 (batched single-pass prefill)](../adr/ADR-020-batched-single-pass-prefill.md) +remains **not done** and is still the lever for *single-turn* and *first-turn* +prefill (where there is no prefix to reuse). AC-3 and ADR-020 are complementary: +**AC-3 removes the unnecessary prefill; ADR-020 makes the necessary prefill fast.** + +### 1.1 Versus the [2026-06-22 run](./2026-06-22-adr-018-resident-model-full-suite.md) + +Same harness, same inputs, same host, same configuration as 2026-06-22 — the **only** +change is the AC-3 implementation in the working tree (`el-runtime/src/{session,ports}.rs`, +`el-engine-candle/src/lib.rs`). So this is a clean run-for-run comparison. + +| Metric | 2026-06-22 (no AC-3) | 2026-06-23 (AC-3) | Change | +|---|---|---|---| +| **Total run wall** | 3221 s (53.7 min) | **2141.5 s (35.7 min)** | **−33.5%** ✅ | +| Prefill wall / share | 1827.2 s / 56.8% | **632.1 s / 29.5%** | **−65% wall** ✅ | +| Decode wall / share | 1387.9 s / 43.2% | 1509.2 s / 70.5% | ~flat (now dominant) | +| Prefill forwards | ~27,148 (1/token) | **9,893** | **−64%** (reuse) ✅ | +| Prefill per-forward rate | 14.9 tok/s | 15.7 tok/s | unchanged (no ADR-020) | +| Decode throughput | 14.2 tok/s | 13.1 tok/s | ~flat (−8%, thermal/noise) | +| `mindeval` avg ms/reply | 52,225 | **17,478** | **~3× faster** ✅ | +| `mindeval` e2e tok/s | 2.7 | 8.1 | **~3×** ✅ | +| Model forward share of compute | 99.3% | 99.2% | unchanged (SDK glue negligible) | +| Per-turn weight reload | 0.0 ms | 0.0 ms | unchanged (ADR-018 core) | + +**Reading:** AC-3 targeted exactly the structural cost the prior report isolated — +re-prefilling the unchanged prefix every turn — and eliminated it. The decode floor +(#3/#5) and the un-batched prefill *kernel* (ADR-020) are untouched by design, so +decode is now the dominant cost and first-turn prefill rate is unchanged. + +--- + +## 2. Method + +### 2.1 Harness + +The benchmark drives the **actual** `el-bench` release binary (SDK v0.3.11) — no +synthetic harness — so every number reflects the real public SDK path +(`LlmProvider::chat` → `QwenChatProvider` → `QwenEngine` + `InferenceSession`), +including the ADR-018 resident session **and** the AC-3 cross-turn prefix reuse +landed in this branch (`InferenceSession::continue_prompt` → +`InferenceEngine::prefill_reuse`, which reuses the longest cached token prefix that +still matches the new context and feeds only the divergent suffix). + +Per-phase timing comes from the same opt-in, env-gated instrumentation +(`EL_BENCH=1`) used in the prior reports (`crates/adapters/el-engine-candle/src/lib.rs`): +each `chat()` is split into **session setup → tokenize → prefill → decode → +detokenize**, and each `forward_one` into **model** (candle transformer forward) vs +**seam** (tensor build + `to_vec1` + float→milli-logit), with the remainder +attributed to the runtime **loop**. Zero-cost when unset. + +Per-suite figures are aggregated from the transcript JSONL (`prompt_tokens`, +`completion_tokens`, `ms` per exchange); the prefill/decode split and forward-call +counts from the `EL_BENCH` stderr log. + +### 2.2 Configuration + +- **Model:** `qwen2.5-0.5b-instruct-q4_k_m.gguf` (491 MB) + `tokenizer.json`, local `models/` (air-gapped, ADR-004). +- **Build:** `cargo build --release -p el-bench` (`opt-level=3`, LTO), SDK v0.3.11. +- **Safety:** `SafetyMode::Off` — el-bench benches the raw model; the ADR-005/012 safety loop is evaluated in the clinical-safety report. +- **Decoding:** deterministic greedy argmax → reproducible. +- **Tasks:** `benchmarks/tasks/{counselbench-adv,counselbench-eval,mindeval,veramh}.jsonl`, default `--max-tokens 256`, no `--limit` (full suite). +- **CPU:** Intel i5-14500 (6 P + 8 E, 20 threads); candle `gemm`/rayon CPU backend; no GPU. + +--- + +## 3. Raw results + +### 3.1 Per-suite (end-to-end, from the transcript) + +`avg ms/reply` is wall time per `chat()` (prefill + decode); `e2e tok/s` is +`completion_tokens / total_time`. + +| Suite | tasks | replies | prompt tok | compl tok | avg ms/reply | e2e tok/s | +|-------|------:|--------:|-----------:|----------:|-------------:|----------:| +| counselbench-adv | 24 | 24 | 3,168 | 5,717 | 26,248 | 9.1 | +| counselbench-eval | 20 | 20 | 3,126 | 5,112 | 26,880 | 9.5 | +| mindeval | 5 | 30 | 16,491 | 4,258 | **17,478** | **8.1** | +| veramh | 17 | 23 | 4,363 | 4,706 | 19,544 | 10.5 | +| **TOTAL** | 66 | 97 | 27,148 | 19,793 | 22,076 | 9.2 | + +> `mindeval` is where AC-3 pays off: in the 2026-06-22 run it averaged 52,225 ms/reply +> (2.7 tok/s) because each turn re-prefilled the entire growing conversation; here it +> is 17,478 ms/reply (8.1 tok/s), now in line with the largely single-turn suites. + +### 3.2 Prefill vs decode (summed over all 97 replies, from `EL_BENCH`) + +| Phase | wall | share | forwards | presented tokens | rate | +|-------|-----:|------:|---------:|-----------------:|-----:| +| **prefill** | 632.1 s | **29.5%** | 9,893 | 27,148 | **42.9 tok/s effective** / 15.7 tok/s per-forward | +| **decode** | 1509.2 s | **70.5%** | 19,696 | 19,793 | **13.1 tok/s** | + +- **Effective vs per-forward prefill rate:** 42.9 tok/s is *presented* prompt tokens + ÷ wall — it counts prefix tokens served from cache for free. The true kernel rate + is **9,893 forwards / 632.1 s = 15.7 tok/s** (≈ the 14.9 tok/s baseline). The gap is + the AC-3 win: **17,255 of 27,148 prefill positions (64%) never hit the model.** +- **Compute attribution:** candle model forward is **99.2%** of prefill+decode wall + (628.0 s prefill + 1495.7 s decode = 2123.7 s of 2141.3 s); seam quantization + the + runtime decode loop are ~0.8% combined. Per decoded token: **76.6 ms = model 75.9 + + seam 0.4 + loop 0.3**. +- **Session setup:** `0.0 ms` on all 97 replies (weights loaded once — ADR-018 core). + +--- + +## 4. Analysis + +### 4.1 AC-3 (cross-turn prefix reuse) — the win, proven per-conversation + +The aggregate (9,893 forwards for 27,148 presented tokens) is the suite-wide +signature of reuse, but the mechanism is clearest **within a single multi-turn +conversation**. From the `EL_BENCH` log for `mindeval-f28-marketingc` (6 turns), the +prefill forward count **falls turn-over-turn even as the prompt grows**, because each +turn reuses the prior turns' cached KV and forwards only the new suffix: + +| Turn | presented prompt tok | prefill forwards | reused from cache | +|-----:|---------------------:|-----------------:|------------------:| +| 1 | 90 | 90 | 0 (cold) | +| 2 | 149 | 39 | 110 | +| 3 | 268 | 24 | 244 | +| 4 | 364 | 24 | 340 | +| 5 | 449 | 14 | 435 | + +This is `InferenceSession::continue_prompt` → `prefill_reuse` reusing the longest +matching cached token prefix (system prompt + all prior turns, now baked into the +re-rendered context) and feeding only the divergent suffix. Single-turn replies +(all of counselbench, most of veramh) show `forwards == prompt_tokens` — no prefix +to reuse on a cold conversation, so they are unchanged from the baseline (correctly). + +### 4.2 The prefill kernel is unchanged — ADR-020 still the lever for first-turn + +AC-3 does **not** batch prefill. The per-forward prefill rate is **15.7 tok/s**, +statistically identical to the 14.9 tok/s baseline, because each prefill step is +still **one `(1,1)` forward per token** (`QwenEngine::prefill`). A cold first turn, +or any single-turn task, still pays the full token-by-token prefill at ~15 tok/s. +Replacing that loop with one batched `(1, suffix_len)` forward +([ADR-020](../adr/ADR-020-batched-single-pass-prefill.md)) is now the highest-value +remaining prefill optimization, and it composes with AC-3: AC-3 shrinks the *number* +of suffix tokens that must be prefilled; ADR-020 makes prefilling them fast. + +### 4.3 Decode (~14 tok/s) is now the dominant cost — unchanged floor + +With prefill cut by 65%, **decode is now 70.5% of compute** at 13.1 tok/s (per-token +76.6 ms, 99.2% candle q4_k_m forward) — a memory-bandwidth-bound batch-1 decode +floor, an engine/kernel concern, not SDK orchestration. It is ~8% slower than the +14.2 tok/s measured on 2026-06-22; with model forward = 99.2% and the SDK glue +unchanged, that delta is consistent with run-to-run thermal/scheduling variance over +a 36-min CPU-bound run, not a regression in SDK code. This is the same floor called +out as bottleneck #3/#5 in the prior reports and remains out of scope for P0. + +### 4.4 Correctness of reuse + +AC-3 ships with a soundness contract: `prefill_reuse` must leave the cache *byte-for-byte* +as a full `reset_cache()` + `prefill(full_context)` would (identical subsequent +logits) — reuse is purely a compute optimization, never a change to *what* the cache +represents. The implementation fails **open** (a wrong/short prefix match discards +the cache and re-prefills the whole context), and a `ReuseEngine` test mock proves +reuse-equals-fresh-prefill without a real model. The full suite ran with **0 errors** +and deterministic greedy decoding, and the per-suite completion-token counts match +the 2026-06-22 run (e.g. mindeval 4,258 compl tok both runs) — i.e. AC-3 changed the +*speed*, not the *output*. + +--- + +## 5. Recommendations (prioritized) + +1. **Batch the prefill — [ADR-020](../adr/ADR-020-batched-single-pass-prefill.md).** + Now the largest *remaining* prefill lever (AC-3 cut the count, not the per-token + cost). Replace the per-token loop in `QwenEngine::prefill` with one + `(1, suffix_len)` forward. Composes with AC-3; biggest win for first/single-turn. +2. **Faster quantized decode kernel** — decode is now the dominant cost (70.5%) at + the ~14 tok/s candle CPU matmul floor. Largest absolute ceiling, hardest; revisit + after ADR-020. +3. **Real token streaming — [ADR-019](../adr/ADR-019-in-loop-incremental-decoding-and-token-streaming.md).** + TTFT is still full-generation time; a per-token emit hook drops it to + `prefill + 1 token`. UX, separate from throughput. +4. **Baseline instrumentation as a CI gate — [ADR-023](../adr/ADR-023-baseline-performance-instrumentation.md).** + Capture this run (prefill share 29.5%, forwards 9,893/27,148, decode 13.1 tok/s) as + the machine-readable baseline so ADR-020's win is regression-gated and the AC-3 gain + here is protected. + +--- + +## 6. Reproduce + +```sh +cargo build --release -p el-bench + +# Full suite (this run): all tasks, default 256 max-tokens, EL_BENCH timing. +EL_BENCH=1 ./target/release/el-bench.exe \ + --out benchmarks/out/transcripts.jsonl + +# Bounded smoke (fast): a multi-turn task to see AC-3 reuse in the EL_BENCH log +# (prefill `fwd calls` drops on turns 2+ while prompt_tokens grows). +EL_BENCH=1 ./target/release/el-bench.exe \ + --suite mindeval --limit 1 --out /tmp/perf.jsonl +``` + +`EL_BENCH` prints the per-phase table + forward attribution to stderr; the transcript +JSONL records per-reply `prompt_tokens` / `completion_tokens` / `ms`. Per-suite +figures are aggregated from the transcript, the prefill/decode split and forward-call +counts from the `EL_BENCH` stderr log.