Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,4 @@ models/
crates/adapters/el-ffi/src/frb_generated.rs
# Claude Code local state (worktrees, settings.local.json)
.claude/
benchmarks/data/
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

41 changes: 38 additions & 3 deletions crates/adapters/el-engine-candle/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,12 @@ impl InferenceEngine for CandleEngine {
fn eos_token(&self) -> Token {
self.eos
}

/// Stateless: this engine's forward is `embed[committed.last()] · w_out`, so
/// it holds no KV cache to restore — a rollback is a no-op.
fn rollback(&mut self, _keep_committed: u32) -> Result<()> {
Ok(())
}
}

// ── LlmProvider (text-level) wrapper (ADR-010) ───────────────────────────────
Expand Down Expand Up @@ -343,16 +349,25 @@ mod bench {
///
/// Holds candle's stateful KV cache. Within one generation it is fed
/// incrementally (prefill, then one new token per `next_logits` call); candle
/// exposes no public cache reset, so a fresh conversation builds a new engine.
/// Float logits are quantised to integer milli-logits at the seam, exactly like
/// [`CandleEngine`], so the runtime stays float-free.
/// exposes no public cache reset, so a *fresh conversation* builds a new engine.
///
/// A *within-generation* safety backtrack (ADR-012) is supported via
/// [`InferenceEngine::rollback`]: candle's attention discards its cache when a
/// forward runs at `index_pos == 0`, so we retain the prompt and replay it from
/// position 0 to rebuild the cache for the safe prefix (the session then
/// re-feeds the retained committed tokens). Float logits are quantised to
/// integer milli-logits at the seam, exactly like [`CandleEngine`], so the
/// runtime stays float-free.
pub struct QwenEngine {
model: Qwen2Weights,
device: Device,
/// Absolute KV position written so far (candle's `index_pos`).
index_pos: usize,
/// How many of the runtime-`committed` tokens have already been fed.
fed: usize,
/// The prefill prompt, retained so a rollback can replay it from position 0
/// to rebuild candle's KV cache (which has no public truncation).
prompt: Vec<Token>,
/// Milli-logits produced after the most recent forward.
last_logits: Vec<i32>,
vocab: usize,
Expand All @@ -375,6 +390,7 @@ impl QwenEngine {
device,
index_pos: 0,
fed: 0,
prompt: Vec::new(),
last_logits: Vec::new(),
vocab: 0,
eos,
Expand Down Expand Up @@ -416,6 +432,7 @@ impl InferenceEngine for QwenEngine {
fn prefill(&mut self, tokens: &[Token]) -> Result<u32> {
self.index_pos = 0;
self.fed = 0;
self.prompt = tokens.to_vec(); // retained for rollback replay
for &t in tokens {
self.last_logits = self.forward_one(t)?;
}
Expand All @@ -441,6 +458,24 @@ impl InferenceEngine for QwenEngine {
fn eos_token(&self) -> Token {
self.eos
}

fn rollback(&mut self, _keep_committed: u32) -> Result<()> {
// candle's KV cache is append-only with no public truncation, but its
// attention discards the cache on a forward at `index_pos == 0` (see
// quantized_qwen2). So rebuild deterministically: replay the prompt from
// position 0 — the first forward resets the cache, the rest re-append it —
// leaving the engine in its exact post-prefill state. We reset `fed` to 0
// so the session's next `next_logits` re-feeds the retained committed
// prefix (already truncated to `keep_committed`) on top. Cost is bounded
// by `max_rollbacks` (ADR-012).
self.index_pos = 0;
self.fed = 0;
for i in 0..self.prompt.len() {
let t = self.prompt[i];
self.last_logits = self.forward_one(t)?;
}
Ok(())
}
}

/// A real local chat backend: a Qwen2 GGUF model + its tokenizer, driven
Expand Down
3 changes: 3 additions & 0 deletions crates/el-grammar/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,9 @@ mod tests {
fn eos_token(&self) -> Token {
99
}
fn rollback(&mut self, _keep: u32) -> el_core::Result<()> {
Ok(()) // stateless
}
}

struct OkVerifier;
Expand Down
22 changes: 22 additions & 0 deletions crates/el-memory/src/kv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,14 @@ impl KvRegion {
&self.slots
}

/// Drop tail descriptors so the region retains only the first `len` slots.
/// `O(dropped)`; survivors keep their `offset` (no data copy). This is the
/// rollback primitive for the safety control loop (ADR-012): restoring a
/// checkpoint rewinds committed KV without replaying prefill.
pub fn truncate(&mut self, len: u32) {
self.slots.truncate(len as usize);
}

/// Remove pruned descriptors and re-index survivors. Returns how many were
/// reclaimed. **Survivors keep their original `offset`** — proof that data
/// is not moved, only descriptors are reshuffled.
Expand Down Expand Up @@ -87,4 +95,18 @@ mod tests {
// ...but its logical index was shuffled down.
assert_eq!(kv.slots()[1].token_index, 1);
}

#[test]
fn truncate_rewinds_to_len_without_touching_survivors() {
let mut kv = KvRegion::new();
kv.push(0);
kv.push(64);
kv.push(128);
let off1 = kv.slots()[1].offset;
kv.truncate(2);
assert_eq!(kv.len(), 2);
assert_eq!(kv.slots()[1].offset, off1); // survivor untouched (no data copy)
kv.truncate(0);
assert!(kv.is_empty());
}
}
5 changes: 5 additions & 0 deletions crates/el-runtime/src/defaults.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,4 +52,9 @@ impl InferenceEngine for NullEngine {
fn eos_token(&self) -> Token {
self.eos
}

/// Stateless: `next_logits` ignores `committed`, so there is nothing to undo.
fn rollback(&mut self, _keep_committed: u32) -> Result<()> {
Ok(())
}
}
7 changes: 5 additions & 2 deletions crates/el-runtime/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,8 @@ pub use defaults::{AllowAllMasker, IdentityCompressor, NullEngine};
pub use ports::{GrammarMasker, HybridRelay, InferenceEngine, Ports, PromptCompressor};
pub use session::InferenceSession;

// Re-export the safety port so callers wire one type system.
pub use el_safety::{LogitAdjustment, SafetySteerer};
// Re-export the safety ports so callers wire one type system.
pub use el_safety::{
Checkpoint, CheckpointManager, ChunkGuard, LogitAdjustment, RollbackPolicy, SafetyScore,
SafetySteerer,
};
30 changes: 28 additions & 2 deletions crates/el-runtime/src/ports.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Port traits the Inference Runtime depends on (the collaborator contexts).

use el_core::{Result, Token};
use el_safety::SafetySteerer;
use el_safety::{ChunkGuard, SafetySteerer};

/// The inference engine adapter (`RuntimeAcl`). Implemented for real by Candle
/// in the excluded adapter `el-engine-candle` (ADR-002).
Expand All @@ -15,6 +15,28 @@ pub trait InferenceEngine {
fn next_logits(&mut self, committed: &[Token]) -> Vec<i32>;
/// The end-of-sequence token id.
fn eos_token(&self) -> Token;

/// Roll the engine's internal state back so its context is exactly the
/// prompt plus `keep_committed` generated tokens.
///
/// The ADR-012 control loop truncates the session's committed output and KV
/// descriptors on a safety backtrack. A **stateful** engine (one holding a
/// real KV cache and position counters, e.g. a transformer) must mirror that
/// truncation here — otherwise it keeps serving logits from the abandoned
/// (unsafe) branch and never re-feeds the replacement tokens, so the rollback
/// is silently a no-op at the engine level.
///
/// After `Ok(())`, the next [`next_logits`](Self::next_logits) call — passed a
/// `committed` slice of length `keep_committed` — must produce logits
/// consistent with that prefix. Returning `Err` makes the loop fail closed
/// rather than resume on an inconsistent cache.
///
/// This method is **required, with no default**, deliberately: a default
/// no-op would let a stateful adapter that forgot to override silently resume
/// on a stale KV cache — a safety bug that fails *open*. Every engine must
/// make the choice explicit. A stateless engine whose `next_logits`
/// recomputes purely from `committed` implements it as `Ok(())`.
fn rollback(&mut self, keep_committed: u32) -> Result<()>;
}

/// Prompt Compression port (LLMLingua-2 — context 2).
Expand All @@ -40,17 +62,21 @@ pub struct Ports {
pub compressor: Box<dyn PromptCompressor>,
pub grammar: Box<dyn GrammarMasker>,
pub safety: Box<dyn SafetySteerer>,
/// Optional chunk guard for the checkpointed-rollback control loop
/// (ADR-012). `None` runs the plain single-pass decode.
pub guard: Option<Box<dyn ChunkGuard>>,
pub relay: Option<Box<dyn HybridRelay>>,
}

impl Ports {
/// Defaults: identity compression, all-tokens-allowed grammar, no safety
/// steering, no relay (air-gapped).
/// steering, no guard, no relay (air-gapped).
pub fn permissive() -> Self {
Self {
compressor: Box::new(super::defaults::IdentityCompressor),
grammar: Box::new(super::defaults::AllowAllMasker),
safety: Box::new(el_safety::NoSafety),
guard: None,
relay: None,
}
}
Expand Down
Loading
Loading