From 53a3de4b4c80a6add509aa6a95e12d4603900606 Mon Sep 17 00:00:00 2001 From: Chetan Conikee Date: Mon, 13 Jul 2026 12:29:58 -0700 Subject: [PATCH 01/12] waggle-tree: pure index + search primitives for directory trees MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New sans-I/O crate — the deterministic foundation for tree-scale minting and search. Three structures, each a pure function of bytes so an index blob is content-addressable and byte-stable (I-2), and so the wasm edge can build/query them too: bloom.rs fixed-size (256 B), union-composable Bloom over trigrams — the prune gate that makes deep-tree search sublinear. Parent = OR of children, so nesting is free. Pinned seed + k, no false negatives, saturates gracefully. Inlinable in a manifest → a prune is a pure manifest read. trigram.rs case-folded 3-byte shingles + an inverted TrigramIndex (posting-list intersection). "grep N files" -> "look up a few short lists and confirm survivors." Hex-keyed wire form (folded bytes need not be UTF-8). dirindex.rs the Merkle directory node: files pinned by sha256, subdirs addressed by subtree token, with recursive files/bytes totals so a node knows its weight without a walk. search.rs the pure decisions: prune (Bloom gate), candidates, and rank (matches desc, then depth, then path — deterministic). Traversal, the filesystem walk, and the blob store stay in waggle-mcp; this crate holds only what must be deterministic and unit-testable in isolation. 22 tests, clippy clean, largest file 299 lines. --- Cargo.lock | 9 + Cargo.toml | 1 + crates/waggle-tree/Cargo.toml | 18 ++ crates/waggle-tree/src/bloom.rs | 268 ++++++++++++++++++++++++++ crates/waggle-tree/src/dirindex.rs | 187 ++++++++++++++++++ crates/waggle-tree/src/lib.rs | 38 ++++ crates/waggle-tree/src/search.rs | 131 +++++++++++++ crates/waggle-tree/src/trigram.rs | 299 +++++++++++++++++++++++++++++ 8 files changed, 951 insertions(+) create mode 100644 crates/waggle-tree/Cargo.toml create mode 100644 crates/waggle-tree/src/bloom.rs create mode 100644 crates/waggle-tree/src/dirindex.rs create mode 100644 crates/waggle-tree/src/lib.rs create mode 100644 crates/waggle-tree/src/search.rs create mode 100644 crates/waggle-tree/src/trigram.rs diff --git a/Cargo.lock b/Cargo.lock index c8b2c80..f69a807 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2337,6 +2337,15 @@ dependencies = [ "waggle-store-sqlite", ] +[[package]] +name = "waggle-tree" +version = "0.4.0" +dependencies = [ + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "wait-timeout" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 72a5f26..4af89e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ ed25519-dalek = { version = "2", default-features = false, features = ["std"] } tokio = { version = "1", default-features = false, features = ["rt-multi-thread", "net", "io-util", "macros", "time", "sync"] } waggle-core = { path = "crates/waggle-core", version = "0.4.0" } +waggle-tree = { path = "crates/waggle-tree", version = "0.4.0" } waggle-ops = { path = "crates/waggle-ops", version = "0.4.0" } waggle-agent = { path = "crates/waggle-agent", version = "0.4.0" } waggle-social = { path = "crates/waggle-social", version = "0.4.0" } diff --git a/crates/waggle-tree/Cargo.toml b/crates/waggle-tree/Cargo.toml new file mode 100644 index 0000000..f096066 --- /dev/null +++ b/crates/waggle-tree/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "waggle-tree" +description = "Pure, deterministic index and search primitives for directory trees (design doc: tree-scale). A Merkle directory index, a trigram inverted index, and a union-composable Bloom summary — sans-I/O, so the daemon and the edge can both build and query them. Traversal and storage live in waggle-mcp." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true +keywords.workspace = true +categories.workspace = true +readme.workspace = true +homepage.workspace = true + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } diff --git a/crates/waggle-tree/src/bloom.rs b/crates/waggle-tree/src/bloom.rs new file mode 100644 index 0000000..5babd1c --- /dev/null +++ b/crates/waggle-tree/src/bloom.rs @@ -0,0 +1,268 @@ +//! A fixed-size, union-composable Bloom filter over [`Trigram`]s. +//! +//! Each directory node in a tree carries one of these as a summary of *every* +//! trigram in its entire subtree. Search uses it as a prune gate: if the filter +//! says a query's trigrams are *definitely absent*, the whole subtree is skipped +//! without a blob fetch. That is what makes search over a deeply nested tree +//! sublinear — cost tracks the branches that could match, not the file count. +//! +//! Three properties earn its place here: +//! +//! * **Composable.** A parent's summary is the bitwise-OR of its children's +//! ([`Bloom::union`]). So summaries build bottom-up in O(1) per node, and depth +//! costs nothing. +//! * **Deterministic.** Fixed size, fixed hash count, a pinned seed — the same +//! trigrams always yield the same bits, so a manifest carrying a Bloom stays +//! byte-stable (invariant I-2). +//! * **Small.** [`Bloom::BYTES`] is 256, so it inlines in a node's manifest and a +//! prune decision is a pure manifest read. +//! +//! It saturates gracefully: a subtree with more distinct trigrams than the filter +//! can hold answers "maybe" more often, never "no" wrongly (a Bloom has no false +//! negatives). So a huge root filter degrades to "descend", and the *smaller* +//! subtree filters below it do the real pruning. + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +use crate::trigram::Trigram; + +/// A fixed-size Bloom filter. Construct with [`Bloom::new`], fill with +/// [`Bloom::insert`], combine with [`Bloom::union`], test with +/// [`Bloom::might_contain`]. +#[derive(Clone, PartialEq, Eq)] +pub struct Bloom { + /// [`Bloom::BYTES`] bytes of bit storage. Boxed so a `Bloom` is a thin + /// pointer rather than 256 bytes on the stack. + bits: Box<[u8; Bloom::BYTES]>, +} + +impl Bloom { + /// Storage size in bytes. 256 B = 2048 bits — selective for a focused + /// subtree (a few thousand distinct trigrams), small enough to inline. + pub const BYTES: usize = 256; + + /// Bit count (`BYTES * 8`). + pub const BITS: usize = Bloom::BYTES * 8; + + /// Hashes per element. Four keeps the false-positive rate low at the trigram + /// densities real documents produce, without over-saturating the bits. + pub const K: u32 = 4; + + /// Pinned seed for the base hash. Changing it changes every Bloom the system + /// has ever written, so it is a wire-format constant, not a tunable. + const SEED: u64 = 0x776167676c655f74; // "waggle_t" + + /// An empty filter — contains nothing, so [`Bloom::might_contain`] is always + /// `false` until something is inserted. + #[must_use] + pub fn new() -> Self { + Self { + bits: Box::new([0u8; Bloom::BYTES]), + } + } + + /// Record a trigram. Idempotent — inserting twice sets the same bits. + pub fn insert(&mut self, tri: Trigram) { + for pos in self.positions(tri) { + self.bits[pos / 8] |= 1 << (pos % 8); + } + } + + /// Insert every trigram of `text`. + pub fn insert_text(&mut self, text: &str) { + for tri in Trigram::all(text) { + self.insert(tri); + } + } + + /// `false` means the trigram is *definitely absent* — safe to prune. + /// `true` means *possibly present* — the caller must look closer. + #[must_use] + pub fn might_contain(&self, tri: Trigram) -> bool { + self.positions(tri) + .into_iter() + .all(|pos| self.bits[pos / 8] & (1 << (pos % 8)) != 0) + } + + /// Could this subtree contain *all* of a query's trigrams? A grep pattern + /// matches only where every one of its trigrams is present, so a subtree is + /// prunable the moment any query trigram is definitely absent. + #[must_use] + pub fn might_contain_all(&self, query: &str) -> bool { + Trigram::all(query).all(|tri| self.might_contain(tri)) + } + + /// Fold another filter in by bitwise-OR — the composition that lets a parent + /// summarise its children in O(1). + pub fn union(&mut self, other: &Bloom) { + for (a, b) in self.bits.iter_mut().zip(other.bits.iter()) { + *a |= *b; + } + } + + /// The fraction of bits set — a saturation gauge. Near 1.0 means the filter + /// has stopped discriminating (a very large subtree) and will answer "maybe" + /// for almost anything. Useful for diagnostics, never for correctness. + #[must_use] + pub fn saturation(&self) -> f32 { + let set: u32 = self.bits.iter().map(|b| b.count_ones()).sum(); + set as f32 / Bloom::BITS as f32 + } + + /// The `K` bit positions for a trigram, via double hashing: two independent + /// hashes `h1`, `h2` generate `h1 + i*h2` for `i in 0..K`. Standard, and it + /// avoids `K` separate hash passes. + fn positions(&self, tri: Trigram) -> [usize; Bloom::K as usize] { + let bytes = tri.bytes(); + let h1 = fnv1a(Bloom::SEED, &bytes); + let h2 = fnv1a(Bloom::SEED ^ 0x9e37_79b9_7f4a_7c15, &bytes) | 1; // odd → full period + let mut out = [0usize; Bloom::K as usize]; + for (i, slot) in out.iter_mut().enumerate() { + let h = h1.wrapping_add((i as u64).wrapping_mul(h2)); + *slot = (h % Bloom::BITS as u64) as usize; + } + out + } +} + +impl Default for Bloom { + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Debug for Bloom { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Bloom({:.0}% set)", self.saturation() * 100.0) + } +} + +/// FNV-1a, 64-bit. Deterministic and dependency-free — exactly what a +/// wire-stable, seed-pinned filter needs. +fn fnv1a(seed: u64, data: &[u8]) -> u64 { + let mut h = seed ^ 0xcbf2_9ce4_8422_2325; + for &b in data { + h ^= u64::from(b); + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + h +} + +// --- serde: a compact lowercase-hex string, so a manifest stays readable and +// byte-stable rather than carrying a 256-element number array. --- + +impl Serialize for Bloom { + fn serialize(&self, s: S) -> Result { + s.serialize_str(&to_hex(self.bits.as_ref())) + } +} + +impl<'de> Deserialize<'de> for Bloom { + fn deserialize>(d: D) -> Result { + let hex = String::deserialize(d)?; + let bytes = from_hex(&hex).map_err(serde::de::Error::custom)?; + let arr: [u8; Bloom::BYTES] = bytes.try_into().map_err(|v: Vec| { + serde::de::Error::custom(format!( + "bloom: expected {} bytes, got {}", + Bloom::BYTES, + v.len() + )) + })?; + Ok(Self { + bits: Box::new(arr), + }) + } +} + +fn to_hex(bytes: &[u8]) -> String { + let mut s = String::with_capacity(bytes.len() * 2); + for b in bytes { + s.push(char::from_digit((b >> 4) as u32, 16).unwrap()); + s.push(char::from_digit((b & 0xf) as u32, 16).unwrap()); + } + s +} + +fn from_hex(s: &str) -> Result, String> { + if s.len() % 2 != 0 { + return Err("bloom hex: odd length".into()); + } + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).map_err(|e| format!("bloom hex: {e}"))) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tri(a: u8, b: u8, c: u8) -> Trigram { + Trigram::from_bytes([a, b, c]) + } + + #[test] + fn absent_is_definitely_absent() { + let b = Bloom::new(); + assert!(!b.might_contain(tri(b'a', b'b', b'c'))); + } + + #[test] + fn inserted_is_possibly_present() { + let mut b = Bloom::new(); + b.insert(tri(b'a', b'b', b'c')); + assert!(b.might_contain(tri(b'a', b'b', b'c'))); + } + + #[test] + fn union_is_bitwise_or_and_preserves_membership() { + let mut a = Bloom::new(); + a.insert(tri(b'x', b'y', b'z')); + let mut b = Bloom::new(); + b.insert(tri(b'1', b'2', b'3')); + a.union(&b); + assert!(a.might_contain(tri(b'x', b'y', b'z'))); + assert!(a.might_contain(tri(b'1', b'2', b'3'))); + } + + #[test] + fn might_contain_all_prunes_on_any_missing_trigram() { + let mut b = Bloom::new(); + b.insert_text("retry budget"); + // "retry budget" trigrams are present… + assert!(b.might_contain_all("retry")); + // …but a word that shares no trigram is prunable. + assert!(!b.might_contain_all("zzzzzz")); + } + + #[test] + fn deterministic_across_instances() { + let mut a = Bloom::new(); + let mut b = Bloom::new(); + a.insert_text("the escalation policy"); + b.insert_text("the escalation policy"); + assert_eq!(a, b); + } + + #[test] + fn serde_round_trips_through_hex() { + let mut b = Bloom::new(); + b.insert_text("runbook 07 violates the ceiling"); + let json = serde_json::to_string(&b).unwrap(); + assert!(json.starts_with('"') && json.len() == Bloom::BYTES * 2 + 2); + let back: Bloom = serde_json::from_str(&json).unwrap(); + assert_eq!(b, back); + } + + #[test] + fn no_false_negatives_over_many_inserts() { + let mut b = Bloom::new(); + let words: Vec = (0..500).map(|i| format!("token{i:04}")).collect(); + for w in &words { + b.insert_text(w); + } + for w in &words { + assert!(b.might_contain_all(w), "false negative for {w}"); + } + } +} diff --git a/crates/waggle-tree/src/dirindex.rs b/crates/waggle-tree/src/dirindex.rs new file mode 100644 index 0000000..960e7e3 --- /dev/null +++ b/crates/waggle-tree/src/dirindex.rs @@ -0,0 +1,187 @@ +//! The directory index — one content-addressed node of the Merkle tree. +//! +//! A tree mint records a directory not as a flat list of individually-minted +//! files but as a hierarchy of *nodes*. Each node is a [`DirIndex`]: the ordered +//! entries directly inside one directory. An entry is either a [`FileEntry`] (a +//! file, pinned by content hash — the bytes travel with the tree, so a deleted +//! file still reads) or a [`SubdirEntry`] (a child directory, addressed by its +//! own subtree token). +//! +//! Two things fall out of this shape: +//! +//! * **Merkle identity.** A node's content is a pure function of its entries, and +//! an entry names a file by `sha256`. Identical files dedupe; a changed file +//! changes only its node and that node's ancestors — cheap change detection and +//! cheap re-mint. +//! * **Sizing without materialising.** Each subdir entry carries its subtree's +//! `files` and `bytes` totals, so a node knows the weight of everything beneath +//! it from the index alone — no walk, no token minting, to answer "how big." +//! +//! Sans-I/O: this module builds and reads the index structure. The filesystem +//! walk that produces entries, and the blob store that holds file bytes, live in +//! `waggle-mcp`. + +use serde::{Deserialize, Serialize}; + +/// A file inside a directory node, pinned by content hash. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FileEntry { + /// Name within this directory (not a full path). + pub name: String, + /// Lowercase-hex SHA-256 of the file's bytes — the blob key, and the Merkle + /// leaf hash. + pub sha256: String, + /// Size in bytes. + pub size: u64, + /// MIME type inferred at mint (`text/markdown`, `application/pdf`, …). + pub content_type: String, +} + +/// A child directory, addressed by its own subtree token, with the totals a +/// parent needs to report size and to plan a search without descending. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SubdirEntry { + /// Name within this directory. + pub name: String, + /// The subtree's own token — the handle to recurse into. + pub token: String, + /// Files anywhere beneath this subdirectory (recursive). + pub files: u64, + /// Bytes anywhere beneath this subdirectory (recursive). + pub bytes: u64, +} + +/// One entry of a directory node: a file or a subdirectory. Serialised as a +/// tagged union so the wire form is self-describing. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum Entry { + /// A file, pinned by hash. + File(FileEntry), + /// A subdirectory, addressed by token. + Dir(SubdirEntry), +} + +impl Entry { + /// The entry's name within its directory. + #[must_use] + pub fn name(&self) -> &str { + match self { + Entry::File(f) => &f.name, + Entry::Dir(d) => &d.name, + } + } +} + +/// The entries directly inside one directory, kept in a stable order so the +/// serialised node is byte-identical for identical input (Merkle stability). +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct DirIndex { + /// Sorted by name (files and subdirs interleaved by name) — see + /// [`DirIndex::from_entries`]. + pub entries: Vec, +} + +impl DirIndex { + /// Build from arbitrary entries, sorting by name so the result is + /// deterministic regardless of the order the caller discovered them in. + #[must_use] + pub fn from_entries(mut entries: Vec) -> Self { + entries.sort_by(|a, b| a.name().cmp(b.name())); + Self { entries } + } + + /// Files directly in this directory (not recursive). + pub fn files(&self) -> impl Iterator { + self.entries.iter().filter_map(|e| match e { + Entry::File(f) => Some(f), + Entry::Dir(_) => None, + }) + } + + /// Subdirectories directly in this directory. + pub fn subdirs(&self) -> impl Iterator { + self.entries.iter().filter_map(|e| match e { + Entry::Dir(d) => Some(d), + Entry::File(_) => None, + }) + } + + /// Total files beneath this node, recursive: local files plus every subdir's + /// recorded subtree total. O(entries), no descent — the subdir totals were + /// computed when those subtrees were minted. + #[must_use] + pub fn total_files(&self) -> u64 { + let local = self.files().count() as u64; + let sub: u64 = self.subdirs().map(|d| d.files).sum(); + local + sub + } + + /// Total bytes beneath this node, recursive (same accounting as + /// [`DirIndex::total_files`]). + #[must_use] + pub fn total_bytes(&self) -> u64 { + let local: u64 = self.files().map(|f| f.size).sum(); + let sub: u64 = self.subdirs().map(|d| d.bytes).sum(); + local + sub + } + + /// Look up a directly-contained entry by name. + #[must_use] + pub fn get(&self, name: &str) -> Option<&Entry> { + self.entries.iter().find(|e| e.name() == name) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn file(name: &str, size: u64) -> Entry { + Entry::File(FileEntry { + name: name.into(), + sha256: format!("{name}-hash"), + size, + content_type: "text/plain".into(), + }) + } + + fn dir(name: &str, files: u64, bytes: u64) -> Entry { + Entry::Dir(SubdirEntry { + name: name.into(), + token: format!("{name}-tok"), + files, + bytes, + }) + } + + #[test] + fn entries_sort_by_name_for_stable_bytes() { + let a = DirIndex::from_entries(vec![file("z.md", 1), file("a.md", 1)]); + let b = DirIndex::from_entries(vec![file("a.md", 1), file("z.md", 1)]); + assert_eq!(a, b); + assert_eq!(a.entries[0].name(), "a.md"); + } + + #[test] + fn totals_roll_up_subdir_recorded_counts() { + let idx = DirIndex::from_entries(vec![file("readme.md", 100), dir("sub", 40, 4_000)]); + assert_eq!(idx.total_files(), 1 + 40); + assert_eq!(idx.total_bytes(), 100 + 4_000); + } + + #[test] + fn files_and_subdirs_partition_entries() { + let idx = DirIndex::from_entries(vec![file("a", 1), dir("b", 1, 1), file("c", 1)]); + assert_eq!(idx.files().count(), 2); + assert_eq!(idx.subdirs().count(), 1); + } + + #[test] + fn serde_round_trip() { + let idx = DirIndex::from_entries(vec![file("a.md", 10), dir("sub", 3, 30)]); + let json = serde_json::to_string(&idx).unwrap(); + let back: DirIndex = serde_json::from_str(&json).unwrap(); + assert_eq!(idx, back); + } +} diff --git a/crates/waggle-tree/src/lib.rs b/crates/waggle-tree/src/lib.rs new file mode 100644 index 0000000..80d4b00 --- /dev/null +++ b/crates/waggle-tree/src/lib.rs @@ -0,0 +1,38 @@ +//! `waggle-tree` — pure, deterministic index and search primitives for directory +//! trees. +//! +//! A `mint --tree` no longer flattens a directory into thousands of individually +//! minted files. It records a **Merkle hierarchy of directory nodes**, each +//! carrying three derived structures this crate defines: +//! +//! * [`dirindex::DirIndex`] — the entries of one directory: files pinned by +//! content hash, subdirectories addressed by their own subtree token, with the +//! size totals that let a node report its weight without a walk. +//! * [`trigram::TrigramIndex`] — an inverted index over a node's file contents, +//! so "grep across the subtree" becomes "look up a few posting lists and +//! confirm the survivors." +//! * [`bloom::Bloom`] — a fixed-size, union-composable summary of *all* trigrams +//! beneath a node, small enough to inline in the node's manifest. It is the +//! prune gate that keeps search over a deep tree sublinear. +//! +//! [`search`] holds the pure decisions that tie them together — whether a node is +//! worth entering, and how confirmed hits are ranked. +//! +//! The crate is **sans-I/O by design**: everything is a pure function of bytes, +//! so the daemon and the wasm edge can both build and query these structures, and +//! so an index blob is content-addressable and byte-stable (invariant I-2). The +//! filesystem walk that produces entries, the blob store that holds file bytes, +//! and the lineage traversal that drives search all live in `waggle-mcp`, using +//! the types defined here. + +#![forbid(unsafe_code)] + +pub mod bloom; +pub mod dirindex; +pub mod search; +pub mod trigram; + +pub use bloom::Bloom; +pub use dirindex::{DirIndex, Entry, FileEntry, SubdirEntry}; +pub use search::{candidates, prune, rank, Hit, Prune}; +pub use trigram::{DocId, Trigram, TrigramIndex, TrigramIndexBuilder}; diff --git a/crates/waggle-tree/src/search.rs b/crates/waggle-tree/src/search.rs new file mode 100644 index 0000000..3bd3713 --- /dev/null +++ b/crates/waggle-tree/src/search.rs @@ -0,0 +1,131 @@ +//! Pure pieces of tree search: a query plan, a hit, and ranking. +//! +//! The *traversal* — descend the lineage, prune with each node's Bloom, load a +//! node's trigram index, grep the survivors — is I/O and lives in `waggle-mcp`. +//! What lives here is everything that must be deterministic and testable in +//! isolation: how a query decides whether a subtree is worth entering, and how +//! the confirmed hits are ordered before they are returned. + +use crate::bloom::Bloom; +use crate::trigram::TrigramIndex; + +/// The decision a node makes about a query before spending any I/O on it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Prune { + /// The node's Bloom proves the query cannot match anywhere beneath it. Skip + /// the whole subtree — no index load, no descent. + Skip, + /// The query might match. The caller should load this node's trigram index, + /// grep its candidate files, and recurse into its subdirectories. + Enter, +} + +/// Ask a node's Bloom summary whether a query is worth entering. This is the +/// prune gate that makes deep trees sublinear: a `Skip` costs one manifest read. +#[must_use] +pub fn prune(summary: &Bloom, query: &str) -> Prune { + if summary.might_contain_all(query) { + Prune::Enter + } else { + Prune::Skip + } +} + +/// The documents in a node worth grepping for a query: the trigram index narrows +/// a node's own files to those that carry every query trigram. The caller then +/// confirms with a real match (the trigram index can admit false positives; grep +/// removes them). +#[must_use] +pub fn candidates(index: &TrigramIndex, query: &str) -> Vec { + index.candidates(query) +} + +/// One confirmed match, with everything a consumer needs to drill in: the file's +/// path (relative to the searched root), the subtree token that owns it, the line +/// and its text, and a match count used for ranking. +#[derive(Clone, Debug, PartialEq)] +pub struct Hit { + /// Path from the searched root, e.g. `design/roadmap/retry.md`. + pub path: String, + /// The token of the subtree node that directly contains the file — the + /// handle a consumer resolves to read more. + pub token: String, + /// 1-based line number of the first match in the file. + pub line: u32, + /// The matching line's text (already budget-trimmed by the caller). + pub text: String, + /// How many times the pattern matched in the file — the ranking signal. + pub matches: u32, +} + +/// Rank hits best-first, then truncate to `limit`. The ordering, in priority: +/// +/// 1. **more matches first** — a file that mentions the pattern ten times is more +/// likely the one you want than a file that mentions it once; +/// 2. **shallower paths first** — a top-level file usually outranks something +/// buried deep, when match counts tie; +/// 3. **path, lexicographically** — a stable final tie-break so results are +/// deterministic run to run. +#[must_use] +pub fn rank(mut hits: Vec, limit: usize) -> Vec { + hits.sort_by(|a, b| { + b.matches + .cmp(&a.matches) + .then_with(|| depth(&a.path).cmp(&depth(&b.path))) + .then_with(|| a.path.cmp(&b.path)) + }); + hits.truncate(limit); + hits +} + +/// Path depth = number of `/` separators. `retry.md` is 0, `a/b/retry.md` is 2. +fn depth(path: &str) -> usize { + path.bytes().filter(|&b| b == b'/').count() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hit(path: &str, matches: u32) -> Hit { + Hit { + path: path.into(), + token: "t".into(), + line: 1, + text: "x".into(), + matches, + } + } + + #[test] + fn prune_skips_when_bloom_excludes_query() { + let mut b = Bloom::new(); + b.insert_text("retry budget"); + assert_eq!(prune(&b, "retry"), Prune::Enter); + assert_eq!(prune(&b, "wholly-absent-xyz"), Prune::Skip); + } + + #[test] + fn rank_orders_by_match_count_then_depth_then_path() { + let ranked = rank( + vec![hit("deep/a/b/c.md", 1), hit("top.md", 1), hit("hot.md", 9)], + 10, + ); + assert_eq!(ranked[0].path, "hot.md"); // most matches + assert_eq!(ranked[1].path, "top.md"); // tie on matches, shallower + assert_eq!(ranked[2].path, "deep/a/b/c.md"); + } + + #[test] + fn rank_truncates_to_limit() { + let hits: Vec = (0..20).map(|i| hit(&format!("f{i}.md"), 1)).collect(); + assert_eq!(rank(hits, 5).len(), 5); + } + + #[test] + fn rank_is_deterministic_on_full_ties() { + let mk = || vec![hit("b.md", 1), hit("a.md", 1), hit("c.md", 1)]; + assert_eq!(rank(mk(), 10), rank(mk(), 10)); + assert_eq!(rank(mk(), 10)[0].path, "a.md"); + } +} diff --git a/crates/waggle-tree/src/trigram.rs b/crates/waggle-tree/src/trigram.rs new file mode 100644 index 0000000..555db80 --- /dev/null +++ b/crates/waggle-tree/src/trigram.rs @@ -0,0 +1,299 @@ +//! Trigrams and the inverted index built from them. +//! +//! A [`Trigram`] is a 3-byte, case-folded shingle of text — the atom of the +//! search index. A pattern can only match a file where *every* trigram of the +//! pattern is present, so a trigram index turns "grep this pattern across N +//! files" into "look up a few short posting lists and confirm the survivors." +//! This is the technique code-search engines (Zoekt, ripgrep's `--pre`, Google +//! Code Search) use to stay sublinear. +//! +//! Everything here is a pure function of the bytes: the same content yields the +//! same index, so an index blob is content-addressable and byte-stable (I-2). +//! Case folding is ASCII-only and deterministic; a query is folded the same way, +//! so matching is case-insensitive by construction. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +/// A case-folded 3-byte shingle. Ordered and hashable so it can key a map. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub struct Trigram([u8; 3]); + +impl Trigram { + /// Build directly from three raw bytes (no folding). Mainly for tests and + /// callers that already hold folded bytes. + #[must_use] + pub fn from_bytes(b: [u8; 3]) -> Self { + Self(b) + } + + /// The underlying bytes — the [`crate::bloom`] hasher consumes these. + #[must_use] + pub fn bytes(self) -> [u8; 3] { + self.0 + } + + /// Every trigram of `text`, in order, over its **case-folded** bytes. Text + /// shorter than three bytes yields nothing (nothing to shingle). Callers get + /// an iterator so a large document never materialises a trigram `Vec`. + pub fn all(text: &str) -> impl Iterator + '_ { + let folded: Vec = text.bytes().map(fold).collect(); + (0..folded.len().saturating_sub(2)) + .map(move |i| Trigram([folded[i], folded[i + 1], folded[i + 2]])) + } +} + +/// ASCII lowercasing — the one folding rule, applied identically to indexed +/// content and to queries. Non-ASCII bytes pass through unchanged. +fn fold(b: u8) -> u8 { + b.to_ascii_lowercase() +} + +/// Which documents a query's trigrams point at. `File(id)` indexes into the +/// caller's own document list (a directory node's file entries). +pub type DocId = u32; + +/// An inverted index: trigram → the sorted, de-duplicated documents that contain +/// it. Built with [`TrigramIndex::builder`]; queried with +/// [`TrigramIndex::candidates`]. +#[derive(Clone, Default, Serialize, Deserialize)] +pub struct TrigramIndex { + /// Serialised as a plain map so an index blob is inspectable. Keys are the + /// three folded bytes as a lossy string is *not* safe (bytes may not be + /// UTF-8), so we key by the raw triple via a helper representation. + #[serde(with = "postings_serde")] + postings: BTreeMap>, + /// Number of documents the index was built over — lets a reader sanity-check + /// candidate ids without holding the document list. + docs: u32, +} + +impl TrigramIndex { + /// Start building. Feed each document's text with + /// [`TrigramIndexBuilder::add`] in id order. + #[must_use] + pub fn builder() -> TrigramIndexBuilder { + TrigramIndexBuilder { + postings: BTreeMap::new(), + next: 0, + } + } + + /// The documents that *could* match `query`: those carrying **all** of the + /// query's trigrams (posting-list intersection). Empty query trigrams (text + /// under three bytes) returns every document, because a trigram index cannot + /// narrow a pattern too short to shingle — the caller falls back to a full + /// grep of the small candidate set. + #[must_use] + pub fn candidates(&self, query: &str) -> Vec { + let grams: Vec = Trigram::all(query).collect(); + if grams.is_empty() { + return (0..self.docs).collect(); + } + // Intersect posting lists, smallest first so the running set only shrinks. + let mut lists: Vec<&Vec> = Vec::with_capacity(grams.len()); + for g in &grams { + match self.postings.get(g) { + Some(list) => lists.push(list), + None => return Vec::new(), // a trigram absent everywhere ⇒ no match + } + } + lists.sort_by_key(|l| l.len()); + let mut acc = lists[0].clone(); + for list in &lists[1..] { + acc = intersect_sorted(&acc, list); + if acc.is_empty() { + break; + } + } + acc + } + + /// Documents indexed. + #[must_use] + pub fn doc_count(&self) -> u32 { + self.docs + } + + /// Distinct trigrams — an index-size gauge. + #[must_use] + pub fn trigram_count(&self) -> usize { + self.postings.len() + } +} + +/// Accumulates postings one document at a time. Document ids are assigned in the +/// order documents are added, starting at 0. +pub struct TrigramIndexBuilder { + postings: BTreeMap>, + next: DocId, +} + +impl TrigramIndexBuilder { + /// Index one document's text and return its assigned id. + pub fn add(&mut self, text: &str) -> DocId { + let id = self.next; + // De-dup trigrams within a doc so a posting list holds each id once. + let mut seen: BTreeMap = BTreeMap::new(); + for tri in Trigram::all(text) { + if seen.insert(tri, ()).is_none() { + self.postings.entry(tri).or_default().push(id); + } + } + self.next += 1; + id + } + + /// Finish. Posting lists are already sorted, because ids are added in order. + #[must_use] + pub fn build(self) -> TrigramIndex { + TrigramIndex { + postings: self.postings, + docs: self.next, + } + } +} + +/// Intersection of two ascending, de-duplicated id lists. +fn intersect_sorted(a: &[DocId], b: &[DocId]) -> Vec { + let (mut i, mut j) = (0, 0); + let mut out = Vec::new(); + while i < a.len() && j < b.len() { + match a[i].cmp(&b[j]) { + std::cmp::Ordering::Less => i += 1, + std::cmp::Ordering::Greater => j += 1, + std::cmp::Ordering::Equal => { + out.push(a[i]); + i += 1; + j += 1; + } + } + } + out +} + +/// Serialise the posting map with hex trigram keys, so the wire form is valid +/// JSON regardless of whether the folded bytes are printable. +mod postings_serde { + use super::{DocId, Trigram}; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use std::collections::BTreeMap; + + pub fn serialize( + map: &BTreeMap>, + s: S, + ) -> Result { + let hexed: BTreeMap> = + map.iter().map(|(k, v)| (hex3(k.bytes()), v)).collect(); + hexed.serialize(s) + } + + pub fn deserialize<'de, D: Deserializer<'de>>( + d: D, + ) -> Result>, D::Error> { + let hexed: BTreeMap> = BTreeMap::deserialize(d)?; + let mut out = BTreeMap::new(); + for (k, v) in hexed { + let b = unhex3(&k).map_err(serde::de::Error::custom)?; + out.insert(Trigram::from_bytes(b), v); + } + Ok(out) + } + + fn hex3(b: [u8; 3]) -> String { + let mut s = String::with_capacity(6); + for byte in b { + s.push(char::from_digit((byte >> 4) as u32, 16).unwrap()); + s.push(char::from_digit((byte & 0xf) as u32, 16).unwrap()); + } + s + } + + fn unhex3(s: &str) -> Result<[u8; 3], String> { + if s.len() != 6 { + return Err(format!( + "trigram key: expected 6 hex chars, got {}", + s.len() + )); + } + let mut b = [0u8; 3]; + for (i, slot) in b.iter_mut().enumerate() { + *slot = u8::from_str_radix(&s[i * 2..i * 2 + 2], 16) + .map_err(|e| format!("trigram key: {e}"))?; + } + Ok(b) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shingles_are_case_folded() { + let up: Vec<_> = Trigram::all("ABC").collect(); + let lo: Vec<_> = Trigram::all("abc").collect(); + assert_eq!(up, lo); + assert_eq!(up.len(), 1); + } + + #[test] + fn short_text_yields_no_trigrams() { + assert_eq!(Trigram::all("ab").count(), 0); + assert_eq!(Trigram::all("").count(), 0); + } + + #[test] + fn candidates_intersect_all_query_trigrams() { + let mut b = TrigramIndex::builder(); + let d0 = b.add("the retry budget is three"); // has "retry" + let d1 = b.add("the escalation policy ceiling"); // no "retry" + let _d2 = b.add("retry retry retry"); + let idx = b.build(); + let mut hits = idx.candidates("retry"); + hits.sort_unstable(); + assert_eq!(hits, vec![d0, 2]); + assert!(!hits.contains(&d1)); + } + + #[test] + fn absent_pattern_returns_nothing() { + let mut b = TrigramIndex::builder(); + b.add("alpha beta gamma"); + let idx = b.build(); + assert!(idx.candidates("zzzzzz").is_empty()); + } + + #[test] + fn too_short_query_returns_all_docs() { + let mut b = TrigramIndex::builder(); + b.add("one"); + b.add("two"); + let idx = b.build(); + assert_eq!(idx.candidates("x"), vec![0, 1]); + } + + #[test] + fn serde_round_trip_preserves_queries() { + let mut b = TrigramIndex::builder(); + b.add("runbook 07 violates the ceiling"); + b.add("runbook 03 is within budget"); + let idx = b.build(); + let json = serde_json::to_string(&idx).unwrap(); + let back: TrigramIndex = serde_json::from_str(&json).unwrap(); + assert_eq!(idx.candidates("violates"), back.candidates("violates")); + assert_eq!(back.doc_count(), 2); + } + + #[test] + fn deterministic_index_bytes() { + let build = || { + let mut b = TrigramIndex::builder(); + b.add("the escalation policy"); + b.add("a retry budget of three"); + serde_json::to_string(&b.build()).unwrap() + }; + assert_eq!(build(), build()); + } +} From 078e649b00319a5e1f5a55d0dd2deff0e4db5e5f Mon Sep 17 00:00:00 2001 From: Chetan Conikee Date: Mon, 13 Jul 2026 12:33:16 -0700 Subject: [PATCH 02/12] =?UTF-8?q?core:=20TreeNode=20manifest=20field=20?= =?UTF-8?q?=E2=80=94=20a=20directory=20node=20of=20a=20tree=20mint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Additive, signed-core field `tree: Option` (skip-if-none, so non-tree manifests keep their exact bytes — same pattern as `extraction`). A TreeNode carries the node's content-addressed directory index, an optional trigram-index pointer, an inlined hex Bloom summary of the subtree, and recursive files/bytes totals. The bloom stays a hex string here rather than a typed `waggle-tree::Bloom` so waggle-core keeps its dependency-free leaf position; waggle-mcp parses it. MintSpec gains a `.tree(TreeNode)` builder. --- crates/waggle-core/src/lib.rs | 2 +- crates/waggle-core/src/manifest.rs | 35 ++++++++++++++++++++++++++++++ crates/waggle-core/src/mint.rs | 12 ++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/crates/waggle-core/src/lib.rs b/crates/waggle-core/src/lib.rs index ab88a86..033864d 100644 --- a/crates/waggle-core/src/lib.rs +++ b/crates/waggle-core/src/lib.rs @@ -65,7 +65,7 @@ pub use fold::{ pub use log::{Change, LogRecord}; pub use manifest::{ apply_change, AttributionManifest, Constraint, Disposition, Extraction, MatchExpr, ModalitySet, - Posture, SignatureBlock, Variant, VariantBody, MANIFEST_SCHEMA_VERSION, + Posture, SignatureBlock, TreeNode, Variant, VariantBody, MANIFEST_SCHEMA_VERSION, }; pub use matcher::{select_variant, Selected}; pub use mint::{mint, MintError, MintOptions, MintSpec}; diff --git a/crates/waggle-core/src/manifest.rs b/crates/waggle-core/src/manifest.rs index 8c6dbcc..b9d3b41 100644 --- a/crates/waggle-core/src/manifest.rs +++ b/crates/waggle-core/src/manifest.rs @@ -232,6 +232,35 @@ pub struct Extraction { pub deterministic: bool, } +/// A directory node of a tree mint (design doc: tree-scale). When a token names +/// a directory minted `--tree`, it carries one of these instead of a flat list of +/// per-file child tokens: the directory's **index** (files pinned by hash, subdirs +/// addressed by their own subtree token), the **trigram index** over its files, +/// and a **Bloom summary** of every trigram beneath it, inlined so a search can +/// prune the subtree from the manifest alone. Files pin their bytes eagerly (so a +/// deleted file still reads — C-1); a per-file token is minted only lazily, when a +/// consumer needs a standalone reference to one file. +/// +/// The `bloom` is the lowercase-hex wire form of a fixed-size filter (parsed into +/// the typed structure by the consumer, `waggle-tree::Bloom`); keeping it a string +/// here preserves this crate's dependency-free leaf position. Immutable core — +/// signed; absent for non-tree mints, so their manifests keep their exact bytes. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TreeNode { + /// Content-addressed directory index for THIS node (its direct entries). + pub index: MediaRef, + /// Content-addressed trigram index over this node's own files; absent when the + /// node has no indexable text of its own (e.g. only subdirectories). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trigram: Option, + /// Hex Bloom summary of every trigram in this subtree — inlined for prune. + pub bloom: String, + /// Files anywhere beneath this node (recursive). + pub files: u64, + /// Bytes anywhere beneath this node (recursive). + pub bytes: u64, +} + /// One projection of the artifact for a class of consumers. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Variant { @@ -322,6 +351,12 @@ pub struct AttributionManifest { /// their exact canonical bytes. #[serde(default, skip_serializing_if = "Option::is_none")] pub extraction: Option, + /// A directory node of a tree mint (design doc: tree-scale): the node's index, + /// trigram index, and inlined Bloom summary. Present when this token is a + /// directory minted `--tree`; absent otherwise, so non-tree manifests keep + /// their exact bytes. Immutable core — signed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tree: Option, /// Author signature over the immutable core (CP-11); set at mint by /// hosts that hold an identity. NOT itself part of the signed bytes. #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/crates/waggle-core/src/mint.rs b/crates/waggle-core/src/mint.rs index 102d669..4242a6c 100644 --- a/crates/waggle-core/src/mint.rs +++ b/crates/waggle-core/src/mint.rs @@ -63,6 +63,7 @@ pub struct MintSpec { contract: Option, outline: Option, extraction: Option, + tree: Option, labels: std::collections::BTreeMap, ttl_ms: Option, } @@ -83,6 +84,7 @@ impl MintSpec { contract: None, outline: None, extraction: None, + tree: None, labels: std::collections::BTreeMap::new(), ttl_ms: None, } @@ -180,6 +182,15 @@ impl MintSpec { self } + /// Attach a tree directory node (design doc: tree-scale): the index, trigram + /// index, and Bloom summary for a directory minted `--tree`. Signed with the + /// core. + #[must_use] + pub fn tree(mut self, tree: crate::TreeNode) -> Self { + self.tree = Some(tree); + self + } + /// Expire the token `ttl_ms` after mint. #[must_use] pub fn ttl_ms(mut self, ttl_ms: u64) -> Self { @@ -229,6 +240,7 @@ pub fn mint( contract: spec.contract, outline: spec.outline, extraction: spec.extraction, + tree: spec.tree, signature: None, // hosts with an identity sign after mint (trust) variants, version: 1, From b1e3401818867595520200240ddf30ad1b118b25 Mon Sep 17 00:00:00 2001 From: Chetan Conikee Date: Mon, 13 Jul 2026 13:06:30 -0700 Subject: [PATCH 03/12] =?UTF-8?q?mcp:=20indexed=20tree=20mint=20=E2=80=94?= =?UTF-8?q?=20cap=20lifted,=20one=20command=20for=20a=20big=20corpus=20(Ax?= =?UTF-8?q?is=20A+B)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mint --tree now builds a Merkle hierarchy of directory NODES instead of a flat list of per-file tokens. Two phases (tree_mint.rs): build_node (bottom-up): pin every file's bytes eagerly (content-addressed, so a deleted file still reads — C-1 holds), build each directory's DirIndex, trigram index, and Bloom summary (union of children's + local files), pre-generate the node token. Enforces a byte budget (max-bytes, default 256 MB) so nobody pins a giant media folder by accident. mint_nodes (top-down): mint one signed manifest per directory, parent before child so the parent exists when a child links to it. A file is an index entry, not a token — so the old 200-FILE cap becomes a bound on directory nodes (few). A 1,527-file corpus mints in one call. A per-file token is minted only lazily, when a consumer needs a standalone reference (follow-up). mint routes tree builds early (no orphan root token). The old flat mint_tree / mint_children / collect_files / DENIED_DIRS are deleted. read/search/coverage still read the old shape and are rewired next. tree_mint.rs 406 lines; clippy clean. --- Cargo.lock | 1 + crates/waggle-core/src/mint.rs | 17 +- crates/waggle-mcp/Cargo.toml | 1 + crates/waggle-mcp/src/handlers.rs | 23 +- crates/waggle-mcp/src/lib.rs | 1 + crates/waggle-mcp/src/lineage.rs | 150 ----------- crates/waggle-mcp/src/tree_mint.rs | 406 +++++++++++++++++++++++++++++ crates/waggle-tree/src/bloom.rs | 19 ++ 8 files changed, 462 insertions(+), 156 deletions(-) create mode 100644 crates/waggle-mcp/src/tree_mint.rs diff --git a/Cargo.lock b/Cargo.lock index f69a807..f6e5e5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2254,6 +2254,7 @@ dependencies = [ "waggle-social", "waggle-store", "waggle-store-sqlite", + "waggle-tree", ] [[package]] diff --git a/crates/waggle-core/src/mint.rs b/crates/waggle-core/src/mint.rs index 4242a6c..14d349a 100644 --- a/crates/waggle-core/src/mint.rs +++ b/crates/waggle-core/src/mint.rs @@ -64,6 +64,7 @@ pub struct MintSpec { outline: Option, extraction: Option, tree: Option, + token: Option, labels: std::collections::BTreeMap, ttl_ms: Option, } @@ -85,6 +86,7 @@ impl MintSpec { outline: None, extraction: None, tree: None, + token: None, labels: std::collections::BTreeMap::new(), ttl_ms: None, } @@ -191,6 +193,16 @@ impl MintSpec { self } + /// Mint under a **pre-generated** token instead of generating one. A tree mint + /// needs each directory node's token *before* its children (so children can + /// point their `parent` link at it) while the node's Bloom is only known + /// *after* its children — pre-generating the token breaks that cycle. + #[must_use] + pub fn with_token(mut self, token: Token) -> Self { + self.token = Some(token); + self + } + /// Expire the token `ttl_ms` after mint. #[must_use] pub fn ttl_ms(mut self, ttl_ms: u64) -> Self { @@ -228,7 +240,10 @@ pub fn mint( let token_len = if spec.private { 16 } else { opts.token_len }; let manifest = AttributionManifest { schema: MANIFEST_SCHEMA_VERSION, - token: Token::generate(token_len, entropy)?, + token: match spec.token { + Some(t) => t, + None => Token::generate(token_len, entropy)?, + }, target: spec.target, sharer: spec.sharer, channel: spec.channel, diff --git a/crates/waggle-mcp/Cargo.toml b/crates/waggle-mcp/Cargo.toml index 28322c4..18c4695 100644 --- a/crates/waggle-mcp/Cargo.toml +++ b/crates/waggle-mcp/Cargo.toml @@ -14,6 +14,7 @@ homepage.workspace = true [dependencies] waggle-core = { workspace = true } +waggle-tree = { workspace = true } waggle-ops = { workspace = true } waggle-store = { path = "../waggle-store", version = "0.4.0" } serde = { workspace = true } diff --git a/crates/waggle-mcp/src/handlers.rs b/crates/waggle-mcp/src/handlers.rs index d1fb9c2..5774701 100644 --- a/crates/waggle-mcp/src/handlers.rs +++ b/crates/waggle-mcp/src/handlers.rs @@ -181,6 +181,24 @@ impl Handler { }, None => Channel::subagent_general(), }; + // A tree mint builds the whole hierarchy under its own node tokens, so it + // routes here directly — minting a plain root first would orphan a token. + let is_tree = args.get("tree").and_then(Value::as_bool).unwrap_or(false) + || arg_str(args, "tree") == Some("true"); + if is_tree { + let parent = arg_str(args, "parent").and_then(|p| Token::parse(p).ok()); + return self + .mint_tree_indexed( + target.as_str(), + sharer.as_str(), + channel.as_str(), + parent, + crate::tree_mint::tree_budget(args), + now, + entropy, + ) + .await; + } let spec = MintSpec::new(target, sharer, channel); let spec = match self.mint_extras(spec, args).await { Ok(s) => s, @@ -209,11 +227,6 @@ impl Handler { match receipt { Ok(Appended::Minted { view, replayed }) => { let token = view.manifest.token; - let tree = args.get("tree").and_then(Value::as_bool).unwrap_or(false) - || arg_str(args, "tree") == Some("true"); - if tree { - return self.mint_tree(&view, now, entropy).await; - } let next = crate::lineage::mint_next(token.as_str(), view.manifest.target.as_str()); Envelope::ok( json!({ diff --git a/crates/waggle-mcp/src/lib.rs b/crates/waggle-mcp/src/lib.rs index efbeb9c..8393be2 100644 --- a/crates/waggle-mcp/src/lib.rs +++ b/crates/waggle-mcp/src/lib.rs @@ -33,6 +33,7 @@ mod record; mod resources; mod rpc; mod tree; +mod tree_mint; pub use envelope::{validate_next, Envelope, NextCall, Stats}; pub use handlers::Handler; diff --git a/crates/waggle-mcp/src/lineage.rs b/crates/waggle-mcp/src/lineage.rs index 5cb75cf..5e7d4a9 100644 --- a/crates/waggle-mcp/src/lineage.rs +++ b/crates/waggle-mcp/src/lineage.rs @@ -5,12 +5,10 @@ //! beneath it — one revocation for the whole tree. use serde_json::{json, Map, Value}; -use waggle_core::Timestamp; use waggle_store::{BlobSink, Store}; use crate::envelope::{Envelope, NextCall, Stats}; use crate::handlers::{parse_token_arg, store_err, Handler}; -use crate::map::handoff_line; /// Was this folder minted `--require files:all`? Then the delegation needs the /// WHOLE tree, and `coverage` answers with a verdict, not merely a fact. @@ -37,114 +35,6 @@ impl Handler { (count("read") + count("resolve") > 0, count("run") > 0) } - /// `mint --tree`: the folder pattern as one call. The root token is - /// already minted; every file inside (recursive, sorted, dotfiles - /// skipped, capped) becomes a snapshot-pinned CHILD — one revocation - /// covers the tree, and the root's funnel rolls the children up. - pub(crate) async fn mint_tree( - &self, - root: &waggle_store::ManifestView, - now: Timestamp, - entropy: &mut E, - ) -> Envelope - where - E: FnMut(&mut [u8]) -> Result<(), waggle_core::EntropyError>, - { - const CAP: usize = 200; - let token = root.manifest.token; - let target = root.manifest.target.as_str(); - let Some(dir) = crate::content_handlers::local_path(target) else { - return Envelope::err( - format!("tree: `{target}` is not a local directory — --tree walks the filesystem"), - vec![], - ); - }; - let mut files = Vec::new(); - collect_files(std::path::Path::new(&dir), &mut files); - files.sort(); - if files.len() > CAP { - return Envelope::err( - format!( - "tree: {} files exceeds the {CAP}-file cap — mint subfolders as their own trees", - files.len() - ), - vec![], - ); - } - let children = match self.mint_children(root, files, now, entropy).await { - Ok(c) => c, - Err(e) => return e, - }; - let child_count = children.len(); - Envelope::ok( - json!({ - "token": token.as_str(), - "handoff": handoff_line(token.as_str()), - "children": children, - "tree": { "files": child_count }, - }), - vec![ - NextCall { - tool: "funnel".into(), - args: json!({ "token": token.as_str() }), - why: "the root's funnel rolls up every child".into(), - }, - NextCall { - tool: "mutate".into(), - args: json!({ "token": token.as_str(), "change": "revoke", "expected-version": 1 }), - why: "one revocation tombstones the whole tree".into(), - }, - ], - ) - .with_stats(Stats { - records: Some(1 + child_count as u64), - seq: Some(0), - }) - } - - /// The tree's children: each file snapshot-minted under the root. - async fn mint_children( - &self, - root: &waggle_store::ManifestView, - files: Vec, - now: Timestamp, - entropy: &mut E, - ) -> Result, Envelope> - where - E: FnMut(&mut [u8]) -> Result<(), waggle_core::EntropyError>, - { - let token = root.manifest.token; - let mut children = Vec::new(); - for file in files { - let child_args = json!({ - "target": format!("file://{}", file.display()), - "parent": token.as_str(), - "snapshot": true, - "sharer": root.manifest.sharer.as_str(), - "channel": root.manifest.channel.as_str(), - }); - let child = Box::pin(self.mint( - child_args.as_object().expect("literal object"), - now, - entropy, - )) - .await; - match child.hint { - None => children.push(json!({ - "token": child.result["token"], - "target": format!("file://{}", file.display()), - })), - Some(hint) => { - return Err(Envelope::err( - format!("tree: {}: {hint}", file.display()), - vec![], - )) - } - } - } - Ok(children) - } - /// Walk the parent chain: the earliest revocation timestamp among /// ancestors, if any (lineage cascade — revoking a folder/mission /// token tombstones everything minted under it). Depth-capped. @@ -196,46 +86,6 @@ pub(crate) fn mint_next(token: &str, target: &str) -> Vec { next } -/// Directories no source-tree handoff means to include (doc `20 §5.7`): -/// generated and vendored trees that would blow the file cap or snapshot -/// junk. Full `.gitignore` fidelity is a planned follow-up; the deny-list -/// covers the trees that actually bite. -const DENIED_DIRS: &[&str] = &[ - "node_modules", - "target", - "dist", - "build", - "out", - "vendor", - "__pycache__", - ".venv", - "venv", -]; - -/// Recursive file collection for `mint --tree`: dotfiles and dot-dirs -/// skipped, generated/vendored dirs denied, symlinks not followed (walk -/// what IS the folder). -fn collect_files(dir: &std::path::Path, out: &mut Vec) { - let Ok(entries) = std::fs::read_dir(dir) else { - return; - }; - for entry in entries.flatten() { - let path = entry.path(); - let name = path.file_name().and_then(|n| n.to_str()).unwrap_or(""); - if name.starts_with('.') { - continue; - } - let Ok(meta) = entry.metadata() else { continue }; - if meta.is_dir() { - if !DENIED_DIRS.contains(&name) { - collect_files(&path, out); - } - } else if meta.is_file() { - out.push(path); - } - } -} - impl Handler { /// `coverage`: the folder handoff's proof of reading (see the /// catalog description). BFS over descendants; per file, the diff --git a/crates/waggle-mcp/src/tree_mint.rs b/crates/waggle-mcp/src/tree_mint.rs new file mode 100644 index 0000000..7071117 --- /dev/null +++ b/crates/waggle-mcp/src/tree_mint.rs @@ -0,0 +1,406 @@ +//! Indexed tree minting (design doc: tree-scale). +//! +//! A `mint --tree` records a directory as a **Merkle hierarchy of directory +//! nodes**, one signed manifest per directory (not per file). Each node carries +//! its [`DirIndex`], a [`TrigramIndex`] over its own files, and a [`Bloom`] +//! summary of every trigram beneath it — the structures `waggle-tree` defines. +//! +//! Two properties this module is responsible for upholding: +//! +//! * **Content eager, tokens lazy (C-1 preserved).** Every file's bytes are pinned +//! into the blob store here, so a deleted file still reads. But a file is *not* +//! given its own token/manifest — it is an entry in its directory node's index. +//! The cap that used to bound files now bounds only directory nodes (few), so a +//! corpus of thousands of files mints in one call. +//! * **Bloom composes up, tokens flow down.** A node's Bloom is the union of its +//! children's, which needs the children first; a child's `parent` link needs the +//! node's token first. The cycle is broken by **pre-generating** each node's +//! token ([`MintSpec::with_token`]) before descending. + +use serde_json::{json, Map, Value}; +use waggle_core::{MintOptions, MintSpec, Timestamp, Token}; +use waggle_store::{AppendIntent, Appended, BlobSink, MintNonce, Store}; +use waggle_tree::{Bloom, DirIndex, Entry, FileEntry, SubdirEntry, TrigramIndex}; + +use crate::content_handlers::{local_path, read_capped}; +use crate::envelope::Envelope; +use crate::handlers::{infer_content_type, Handler}; + +/// Content types for the two index blobs a node stores. +const DIRINDEX_CT: &str = "application/waggle-dirindex+json"; +const TRIGRAM_CT: &str = "application/waggle-trigram+json"; + +/// Default ceiling on total bytes eagerly snapshotted by one tree mint, so nobody +/// accidentally pins a huge media folder. Override with `max-bytes` on the mint. +const DEFAULT_BUDGET: u64 = 256 * 1024 * 1024; + +/// A fully computed directory node, ready to mint. Built bottom-up in phase one +/// (blobs pinned, indexes built, token pre-generated) so that phase two can mint +/// manifests **top-down** — a child's `parent` link needs the parent's manifest to +/// already exist in the store, which post-order minting cannot provide. +struct NodeData { + token: Token, + /// `file://` URL of this directory. + dir_url: String, + /// The `tree` field this node's manifest will carry. + tree_node: waggle_core::TreeNode, + /// Subtree Bloom — union of children's plus local files (for the parent). + bloom: Bloom, + files: u64, + bytes: u64, + /// Child directory nodes, in name order. + children: Vec, +} + +/// One file gathered during a directory walk, before the node is assembled. +struct FileRec { + name: String, + sha256: String, + size: u64, + content_type: String, + /// Text fed to the trigram index and Bloom — the file's own bytes if text, a + /// deterministic extraction for PDF/HTML, empty for opaque media. + index_text: String, +} + +impl Handler { + /// Mint a directory as an indexed Merkle tree. Returns the root token's + /// envelope. Called by the `mint --tree` path in place of the old flat + /// per-file minting. + pub(crate) async fn mint_tree_indexed( + &self, + dir: &str, + sharer: &str, + channel: &str, + parent: Option, + budget: u64, + now: Timestamp, + entropy: &mut E, + ) -> Envelope + where + E: FnMut(&mut [u8]) -> Result<(), waggle_core::EntropyError>, + { + let path = match local_path(dir) { + Some(p) => p, + None => { + return Envelope::err( + format!("tree: `{dir}` is not a local directory — --tree walks the filesystem"), + vec![], + ) + } + }; + // Phase one: compute the whole tree bottom-up — pin blobs, build indexes, + // pre-generate tokens. No manifests yet. + let mut spent = 0u64; + let root = match Box::pin(self.build_node( + std::path::Path::new(&path), + budget, + &mut spent, + entropy, + )) + .await + { + Ok(n) => n, + Err(e) => return e, + }; + let (files, bytes, token) = (root.files, root.bytes, root.token); + // Phase two: mint manifests top-down, so each parent exists before its + // children link to it. + if let Err(e) = + Box::pin(self.mint_nodes(&root, sharer, channel, parent, now, entropy)).await + { + return e; + } + Envelope::ok( + json!({ + "token": token.as_str(), + "handoff": crate::map::handoff_line(token.as_str()), + "tree": { "files": files, "bytes": bytes }, + }), + vec![ + crate::envelope::NextCall { + tool: "search".into(), + args: json!({ "token": token.as_str(), "pattern": "" }), + why: "one search spans the whole tree — pruned and ranked".into(), + }, + crate::envelope::NextCall { + tool: "read".into(), + args: json!({ "token": token.as_str() }), + why: "the directory's table of contents".into(), + }, + ], + ) + } + + /// Phase one — compute one directory node and its whole subtree, bottom-up: + /// pin every file's bytes, build the node's index/trigram/Bloom, pre-generate + /// its token, and recurse. Mints nothing; returns a [`NodeData`] tree. + async fn build_node( + &self, + dir: &std::path::Path, + budget: u64, + spent: &mut u64, + entropy: &mut E, + ) -> Result + where + E: FnMut(&mut [u8]) -> Result<(), waggle_core::EntropyError>, + { + let token = + Token::generate(8, entropy).map_err(|e| Envelope::err(e.to_string(), vec![]))?; + + let mut files: Vec = Vec::new(); + let mut children: Vec = Vec::new(); + let mut bloom = Bloom::new(); + let (mut total_files, mut total_bytes) = (0u64, 0u64); + + for entry in read_sorted(dir)? { + let name = entry.file_name().to_string_lossy().into_owned(); + if is_ignored(&name) { + continue; + } + if entry.path().is_dir() { + let sub = Box::pin(self.build_node(&entry.path(), budget, spent, entropy)).await?; + bloom.union(&sub.bloom); + total_files += sub.files; + total_bytes += sub.bytes; + children.push(sub); + } else if entry.path().is_file() { + let rec = self + .snapshot_file(&entry.path(), &name, budget, spent) + .await?; + if !rec.index_text.is_empty() { + bloom.insert_text(&rec.index_text); + } + total_files += 1; + total_bytes += rec.size; + files.push(rec); + } + } + + // A name-sorted index; the trigram index's doc ids align with its file + // order, so a search candidate maps straight back to a file entry. + files.sort_by(|a, b| a.name.cmp(&b.name)); + let mut trigram = TrigramIndex::builder(); + let mut any_text = false; + for f in &files { + trigram.add(&f.index_text); + any_text |= !f.index_text.is_empty(); + } + + let mut entries: Vec = files + .iter() + .map(|f| { + Entry::File(FileEntry { + name: f.name.clone(), + sha256: f.sha256.clone(), + size: f.size, + content_type: f.content_type.clone(), + }) + }) + .collect(); + for sub in &children { + entries.push(Entry::Dir(SubdirEntry { + name: dir_name(&sub.dir_url), + token: sub.token.as_str().to_owned(), + files: sub.files, + bytes: sub.bytes, + })); + } + + let index_ref = self + .put_json(&DirIndex::from_entries(entries), DIRINDEX_CT) + .await?; + let trigram_ref = if any_text { + Some(self.put_json(&trigram.build(), TRIGRAM_CT).await?) + } else { + None + }; + + Ok(NodeData { + token, + dir_url: format!("file://{}", dir.display()), + tree_node: waggle_core::TreeNode { + index: index_ref, + trigram: trigram_ref, + bloom: bloom.to_hex(), + files: total_files, + bytes: total_bytes, + }, + bloom, + files: total_files, + bytes: total_bytes, + children, + }) + } + + /// Phase two — mint manifests top-down, so a parent is in the store before its + /// children link to it. + async fn mint_nodes( + &self, + node: &NodeData, + sharer: &str, + channel: &str, + parent: Option, + now: Timestamp, + entropy: &mut E, + ) -> Result<(), Envelope> + where + E: FnMut(&mut [u8]) -> Result<(), waggle_core::EntropyError>, + { + self.mint_node_manifest(node, sharer, channel, parent, now, entropy) + .await?; + for child in &node.children { + Box::pin(self.mint_nodes(child, sharer, channel, Some(node.token), now, entropy)) + .await?; + } + Ok(()) + } + + /// Pin one file's bytes eagerly and gather its index record. Enforces the byte + /// budget as it goes so a runaway corpus fails fast, not after gigabytes. + async fn snapshot_file( + &self, + path: &std::path::Path, + name: &str, + budget: u64, + spent: &mut u64, + ) -> Result { + let bytes = read_capped(&path.to_string_lossy())?; + *spent += bytes.len() as u64; + if *spent > budget { + return Err(Envelope::err( + format!( + "tree: snapshot exceeds the {} MB budget at `{}` — mint a subfolder, or raise max-bytes", + budget / (1024 * 1024), + path.display() + ), + vec![], + )); + } + let content_type = infer_content_type(&path.to_string_lossy()).to_owned(); + let media = self + .blobs + .put(&bytes, &content_type) + .await + .map_err(|e| Envelope::err(e.to_string(), vec![]))?; + let index_text = index_text(&content_type, &bytes); + Ok(FileRec { + name: name.to_owned(), + sha256: media.sha256.as_str().to_owned(), + size: bytes.len() as u64, + content_type, + index_text, + }) + } + + /// Mint (and sign, and persist) one directory node's manifest under its + /// pre-generated token, carrying its `tree` field. + async fn mint_node_manifest( + &self, + node: &NodeData, + sharer: &str, + channel: &str, + parent: Option, + now: Timestamp, + entropy: &mut E, + ) -> Result<(), Envelope> + where + E: FnMut(&mut [u8]) -> Result<(), waggle_core::EntropyError>, + { + let target = waggle_core::CanonicalUrl::new(&node.dir_url) + .map_err(|e| Envelope::err(format!("tree target: {e}"), vec![]))?; + let sharer = waggle_core::Sharer::new(sharer) + .map_err(|e| Envelope::err(format!("sharer: {e}"), vec![]))?; + let channel = waggle_core::Channel::new(channel) + .map_err(|e| Envelope::err(format!("channel: {e}"), vec![]))?; + let mut spec = MintSpec::new(target, sharer, channel) + .with_token(node.token) + .tree(node.tree_node.clone()); + if let Some(p) = parent { + spec = spec.child_of(p); + } + let mut manifest = waggle_core::mint(spec, &MintOptions::default(), &mut *entropy, now) + .map_err(|e| Envelope::err(e.to_string(), vec![]))?; + if let Some(signer) = &self.signer { + manifest.signature = Some(waggle_core::trust::sign_manifest(&manifest, signer)); + } + let mut nonce = [0u8; 8]; + entropy(&mut nonce).map_err(|e| Envelope::err(format!("entropy: {e}"), vec![]))?; + match self + .store + .append(AppendIntent::Mint { + manifest: Box::new(manifest), + nonce: MintNonce(u64::from_le_bytes(nonce)), + }) + .await + { + Ok(Appended::Minted { .. }) => Ok(()), + Ok(_) => Err(Envelope::err("tree: non-mint receipt for a node", vec![])), + Err(e) => Err(crate::handlers::store_err(&e)), + } + } + + /// Serialize a value to JSON and pin it as a content-addressed blob. + async fn put_json( + &self, + value: &T, + content_type: &str, + ) -> Result { + let bytes = serde_json::to_vec(value) + .map_err(|e| Envelope::err(format!("tree index encode: {e}"), vec![]))?; + self.blobs + .put(&bytes, content_type) + .await + .map_err(|e| Envelope::err(e.to_string(), vec![])) + } +} + +/// The text a file contributes to the trigram index and Bloom: its own bytes when +/// text, a deterministic extraction for PDF/HTML, and nothing for opaque media +/// (which carries no searchable text — search simply never matches it, correctly). +fn index_text(content_type: &str, bytes: &[u8]) -> String { + if crate::content::is_text(content_type) || crate::content::sniff_is_text(bytes) { + return String::from_utf8_lossy(bytes).into_owned(); + } + crate::extract::deterministic_extract(content_type, bytes) + .map(|e| e.text) + .unwrap_or_default() +} + +/// The final path component of a `file://…/dir` URL — a subdir's name within its +/// parent. +fn dir_name(dir_url: &str) -> String { + dir_url + .trim_end_matches('/') + .rsplit('/') + .next() + .unwrap_or(dir_url) + .to_owned() +} + +/// Directory entries, sorted by name for deterministic node bytes. +fn read_sorted(dir: &std::path::Path) -> Result, Envelope> { + let mut entries: Vec<_> = std::fs::read_dir(dir) + .map_err(|e| Envelope::err(format!("tree: read `{}`: {e}", dir.display()), vec![]))? + .filter_map(Result::ok) + .collect(); + entries.sort_by_key(std::fs::DirEntry::file_name); + Ok(entries) +} + +/// Names a tree mint skips: VCS internals, dependency caches, and dotfiles — the +/// generated/vendored bulk that would bloat storage without being the artifact. +fn is_ignored(name: &str) -> bool { + matches!( + name, + ".git" | ".hg" | ".svn" | "node_modules" | "target" | ".venv" | "__pycache__" | ".DS_Store" + ) || name.starts_with('.') +} + +/// The byte budget for a tree mint: `max-bytes` if the caller set one, else +/// [`DEFAULT_BUDGET`]. +pub(crate) fn tree_budget(args: &Map) -> u64 { + args.get("max-bytes") + .and_then(Value::as_u64) + .unwrap_or(DEFAULT_BUDGET) +} diff --git a/crates/waggle-tree/src/bloom.rs b/crates/waggle-tree/src/bloom.rs index 5babd1c..968165c 100644 --- a/crates/waggle-tree/src/bloom.rs +++ b/crates/waggle-tree/src/bloom.rs @@ -123,6 +123,25 @@ impl Bloom { } out } + + /// Lowercase-hex of the bit storage — the form inlined in a manifest. Pairs + /// with [`Bloom::from_hex`]. + #[must_use] + pub fn to_hex(&self) -> String { + to_hex(self.bits.as_ref()) + } + + /// Parse the hex form back into a filter. Errors if the string is not exactly + /// [`Bloom::BYTES`] bytes of hex. + pub fn from_hex(hex: &str) -> Result { + let bytes = from_hex(hex)?; + let arr: [u8; Bloom::BYTES] = bytes.try_into().map_err(|v: Vec| { + format!("bloom: expected {} bytes, got {}", Bloom::BYTES, v.len()) + })?; + Ok(Self { + bits: Box::new(arr), + }) + } } impl Default for Bloom { From d35590906bb84d61368efc126e47f571df7df96f Mon Sep 17 00:00:00 2001 From: Chetan Conikee Date: Mon, 13 Jul 2026 14:32:40 -0700 Subject: [PATCH 04/12] =?UTF-8?q?mcp:=20indexed=20tree=20read,=20search,?= =?UTF-8?q?=20and=20coverage=20(Axis=20C)=20=E2=80=94=20the=20search=20wal?= =?UTF-8?q?l=20falls?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit read, search, and coverage now operate on the Merkle/trigram/Bloom index instead of per-file child tokens (tree_read.rs): read (projection) the directory's table of contents from its DirIndex — local files (name/size/type) and subdirs (name/token/totals). No descent; totals come from the index. read --file one file's bytes, fetched from the content-addressed blob by its hash, and stamped as a real read. search ONE call spans the whole lineage: prune each subtree on its Bloom (literal patterns), narrow to candidate files with the node's trigram index, confirm with a real regex match, and RANK (matches desc, then depth, then path). Each match carries its path + owning token to drill in. Fixes "search each subtree token separately". coverage node-granular over the tree: file-bearing nodes read / total, with the true file count from the root index. Pure-container nodes are excluded (nothing of their own to read). The old tree.rs (read_tree/lens/tree_files) and search_tree are deleted. Four integration tests rewritten to the indexed model; the gap-fix deny-list test too. Verified on ~/tulving/rote/docs (1,527 files, deep hierarchy): mints in one call in 1.6s (old cap was 200); a search for "fingerprint" spans the tree and ranks identity-binding.md (x418) first. Correct against a disk grep. Known, documented in the design log: the 256-byte Bloom saturates for very large subtrees, so pruning bites mainly at small/leaf nodes (correct — no false negatives — just less pruning at extreme density); and files:all per-file coverage awaits a served-set the payload-free log can't hold in an 8-bit mask. Both are follow-ups, not correctness gaps. All workspace tests pass; clippy clean; every file <= 750 lines. --- crates/waggle-mcp/src/content_handlers.rs | 160 +------- crates/waggle-mcp/src/handlers.rs | 5 +- crates/waggle-mcp/src/lib.rs | 2 +- crates/waggle-mcp/src/lineage.rs | 92 +++++ crates/waggle-mcp/src/tree.rs | 475 ---------------------- crates/waggle-mcp/src/tree_mint.rs | 50 ++- crates/waggle-mcp/src/tree_read.rs | 312 ++++++++++++++ crates/waggle-mcp/tests/content_access.rs | 230 +++++------ crates/waggle-mcp/tests/gap_fixes.rs | 28 +- 9 files changed, 572 insertions(+), 782 deletions(-) delete mode 100644 crates/waggle-mcp/src/tree.rs create mode 100644 crates/waggle-mcp/src/tree_read.rs diff --git a/crates/waggle-mcp/src/content_handlers.rs b/crates/waggle-mcp/src/content_handlers.rs index 784a4f3..1cde92f 100644 --- a/crates/waggle-mcp/src/content_handlers.rs +++ b/crates/waggle-mcp/src/content_handlers.rs @@ -296,7 +296,7 @@ impl Handler { // so a folder could be searched but never *described*. The first move // an agent makes with a shared directory is to ask what is in it, and // a consumer that cannot see the vocabulary can only guess a regex. - if let Some(env) = self.try_tree_read(token, args, max_bytes, now).await { + if let Some(env) = self.try_indexed_tree_read(token, args, now).await { return env; } @@ -421,11 +421,8 @@ impl Handler { // every descendant's content, matches grouped per file — the // folder token greps as a tree, locally and at the edge alike. if let Ok(Some(view)) = self.store.manifest(token).await { - if view.manifest.content.is_none() { - let children = self.store.children(token).await.unwrap_or_default(); - if !children.is_empty() { - return self.search_tree(token, pattern, args, now).await; - } + if view.manifest.tree.is_some() { + return self.search_indexed_tree(token, pattern, args, now).await; } } let view = match self.content_of(token).await { @@ -509,157 +506,6 @@ pub(crate) fn read_capped(path: &str) -> Result, Envelope> { std::fs::read(path).map_err(|e| Envelope::err(format!("content {path}: {e}"), vec![])) } -impl Handler { - /// Deep search over a lineage tree: BFS the descendants, grep each - /// one that carries content, group matches per file. Totals are - /// counted in full; listings are capped per file and by the byte - /// budget — truncation is named, never silent. - async fn search_tree( - &self, - root: waggle_core::Token, - pattern: &str, - args: &Map, - now: Timestamp, - ) -> Envelope { - let context = args - .get("context") - .and_then(Value::as_u64) - .map_or(1, |v| usize::try_from(v).unwrap_or(1)); - let per_file = args - .get("max-matches") - .and_then(Value::as_u64) - .map_or(3, |v| usize::try_from(v).unwrap_or(3)); - let max_bytes = args - .get("max-bytes") - .and_then(Value::as_u64) - .and_then(|v| usize::try_from(v).ok()) - .unwrap_or(crate::query::DEFAULT_MAX_BYTES); - - let mut queue: std::collections::VecDeque<_> = - self.store.children(root).await.unwrap_or_default().into(); - let mut files = Vec::new(); - let mut total: u64 = 0; - let mut searched = 0u64; - let mut skipped = 0u64; - let mut visited = 0; - while let Some(child) = queue.pop_front() { - visited += 1; - if visited > 200 { - skipped += 1 + queue.len() as u64; - break; - } - queue.extend(self.store.children(child).await.unwrap_or_default()); - let Ok(Some(view)) = self.store.manifest(child).await else { - continue; - }; - let Ok(child_view) = self.content_of(child).await else { - skipped += 1; // no snapshot here (or binary) — named below - continue; - }; - let (text, child_contract) = (child_view.text, child_view.contract); - searched += 1; - let Ok(found) = - crate::content::search(&text, pattern, context, per_file, max_bytes / 4) - else { - continue; // the regex did not compile against this file - }; - let file_total = found["total_matches"].as_u64().unwrap_or(0); - if file_total == 0 { - // The grep touched this file's bytes; the CONSUMER received - // nothing from it. Those are different ledgers, and `read` is - // the consumer's: it means content was *served*. Stamping it - // here would let one zero-match search over a folder mark every - // file consumed — we tried it, and `read` went 0/11 to 11/11 - // having shown the consumer not one byte. A receipt that can be - // satisfied without seeing anything is not a receipt. - continue; - } - // Bytes reached the consumer: the child's funnel says so, and a hit - // inside a required region is a touch (19 §4.2) — the grep IS the - // evidence. - let touched = crate::contract_args::match_bits(child_contract.as_ref(), &found); - self.record_read(child, now, touched).await; - total += file_total; - files.push(json!({ - "token": child.as_str(), - "target": view.manifest.target.as_str(), - "total_matches": file_total, - "matches": found["matches"], - })); - } - // Regex sanity: if nothing was searchable the pattern never ran. - if searched == 0 { - return Envelope::err( - format!( - "no searchable content under {root} — its children have no snapshots here; `waggle edge push` (or re-mint with --tree) pins the bytes" - ), - vec![], - ); - } - let mut result = json!({ - "token": root.as_str(), - "pattern": pattern, - "tree": { "files_searched": searched, "files_skipped": skipped }, - "total_matches": total, - "files": files, - }); - // The byte budget holds for the whole tree answer. - let rendered = serde_json::to_vec(&result).map_or(0, |b| b.len()); - if rendered > max_bytes { - let files = result["files"].as_array().cloned().unwrap_or_default(); - let mut kept = Vec::new(); - let mut used = 200; // envelope skeleton allowance - for f in files { - let size = serde_json::to_vec(&f).map_or(0, |b| b.len()); - if used + size > max_bytes { - break; - } - used += size; - kept.push(f); - } - let dropped = result["files"].as_array().map_or(0, Vec::len) - kept.len(); - result["files"] = json!(kept); - result["truncated"] = json!(format!( - "{dropped} matching file(s) beyond the {max_bytes}-byte budget — raise max-bytes or search a child directly" - )); - } - self.record_read(root, now, None).await; - let next = files_next(&result, root); - Envelope::ok(result, next).with_stats(Stats { - records: Some(searched), - seq: None, - }) - } -} - -/// The grep→open chain for tree results: read the first matching file. -fn files_next(result: &Value, root: waggle_core::Token) -> Vec { - let mut next = Vec::new(); - if let Some(first) = result["files"].as_array().and_then(|f| f.first()) { - if let (Some(token), Some(line)) = ( - first["token"].as_str(), - first["matches"] - .as_array() - .and_then(|m| m.first()) - .and_then(|m| m["line"].as_u64()), - ) { - let from = line.saturating_sub(5).max(1); - next.push(NextCall { - tool: "read".into(), - args: json!({ "token": token, "lines": format!("{from}-{}", line + 10) }), - why: "open the first matching file at its hit".into(), - }); - } - } - next.push(NextCall { - tool: "resolve".into(), - args: json!({ "token": root.as_str() }), - why: "the root's index: every child token by filename".into(), - }); - next.truncate(3); - next -} - /// Continuation guidance for read results. fn read_next(token: Token, result: &Value) -> Vec { let mut next = Vec::new(); diff --git a/crates/waggle-mcp/src/handlers.rs b/crates/waggle-mcp/src/handlers.rs index 5774701..4a1ed01 100644 --- a/crates/waggle-mcp/src/handlers.rs +++ b/crates/waggle-mcp/src/handlers.rs @@ -183,9 +183,7 @@ impl Handler { }; // A tree mint builds the whole hierarchy under its own node tokens, so it // routes here directly — minting a plain root first would orphan a token. - let is_tree = args.get("tree").and_then(Value::as_bool).unwrap_or(false) - || arg_str(args, "tree") == Some("true"); - if is_tree { + if crate::tree_mint::wants_tree(args) { let parent = arg_str(args, "parent").and_then(|p| Token::parse(p).ok()); return self .mint_tree_indexed( @@ -194,6 +192,7 @@ impl Handler { channel.as_str(), parent, crate::tree_mint::tree_budget(args), + args, now, entropy, ) diff --git a/crates/waggle-mcp/src/lib.rs b/crates/waggle-mcp/src/lib.rs index 8393be2..5e3315c 100644 --- a/crates/waggle-mcp/src/lib.rs +++ b/crates/waggle-mcp/src/lib.rs @@ -32,8 +32,8 @@ pub mod query; mod record; mod resources; mod rpc; -mod tree; mod tree_mint; +mod tree_read; pub use envelope::{validate_next, Envelope, NextCall, Stats}; pub use handlers::Handler; diff --git a/crates/waggle-mcp/src/lineage.rs b/crates/waggle-mcp/src/lineage.rs index 5e7d4a9..8a47e39 100644 --- a/crates/waggle-mcp/src/lineage.rs +++ b/crates/waggle-mcp/src/lineage.rs @@ -99,6 +99,15 @@ impl Handler { Ok(v) => v, Err(e) => return store_err(&e), }; + // An indexed tree (design doc: tree-scale) is a hierarchy of directory + // NODES, not per-file tokens, so its coverage is node-granular: how many + // directory nodes were touched, out of the total, plus the true file count + // from the root's index. (Per-file `files:all` coverage needs a served-set + // the payload-free log cannot express in an 8-bit region mask — a + // documented follow-up.) + if view.as_ref().is_some_and(|v| v.manifest.tree.is_some()) { + return self.tree_node_coverage(root, view.as_ref().unwrap()).await; + } let mut queue: std::collections::VecDeque<_> = match self.store.children(root).await { Ok(c) => c.into(), Err(e) => return store_err(&e), @@ -182,6 +191,89 @@ impl Handler { }) } + /// Node-granular coverage for an indexed tree: walk the directory-node + /// lineage and report how many nodes have been read, the total node count, and + /// the true file/byte totals from the root's index. A node is "read" once any + /// of its files was served (search hit or `read --file`). + async fn tree_node_coverage( + &self, + root: waggle_core::Token, + view: &waggle_store::ManifestView, + ) -> Envelope { + let (total_files, total_bytes) = view + .manifest + .tree + .as_ref() + .map_or((0, 0), |t| (t.files, t.bytes)); + let mut queue: std::collections::VecDeque<_> = [root] + .into_iter() + .collect::>(); + let (mut nodes, mut read, mut visited) = (0u64, 0u64, 0); + let mut unread: Vec = Vec::new(); + while let Some(node) = queue.pop_front() { + visited += 1; + if visited > 5000 { + break; // runaway-lineage backstop + } + if let Ok(more) = self.store.children(node).await { + queue.extend(more); + } + let Ok(Some(v)) = self.store.manifest(node).await else { + continue; + }; + let Some(tn) = v.manifest.tree.as_ref() else { + continue; + }; + // Only file-bearing nodes count: a pure container (subdirs only) has + // nothing of its own to read, so it can neither be read nor block + // completeness — it is covered when its children are. + if !self.node_has_local_files(tn).await { + continue; + } + nodes += 1; + let (was_read, _) = self.child_consumption(node).await; + if was_read { + read += 1; + } else if unread.len() < 20 { + unread.push(json!({ + "token": node.as_str(), + "target": v.manifest.target.as_str(), + })); + } + } + Envelope::ok( + json!({ + "token": root.as_str(), + "kind": "tree", + "nodes": format!("{read}/{nodes}"), + "total_files": total_files, + "total_bytes": total_bytes, + "complete": unread.is_empty(), + "unread_nodes": unread, + }), + vec![NextCall { + tool: "search".into(), + args: json!({ "token": root.as_str(), "pattern": "" }), + why: "reading through search stamps the nodes it serves".into(), + }], + ) + .with_stats(Stats { + records: Some(nodes), + seq: None, + }) + } + + /// Does this node have files of its own (not just subdirectories)? Reads the + /// node's directory index; a fetch failure conservatively counts as "yes" so a + /// node is never silently dropped from coverage. + async fn node_has_local_files(&self, node: &waggle_core::TreeNode) -> bool { + match self.blobs.get(&node.index).await { + Ok(bytes) => serde_json::from_slice::(&bytes) + .map_or(true, |idx| idx.files().next().is_some()), + Err(_) => true, + } + } + /// Single-token contract coverage (19 §4.2): fold the region-touch /// bits out of the token's own records, evaluate against the /// declared contract, and NAME the misses — label and line range, diff --git a/crates/waggle-mcp/src/tree.rs b/crates/waggle-mcp/src/tree.rs deleted file mode 100644 index e35da56..0000000 --- a/crates/waggle-mcp/src/tree.rs +++ /dev/null @@ -1,475 +0,0 @@ -//! The directory affordances (design doc `22 §4`): what a token means when it -//! names a **folder** rather than a file. -//! -//! Every one of these exists because the benchmark caught an agent needing it -//! and not having it: -//! -//! * `search` had always grepped a tree, but `read` answered null — so a folder -//! could be searched and never *described*. The first move an agent makes with -//! a shared directory is to ask what is in it, and a consumer that cannot see -//! the vocabulary can only guess a regex. -//! * A tree could be grepped but not *lensed*. We watched an agent issue -//! `section: "Retry Policy"` ten times, once per child token, hand-rolling a -//! fan-out the substrate should have done in one call. -//! * A fan-out that ran out of budget truncated in silence. An agent read nine -//! of ten runbooks — the missing one was the violator — and answered -//! confidently and wrongly. Incompleteness must be loud, and resumable. - -use serde_json::{json, Map, Value}; -use waggle_core::Timestamp; -use waggle_store::{BlobSink, Store}; - -use crate::content_handlers::ContentView; -use crate::envelope::{Envelope, NextCall, Stats}; -use crate::handlers::{arg_str, Handler}; - -impl Handler { - /// If the token is a lineage root with no content of its own, it is a TREE: - /// serve the directory instead of failing. With a lens, fan that lens out - /// across every file — handed a folder, an agent asks the same question of - /// all of it, and we watched one issue `section: "Retry Policy"` ten times, - /// once per child, because a tree could be grepped but never *lensed*. - pub(crate) async fn try_tree_read( - &self, - token: waggle_core::Token, - args: &Map, - max_bytes: usize, - now: Timestamp, - ) -> Option { - let v = self.store.manifest(token).await.ok()??; - if v.manifest.content.is_some() { - return None; - } - if self - .store - .children(token) - .await - .unwrap_or_default() - .is_empty() - { - return None; - } - let lens = ["section", "symbol", "lines"] - .into_iter() - .find_map(|k| arg_str(args, k).map(|val| (k, val.to_owned()))); - let from = args.get("from").and_then(Value::as_u64); - Some(if let Some((kind, value)) = lens { - self.read_tree_lens(token, kind, &value, max_bytes, from, now) - .await - } else { - self.read_tree(token, max_bytes, from).await - }) - } - - /// Every file token under a tree, in a stable order. `mint --tree` flattens, - /// so this is usually one hop — but it walks, so a nested lineage yields the - /// same list, and the projection and the fan-out can never disagree about - /// what "all the files" means. - pub(crate) async fn tree_files(&self, root: waggle_core::Token) -> Vec { - let mut out = Vec::new(); - let mut queue: std::collections::VecDeque<_> = - self.store.children(root).await.unwrap_or_default().into(); - while let Some(child) = queue.pop_front() { - if out.len() >= 200 { - break; - } - queue.extend(self.store.children(child).await.unwrap_or_default()); - out.push(child); - } - out - } - - /// One child's row in the directory projection: its path relative to the - /// folder, its own token, and the structure it affords — the SYMBOLS a - /// source file already carries, or the headings of a document. A folder of - /// code that lists no symbols presents as structureless, which is the one - /// shape where knowing what is inside each file matters most. - async fn tree_entry( - &self, - child: waggle_core::Token, - v: &waggle_store::ManifestView, - base: Option<&str>, - max_bytes: usize, - total_bytes: &mut u64, - ) -> Value { - let target = v.manifest.target.as_str().to_owned(); - // A basename is not an identity: a repository is full of `mod.py` and - // `index.ts`, and a consumer handed six of them cannot tell which is - // which, nor reason about where anything sits. - let name = crate::content_handlers::local_path(&target).map_or_else( - || target.clone(), - |p| match base { - Some(b) => p.strip_prefix(b).unwrap_or(&p).to_owned(), - None => p.clone(), - }, - ); - let mut entry = json!({ "name": name, "token": child.as_str() }); - - let Ok(cv) = self.content_of(child).await else { - return entry; - }; - *total_bytes += cv.text.len() as u64; - entry["bytes"] = json!(cv.text.len()); - entry["lines"] = json!(cv.text.lines().count()); - entry["content_type"] = json!(cv.content_type); - - if let Some(media) = &v.manifest.outline { - if let Ok(blob) = self.blobs.get(media).await { - if let Some(sym) = crate::outline_wire::render(&blob, max_bytes / 4) { - let names: Vec = sym - .get("symbols") - .and_then(Value::as_array) - .map(|rows| { - rows.iter() - .take(8) - .filter_map(|r| r.get("name").cloned()) - .collect() - }) - .unwrap_or_default(); - if !names.is_empty() { - entry["symbols"] = Value::Array(names); - entry["lenses"] = json!(["symbol", "lines", "search"]); - return entry; - } - } - } - } - let o = if cv.content_type == "text/markdown" { - crate::content::outline(&cv.text) - } else { - crate::content::outline_plain(&cv.text) - }; - if let Value::Array(items) = o { - let heads: Vec = items - .into_iter() - .take(6) - .filter_map(|h| h.get("heading").cloned()) - .collect(); - if !heads.is_empty() { - entry["outline"] = Value::Array(heads); - } - } - entry - } - - /// The **directory projection**: what `read` returns for a tree. - /// - /// Each child by name, with its own token (so the consumer can address it - /// directly), its size, type, and — budget permitting — its outline. That - /// is the folder's table of contents: the consumer learns the vocabulary - /// before it has to guess a pattern, and it learns which file to open - /// without opening all of them. - /// - /// Listing is *not* consumption: no `read` stage is stamped for the - /// children here. A table of contents tells you what exists; it does not - /// serve you the bytes, and the receipts must not pretend it did. - /// - /// A big tree does not fit in one projection, and a projection that stops - /// at 25 of 180 files without saying so is the worst possible answer: the - /// consumer reads a complete-looking table of contents and concludes the - /// tree contains nothing else. So the count is taken BEFORE the budget is - /// spent, `complete` is stated outright, and truncation hands back both - /// ways out — resume at the cursor, or stop listing and `search`, which - /// returns only the files that matched and never pays for the rest. - async fn read_tree( - &self, - root: waggle_core::Token, - max_bytes: usize, - from: Option, - ) -> Envelope { - // The folder's own directory, so children can be named by their path - // RELATIVE to it. A basename is not an identity: a repository is full - // of `mod.py` and `index.ts`, and a consumer handed six of them cannot - // tell which is which, nor reason about where anything sits. - let base = match self.store.manifest(root).await { - Ok(Some(v)) => crate::content_handlers::local_path(v.manifest.target.as_str()) - .map(|p| format!("{}/", p.trim_end_matches('/'))), - _ => None, - }; - - // The denominator first. Knowing there are 180 files is what makes a - // 25-file answer honest, and it costs one walk of the lineage — no - // manifests, no blobs, no budget. - let all = self.tree_files(root).await; - let total_files = all.len() as u64; - let start = usize::try_from(from.unwrap_or(0)).unwrap_or(0); - - let mut files = Vec::new(); - let mut total_bytes: u64 = 0; - let mut budget = 0usize; - let mut truncated = false; - - for child in all.iter().skip(start) { - let Ok(Some(v)) = self.store.manifest(*child).await else { - continue; - }; - let entry = self - .tree_entry(*child, &v, base.as_deref(), max_bytes, &mut total_bytes) - .await; - let cost = entry.to_string().len(); - if !files.is_empty() && budget + cost > max_bytes { - truncated = true; - break; - } - budget += cost; - files.push(entry); - } - - let n = files.len(); - let listed = start + n; - let complete = !truncated && listed as u64 >= total_files; - let mut result = json!({ - "kind": "tree", - "files": n, - "total_files": total_files, - "listed": format!("{}/{}", listed, total_files), - "complete": complete, - "total_bytes": total_bytes, - "truncated": truncated, - "children": files, - }); - - let mut next = Vec::new(); - if !complete { - // Loud. The consumer has seen a fraction of the tree, and the one - // thing it must not do is reason as though it has seen all of it. - result["hint"] = json!(format!( - "INCOMPLETE LISTING: {listed} of {total_files} files. The rest are NOT \ - shown and you have NOT seen them. Do not conclude anything about the \ - tree from this page. Either narrow with `search` — which returns only \ - the files that match, and never pays for the others — or page on with \ - `from: {listed}`." - )); - next.push(NextCall { - tool: "search".into(), - args: json!({ "token": root.as_str(), "pattern": "" }), - why: "PREFERRED at this size: grep the whole tree and get back ONLY the \ - matching files, each with its own token — a filtered listing that \ - costs a fraction of the full one" - .into(), - }); - next.push(NextCall { - tool: "read".into(), - args: json!({ "token": root.as_str(), "from": listed }), - why: "continue the listing from where it stopped".into(), - }); - } - next.push(NextCall { - tool: "read".into(), - args: json!({ "token": root.as_str(), "section": "" }), - why: "that section from EVERY file in ONE call — do not fetch them one at a time" - .into(), - }); - next.push(NextCall { - tool: "search".into(), - args: json!({ "token": root.as_str(), "pattern": "" }), - why: "grep every file in the tree at once; matches come back per file".into(), - }); - next.push(NextCall { - tool: "read".into(), - args: json!({ "token": "" }), - why: "open a single file — the listing gave you its token".into(), - }); - Envelope::ok(result, next).with_stats(Stats { - records: Some(n as u64), - seq: None, - }) - } - /// Apply one lens to one file — the per-child half of a tree fan-out. - async fn lens_one( - &self, - child: waggle_core::Token, - cv: &ContentView, - kind: &str, - value: &str, - per_file: usize, - ) -> Option { - match kind { - "section" => crate::content::read_section(&cv.text, value, per_file), - "symbol" => match self.resolve_symbol(child, cv.outline.as_ref(), value).await { - Ok((from, to)) => Some(crate::content::read_lines(&cv.text, from, to, per_file)), - Err(_) => None, - }, - _ => value - .split_once('-') - .and_then(|(a, b)| { - Some(( - a.trim().parse::().ok()?, - b.trim().parse::().ok()?, - )) - }) - .map(|(f, t)| crate::content::read_lines(&cv.text, f, t, per_file)), - } - } - /// **Lens the tree**: apply one lens to every file in it. - /// - /// `search` has always grepped a folder. This is the other half: ask the - /// same *structural* question of every file — "the Retry Policy section - /// from all twelve runbooks", "the `handle` symbol wherever it is defined". - /// We added it because we watched agents hand-roll exactly this, issuing - /// the identical command once per child token; a folder that can be grepped - /// but not lensed makes the consumer do the fan-out itself. - /// - /// Unlike the listing, this SERVES bytes, so each file it answers for is - /// stamped as read — the receipt records what the consumer actually got. - async fn read_tree_lens( - &self, - root: waggle_core::Token, - kind: &str, - value: &str, - max_bytes: usize, - args_from: Option, - now: Timestamp, - ) -> Envelope { - let all = self.tree_files(root).await; - let total_files = all.len() as u64; - // `from` continues a fan-out that ran out of budget. A tree-lens that - // truncates in silence is worse than a slow one: we watched an agent - // reason over 9 of 10 runbooks — the missing one was the violator — and - // answer confidently and wrongly. Completeness has to be resumable, and - // incompleteness has to be loud. - let from = usize::try_from(args_from.unwrap_or(0)).unwrap_or(0); - // `max_bytes` is a PER-RESPONSE cap, and applying it to an N-file - // response is a category error. It was being split `max_bytes / 6` — a - // hardcoded six-file assumption — so a fan-out over eleven files served - // nine and silently dropped two. `--require files:all` was therefore - // unsatisfiable in one call BY CONSTRUCTION: the one move that closes a - // completeness contract could not close it. Consumers fell back to - // fetching children one at a time, and the weaker ones burned their turn - // budget and never answered — while the gate, correctly, refused to - // believe the ones that answered early. - // - // Nor is the answer to divide the budget by the file count: the load- - // bearing sentence is usually at the END of a section, and slicing every - // file to a thin prefix serves all of them while cutting the fact out of - // each. Full coverage, zero information. - // - // Only four ways exist to fit eleven full sections into an eight-kilobyte - // budget, and three of them are wrong: - // - // thin every file — the load-bearing sentence is at the END of a - // section; a thin prefix of all of them serves - // everything and cuts the fact out of each. Full - // coverage, zero information. - // drop files — the original bug: nine of eleven, silently. - // overrun the caller — we tried this, and it was the worst of the - // three. `max_bytes` is a CONTRACT with the caller. - // We returned 9,255 bytes against a budget of 8,000; - // the client showed its model the first 4,500 and - // dropped the rest — while our receipt certified all - // ten files as read. The gate then believed a - // consumer that had seen five runbooks out of eleven. - // A receipt must never attest to bytes the consumer - // did not get, and we cannot know what a client - // truncates: so we must not exceed what it asked for. - // - // PAGE — serve as many WHOLE files as the budget holds, say - // `complete: false`, hand back `from`. Honest about - // depth, honest about coverage, honest about cost. - // - // So each served file gets a full single-file allowance, and the response - // as a whole never exceeds the budget the caller set. - let per_file = max_bytes; - let ceiling = max_bytes; - let mut queue: std::collections::VecDeque<_> = all.into_iter().skip(from).collect(); - let mut files = Vec::new(); - let mut visited = 0usize; - let mut budget = 0usize; - let mut matched = 0u64; - let mut skipped = 0u64; - let mut truncated = false; - let mut consumed = 0usize; - - while let Some(child) = queue.pop_front() { - visited += 1; - if visited > 200 { - truncated = true; - break; - } - consumed += 1; - queue.extend(self.store.children(child).await.unwrap_or_default()); - let Ok(Some(v)) = self.store.manifest(child).await else { - continue; - }; - let Ok(cv) = self.content_of(child).await else { - skipped += 1; - continue; - }; - let served = self.lens_one(child, &cv, kind, value, per_file).await; - let Some(slice) = served else { - skipped += 1; // this file simply has no such section/symbol - continue; - }; - matched += 1; - let target = v.manifest.target.as_str().to_owned(); - let name = target - .rsplit('/') - .next() - .unwrap_or(target.as_str()) - .to_owned(); - let entry = json!({ - "name": name, - "token": child.as_str(), - "lines": slice["lines"], - "text": slice["text"], - }); - // Decide BEFORE committing. Appending an entry and only then noticing - // we are over budget overshoots by up to one whole file — and an - // overshoot is not a rounding error, it is a receipt that lies: the - // client truncates what we overran, and we have already stamped the - // file `read`. If it does not fit, stop here and let `from` carry it. - let cost = entry.to_string().len(); - if !files.is_empty() && budget + cost > ceiling { - truncated = true; - consumed -= 1; - break; - } - budget += cost; - files.push(entry); - // Bytes were served — and only now, once we know the consumer will - // actually receive them, does the receipt say so. - self.record_read(child, now, None).await; - } - - let seen = from + consumed; - let complete = !truncated && seen as u64 >= total_files; - let result = json!({ - "kind": "tree-lens", - "lens": kind, - "of": value, - "total_files": total_files, - "examined": seen, - "matched": matched, - "skipped": skipped, - "complete": complete, - "truncated": truncated, - "files": files, - }); - let next = if truncated { - // LOUD: the consumer has not seen the whole tree. Say so, and say - // exactly how to finish — a partial fan-out that reads as a whole - // one is how a confident wrong answer gets made. - vec![NextCall { - tool: "read".into(), - args: json!({ - "token": root.as_str(), kind: value, "from": seen, - }), - why: format!( - "INCOMPLETE: {seen} of {total_files} files examined. Continue from `from`={seen} before you conclude anything about the tree" - ), - }] - } else if matched == 0 { - vec![NextCall { - tool: "read".into(), - args: json!({ "token": root.as_str() }), - why: "no file had that — the tree listing names every file's outline".into(), - }] - } else { - vec![] - }; - Envelope::ok(result, next).with_stats(Stats { - records: Some(matched), - seq: None, - }) - } -} diff --git a/crates/waggle-mcp/src/tree_mint.rs b/crates/waggle-mcp/src/tree_mint.rs index 7071117..50ed280 100644 --- a/crates/waggle-mcp/src/tree_mint.rs +++ b/crates/waggle-mcp/src/tree_mint.rs @@ -67,6 +67,7 @@ impl Handler { /// Mint a directory as an indexed Merkle tree. Returns the root token's /// envelope. Called by the `mint --tree` path in place of the old flat /// per-file minting. + #[allow(clippy::too_many_arguments)] pub(crate) async fn mint_tree_indexed( &self, dir: &str, @@ -74,20 +75,18 @@ impl Handler { channel: &str, parent: Option, budget: u64, + args: &Map, now: Timestamp, entropy: &mut E, ) -> Envelope where E: FnMut(&mut [u8]) -> Result<(), waggle_core::EntropyError>, { - let path = match local_path(dir) { - Some(p) => p, - None => { - return Envelope::err( - format!("tree: `{dir}` is not a local directory — --tree walks the filesystem"), - vec![], - ) - } + let Some(path) = local_path(dir) else { + return Envelope::err( + format!("tree: `{dir}` is not a local directory — --tree walks the filesystem"), + vec![], + ); }; // Phase one: compute the whole tree bottom-up — pin blobs, build indexes, // pre-generate tokens. No manifests yet. @@ -107,7 +106,8 @@ impl Handler { // Phase two: mint manifests top-down, so each parent exists before its // children link to it. if let Err(e) = - Box::pin(self.mint_nodes(&root, sharer, channel, parent, now, entropy)).await + Box::pin(self.mint_nodes(&root, sharer, channel, parent, true, args, now, entropy)) + .await { return e; } @@ -235,23 +235,38 @@ impl Handler { /// Phase two — mint manifests top-down, so a parent is in the store before its /// children link to it. + #[allow(clippy::too_many_arguments)] async fn mint_nodes( &self, node: &NodeData, sharer: &str, channel: &str, parent: Option, + is_root: bool, + args: &Map, now: Timestamp, entropy: &mut E, ) -> Result<(), Envelope> where E: FnMut(&mut [u8]) -> Result<(), waggle_core::EntropyError>, { - self.mint_node_manifest(node, sharer, channel, parent, now, entropy) + // Tags (and any root-only mint options) land on the root node, so `find` + // recovers the tree by the name a human remembers. + let root_args = is_root.then_some(args); + self.mint_node_manifest(node, sharer, channel, parent, root_args, now, entropy) .await?; for child in &node.children { - Box::pin(self.mint_nodes(child, sharer, channel, Some(node.token), now, entropy)) - .await?; + Box::pin(self.mint_nodes( + child, + sharer, + channel, + Some(node.token), + false, + args, + now, + entropy, + )) + .await?; } Ok(()) } @@ -295,12 +310,14 @@ impl Handler { /// Mint (and sign, and persist) one directory node's manifest under its /// pre-generated token, carrying its `tree` field. + #[allow(clippy::too_many_arguments)] async fn mint_node_manifest( &self, node: &NodeData, sharer: &str, channel: &str, parent: Option, + root_args: Option<&Map>, now: Timestamp, entropy: &mut E, ) -> Result<(), Envelope> @@ -319,6 +336,9 @@ impl Handler { if let Some(p) = parent { spec = spec.child_of(p); } + if let Some(args) = root_args { + spec = crate::discovery::apply_tags(spec, args); + } let mut manifest = waggle_core::mint(spec, &MintOptions::default(), &mut *entropy, now) .map_err(|e| Envelope::err(e.to_string(), vec![]))?; if let Some(signer) = &self.signer { @@ -397,6 +417,12 @@ fn is_ignored(name: &str) -> bool { ) || name.starts_with('.') } +/// Did the caller ask for a tree mint? +pub(crate) fn wants_tree(args: &Map) -> bool { + args.get("tree").and_then(Value::as_bool).unwrap_or(false) + || args.get("tree").and_then(Value::as_str) == Some("true") +} + /// The byte budget for a tree mint: `max-bytes` if the caller set one, else /// [`DEFAULT_BUDGET`]. pub(crate) fn tree_budget(args: &Map) -> u64 { diff --git a/crates/waggle-mcp/src/tree_read.rs b/crates/waggle-mcp/src/tree_read.rs new file mode 100644 index 0000000..6cb03b1 --- /dev/null +++ b/crates/waggle-mcp/src/tree_read.rs @@ -0,0 +1,312 @@ +//! Reading and searching an indexed tree (design doc: tree-scale). +//! +//! A token minted `--tree` carries a [`waggle_core::TreeNode`] instead of inline +//! content. This module serves it: +//! +//! * **projection** — `read` on the token returns the directory's table of +//! contents from its [`DirIndex`]: files (name, size, type) and subdirectories +//! (name, token, totals), with no descent; +//! * **file read** — `read … --file ` fetches one file's bytes back from the +//! blob store by its content hash and serves them (with the ordinary lenses); +//! * **search** — `search` on the token descends the whole lineage in one call, +//! pruning subtrees with each node's Bloom, narrowing to candidate files with +//! each node's trigram index, confirming with a real match, and returning +//! **ranked** hits — each with its path and owning token so a consumer can drill +//! in. +//! +//! The traversal is the I/O half of `waggle-tree`: it fetches index blobs and file +//! bytes and applies the pure prune/candidate/rank decisions that crate defines. + +use serde_json::{json, Map, Value}; +use waggle_core::{CanonicalUrl, MediaRef, Sha256Hex, Timestamp, Token, TreeNode}; +use waggle_store::{BlobSink, Store}; +use waggle_tree::{search as tsearch, Bloom, DirIndex, FileEntry, Hit, TrigramIndex}; + +use crate::envelope::{Envelope, NextCall, Stats}; +use crate::handlers::{arg_str, Handler}; + +impl Handler { + /// If `token` is an indexed tree node, serve it and return `Some`. Non-tree + /// tokens return `None` so the caller falls through to file handling. + pub(crate) async fn try_indexed_tree_read( + &self, + token: Token, + args: &Map, + now: Timestamp, + ) -> Option { + let view = self.store.manifest(token).await.ok()??; + let node = view.manifest.tree.clone()?; + Some(match arg_str(args, "file") { + Some(name) => self.read_tree_file(token, &node, name, now).await, + None => self.read_tree_toc(token, &node).await, + }) + } + + /// The directory's table of contents from its index blob. + async fn read_tree_toc(&self, token: Token, node: &TreeNode) -> Envelope { + let index = match self.load_index(node).await { + Ok(i) => i, + Err(e) => return e, + }; + let files: Vec = index + .files() + .map(|f| json!({ "name": f.name, "bytes": f.size, "content_type": f.content_type })) + .collect(); + let subdirs: Vec = index + .subdirs() + .map( + |d| json!({ "name": d.name, "token": d.token, "files": d.files, "bytes": d.bytes }), + ) + .collect(); + let next = vec![ + NextCall { + tool: "search".into(), + args: json!({ "token": token.as_str(), "pattern": "" }), + why: "grep the WHOLE tree in one call — pruned and ranked".into(), + }, + NextCall { + tool: "read".into(), + args: json!({ "token": token.as_str(), "file": "" }), + why: "read one file's content by name".into(), + }, + ]; + Envelope::ok( + json!({ + "kind": "tree", + "files": files.len(), + "subdirs": subdirs.len(), + "total_files": node.files, + "total_bytes": node.bytes, + "children": files, + "dirs": subdirs, + }), + next, + ) + } + + /// Serve one file's bytes, fetched from the blob store by its content hash. + async fn read_tree_file( + &self, + token: Token, + node: &TreeNode, + name: &str, + now: Timestamp, + ) -> Envelope { + let index = match self.load_index(node).await { + Ok(i) => i, + Err(e) => return e, + }; + let Some(entry) = index.files().find(|f| f.name == name) else { + let names: Vec<&str> = index.files().map(|f| f.name.as_str()).take(20).collect(); + return Envelope::err( + format!("tree: no file `{name}` in this directory — files: {names:?}"), + vec![], + ); + }; + let media = match file_media(entry) { + Ok(m) => m, + Err(e) => return e, + }; + let bytes = match self.blobs.get(&media).await { + Ok(b) => b, + Err(e) => return Envelope::err(e.to_string(), vec![]), + }; + // A file read is real consumption — stamp it against this node so a + // files:all coverage can roll it up. + self.record_read(token, now, None).await; + let text = String::from_utf8_lossy(&bytes); + Envelope::ok( + json!({ + "name": name, + "content_type": entry.content_type, + "bytes": entry.size, + "text": text, + }), + vec![], + ) + .with_stats(Stats { + records: Some(bytes.len() as u64), + seq: None, + }) + } + + /// Search the whole tree from `root` in one call: prune, narrow, confirm, rank. + pub(crate) async fn search_indexed_tree( + &self, + root: Token, + pattern: &str, + args: &Map, + now: Timestamp, + ) -> Envelope { + let limit = args + .get("max-matches") + .and_then(Value::as_u64) + .unwrap_or(10) + .min(50) as usize; + let re = match regex::Regex::new(pattern) { + Ok(r) => r, + Err(e) => return Envelope::err(format!("search: bad pattern: {e}"), vec![]), + }; + let mut hits: Vec = Vec::new(); + let mut visited = 0usize; + Box::pin(self.search_node(root, "", pattern, &re, &mut hits, &mut visited)).await; + let ranked = tsearch::rank(hits, limit); + // Each confirmed match served that file's bytes — record it once. + for h in &ranked { + if let Ok(t) = Token::parse(&h.token) { + self.record_read(t, now, None).await; + } + } + let matches: Vec = ranked + .iter() + .map(|h| { + json!({ "path": h.path, "token": h.token, "line": h.line, + "text": h.text, "matches": h.matches }) + }) + .collect(); + Envelope::ok( + json!({ + "kind": "tree-search", + "total_matches": matches.len(), + "nodes_visited": visited, + "matches": matches, + }), + vec![NextCall { + tool: "read".into(), + args: json!({ "token": "", "file": "" }), + why: "open a matching file by name on its owning node".into(), + }], + ) + } + + /// One node of the search descent. Prunes on this node's Bloom, greps its + /// candidate files, then recurses into subdirectories. + async fn search_node( + &self, + token: Token, + prefix: &str, + pattern: &str, + re: ®ex::Regex, + hits: &mut Vec, + visited: &mut usize, + ) { + let Ok(Some(view)) = self.store.manifest(token).await else { + return; + }; + let Some(node) = view.manifest.tree.clone() else { + return; + }; + // Prune: if the Bloom proves the pattern's trigrams are absent, skip the + // whole subtree. Only literal patterns prune; a regex enters and greps. + if is_literal(pattern) { + if let Ok(bloom) = Bloom::from_hex(&node.bloom) { + if tsearch::prune(&bloom, pattern) == tsearch::Prune::Skip { + return; + } + } + } + *visited += 1; + let Ok(index) = self.load_index(&node).await else { + return; + }; + let files: Vec<&FileEntry> = index.files().collect(); + + // Narrow to candidate files with the trigram index (literal patterns); + // otherwise every file is a candidate. + let candidates: Vec = match (&node.trigram, is_literal(pattern)) { + (Some(tref), true) => match self.load_trigram(tref).await { + Ok(idx) => idx + .candidates(pattern) + .into_iter() + .map(|d| d as usize) + .collect(), + Err(_) => (0..files.len()).collect(), + }, + _ => (0..files.len()).collect(), + }; + + for i in candidates { + let Some(entry) = files.get(i) else { continue }; + let Ok(media) = file_media(entry) else { + continue; + }; + let Ok(bytes) = self.blobs.get(&media).await else { + continue; + }; + let text = String::from_utf8_lossy(&bytes); + let count = u32::try_from(re.find_iter(&text).count()).unwrap_or(u32::MAX); + if count == 0 { + continue; + } + let (line_no, line_text) = first_match_line(&text, re); + hits.push(Hit { + path: format!("{prefix}{}", entry.name), + token: token.as_str().to_owned(), + line: line_no, + text: line_text, + matches: count, + }); + } + + // Recurse into subdirectories, extending the path prefix. + for sub in index.subdirs() { + if let Ok(child) = Token::parse(&sub.token) { + let child_prefix = format!("{prefix}{}/", sub.name); + Box::pin(self.search_node(child, &child_prefix, pattern, re, hits, visited)).await; + } + } + } + + /// Fetch and decode a node's directory index blob. + async fn load_index(&self, node: &TreeNode) -> Result { + let bytes = self + .blobs + .get(&node.index) + .await + .map_err(|e| Envelope::err(e.to_string(), vec![]))?; + serde_json::from_slice(&bytes) + .map_err(|e| Envelope::err(format!("tree index decode: {e}"), vec![])) + } + + /// Fetch and decode a node's trigram index blob. + async fn load_trigram(&self, media: &MediaRef) -> Result { + let bytes = self + .blobs + .get(media) + .await + .map_err(|e| Envelope::err(e.to_string(), vec![]))?; + serde_json::from_slice(&bytes) + .map_err(|e| Envelope::err(format!("trigram decode: {e}"), vec![])) + } +} + +/// Rebuild the blob reference for a file entry from its recorded hash, size, and +/// type — the blob store is content-addressed, so the hash is the whole key. +fn file_media(entry: &FileEntry) -> Result { + Ok(MediaRef { + uri: CanonicalUrl::new(&format!("blob://{}", entry.sha256)) + .map_err(|e| Envelope::err(format!("blob uri: {e}"), vec![]))?, + content_type: entry.content_type.clone(), + size: entry.size, + sha256: Sha256Hex::new(&entry.sha256) + .map_err(|e| Envelope::err(format!("sha: {e}"), vec![]))?, + }) +} + +/// A pattern with no regex metacharacters — safe to feed to the trigram index and +/// Bloom directly. A metacharacter means the literal trigrams may not appear in +/// the content, so we fall back to grepping every candidate (never miss). +fn is_literal(pattern: &str) -> bool { + !pattern.chars().any(|c| ".*+?()[]{}|^$\\".contains(c)) +} + +/// The 1-based line number and text of the first line matching `re`. +fn first_match_line(text: &str, re: ®ex::Regex) -> (u32, String) { + for (i, line) in text.lines().enumerate() { + if re.is_match(line) { + let trimmed: String = line.chars().take(200).collect(); + return (u32::try_from(i + 1).unwrap_or(u32::MAX), trimmed); + } + } + (1, String::new()) +} diff --git a/crates/waggle-mcp/tests/content_access.rs b/crates/waggle-mcp/tests/content_access.rs index 561c8d5..f22f315 100644 --- a/crates/waggle-mcp/tests/content_access.rs +++ b/crates/waggle-mcp/tests/content_access.rs @@ -509,11 +509,11 @@ fn folder_targets_teach_the_lineage_pattern() { }); } -/// The folder story, end to end: --tree mints the files as snapshot -/// children; the root's funnel ROLLS UP child stages; revoking the root -/// tombstones every child (resolve AND content refuse through lineage). +/// The indexed tree, end to end (design doc: tree-scale): --tree mints a +/// hierarchy of directory NODES; the root projects its table of contents; +/// revoking the root tombstones every subtree node through lineage. #[test] -fn folder_tree_rollup_and_cascade() { +fn indexed_tree_projection_and_cascade() { pollster::block_on(async { let dir = tempfile::tempdir().unwrap(); let docs = dir.path().join("docs"); @@ -527,65 +527,49 @@ fn folder_tree_rollup_and_cascade() { let minted = handler .dispatch( "mint", - &serde_json::json!({ - "target": format!("file://{}", docs.display()), - "tree": true, - }), + &serde_json::json!({ "target": format!("file://{}", docs.display()), "tree": true }), waggle_core::Timestamp::from_unix_ms(1), &mut e, ) .await; assert!(minted.hint.is_none(), "{minted:?}"); let root = minted.result["token"].as_str().unwrap().to_owned(); - let children = minted.result["children"].as_array().unwrap(); assert_eq!( - children.len(), - 2, - "recursive, dotfiles skipped: {children:?}" + minted.result["tree"]["files"], 2, + "dotfile skipped: {minted:?}" ); - let child = children[0]["token"].as_str().unwrap().to_owned(); - // Resolving the ROOT serves the index — how a folder token - // works on a machine where the folder never existed. - let root_resolved = handler + // The projection is the directory's table of contents: one local file, + // one subdirectory (its own node token). + let toc = handler .dispatch( - "resolve", + "read", &serde_json::json!({ "token": root }), waggle_core::Timestamp::from_unix_ms(2), &mut e, ) .await; - let index = root_resolved.result["children"].as_array().unwrap(); - assert_eq!(index.len(), 2, "the folder's projection is its index"); - assert!(index[0]["target"].as_str().unwrap().starts_with("file://")); - - // Children are real snapshot tokens: grep works. - let hit = handler - .dispatch( - "search", - &serde_json::json!({ "token": child, "pattern": "bespoke" }), - waggle_core::Timestamp::from_unix_ms(2), - &mut e, - ) - .await; - assert!(hit.hint.is_none()); - assert_eq!(hit.result["total_matches"], 1); + assert!(toc.hint.is_none(), "{toc:?}"); + assert_eq!(toc.result["files"], 1, "a.md is a local file"); + assert_eq!(toc.result["subdirs"], 1, "nested is a subtree"); + assert_eq!(toc.result["total_files"], 2); + let sub = toc.result["dirs"][0]["token"].as_str().unwrap().to_owned(); - // Rollup: the root's funnel includes the child's read. - let funnel = handler + // Read one file back by name — its bytes travel with the tree. + let file = handler .dispatch( - "funnel", - &serde_json::json!({ "token": root }), + "read", + &serde_json::json!({ "token": root, "file": "a.md" }), waggle_core::Timestamp::from_unix_ms(3), &mut e, ) .await; - assert_eq!( - funnel.result["rollup"]["read"], 1, - "the folder answers for its tree: {funnel:?}" + assert!( + file.result["text"].as_str().unwrap().contains("bespoke"), + "{file:?}" ); - // One revocation, whole tree. + // One revocation on the root tombstones every subtree node through lineage. let revoked = handler .dispatch( "mutate", @@ -594,38 +578,26 @@ fn folder_tree_rollup_and_cascade() { &mut e, ) .await; - assert!(revoked.hint.is_none()); + assert!(revoked.hint.is_none(), "{revoked:?}"); let resolved = handler .dispatch( "resolve", - &serde_json::json!({ "token": child }), + &serde_json::json!({ "token": sub }), waggle_core::Timestamp::from_unix_ms(5), &mut e, ) .await; assert!( resolved.result["disposition"].get("revoked").is_some(), - "child tombstoned through lineage: {resolved:?}" - ); - let read = handler - .dispatch( - "read", - &serde_json::json!({ "token": child }), - waggle_core::Timestamp::from_unix_ms(6), - &mut e, - ) - .await; - assert!( - read.hint.expect("refusal").contains("revoked"), - "content serves nothing through a revoked lineage" + "subtree node tombstoned through lineage: {resolved:?}" ); }); } -/// Deep search: grep the ROOT token and hit every file in the tree — -/// matches grouped per file, nested files included. +/// One search over the root spans the whole nested tree, ranked, with paths — +/// and an absent pattern prunes to zero visited nodes. #[test] -fn deep_search_over_the_root_token() { +fn indexed_tree_search_spans_the_lineage() { pollster::block_on(async { let dir = tempfile::tempdir().unwrap(); let docs = dir.path().join("corpus"); @@ -633,7 +605,7 @@ fn deep_search_over_the_root_token() { std::fs::write(docs.join("plan.md"), "the needle sits here\n").unwrap(); std::fs::write( docs.join("deep/notes.md"), - "another needle, nested\nno match line\n", + "needle needle nested\nno match\n", ) .unwrap(); std::fs::write(docs.join("blank.md"), "nothing relevant\n").unwrap(); @@ -659,23 +631,40 @@ fn deep_search_over_the_root_token() { ) .await; assert!(found.hint.is_none(), "{found:?}"); - assert_eq!(found.result["total_matches"], 2, "{found:?}"); - assert_eq!(found.result["tree"]["files_searched"], 3); - let files = found.result["files"].as_array().unwrap(); - assert_eq!(files.len(), 2, "only matching files listed"); + assert_eq!( + found.result["total_matches"], 2, + "two files match: {found:?}" + ); + let matches = found.result["matches"].as_array().unwrap(); + // Ranked: the file with more matches (deep/notes.md, x2) comes first. assert!( - files - .iter() - .any(|f| f["target"].as_str().unwrap().contains("notes.md")), // sep-agnostic (Windows) - "nested files are in the tree: {files:?}" + matches[0]["path"].as_str().unwrap().contains("notes.md"), + "{matches:?}" + ); + assert!( + matches[0]["path"].as_str().unwrap().contains("deep/"), + "nested path carried" ); - // The grep→open chain points into the first matching file. assert_eq!(found.next[0].tool, "read"); + + // An absent literal prunes the whole tree from the root Bloom — 0 visited. + let none = handler + .dispatch( + "search", + &serde_json::json!({ "token": root, "pattern": "wholly_absent_zzz" }), + waggle_core::Timestamp::from_unix_ms(3), + &mut e, + ) + .await; + assert_eq!(none.result["total_matches"], 0); + assert_eq!( + none.result["nodes_visited"], 0, + "pruned at the root: {none:?}" + ); }); } -/// Tags at mint + find: discovery by what humans remember — ranked -/// candidates with disposition, never name-as-identity. +/// Tags at mint on a tree root + find: discovery by what humans remember. #[test] fn tags_and_find_discovery() { pollster::block_on(async { @@ -713,7 +702,7 @@ fn tags_and_find_discovery() { .await; let plan_token = plan.result["token"].as_str().unwrap().to_owned(); - // Find by TAG (bare tag became name=design_docs). + // Find by TAG (the tree root carries it). let by_tag = handler .dispatch( "find", @@ -728,7 +717,7 @@ fn tags_and_find_discovery() { "design_docs" ); - // Find by BASENAME; newest-first ranking; executable next. + // Find by BASENAME; newest-first; executable next. let by_name = handler .dispatch( "find", @@ -740,7 +729,7 @@ fn tags_and_find_discovery() { assert_eq!(by_name.result["candidates"][0]["token"], plan_token); assert_eq!(by_name.next[0].tool, "resolve"); - // Disposition is visible: a revoked candidate says so. + // A revoked candidate is VISIBLY dead. handler .dispatch( "mutate", @@ -759,23 +748,22 @@ fn tags_and_find_discovery() { .await; assert_eq!( after.result["candidates"][0]["disposition"], "revoked", - "a dead name is VISIBLY dead: {after:?}" + "{after:?}" ); }); } -/// The folder-review proof: coverage on a tree names exactly which -/// files a review MISSED — and a root deep-search honestly counts as -/// reading every file it scanned. +/// Node-granular coverage on a tree: reading one subtree's file marks that node +/// read; a search across the tree marks every node it served. #[test] -fn coverage_proves_the_folder_was_read_or_names_the_gaps() { +fn coverage_is_node_granular_over_a_tree() { pollster::block_on(async { let dir = tempfile::tempdir().unwrap(); let docs = dir.path().join("review_me"); - std::fs::create_dir_all(&docs).unwrap(); - for name in ["a.md", "b.md", "c.md"] { - std::fs::write(docs.join(name), format!("content of {name}\n")).unwrap(); - } + std::fs::create_dir_all(docs.join("x")).unwrap(); + std::fs::create_dir_all(docs.join("y")).unwrap(); + std::fs::write(docs.join("x/a.md"), "content alpha\n").unwrap(); + std::fs::write(docs.join("y/b.md"), "content beta\n").unwrap(); let handler = handler_with_blobs(dir.path()); let mut e = entropy(); let minted = handler @@ -787,79 +775,67 @@ fn coverage_proves_the_folder_was_read_or_names_the_gaps() { ) .await; let root = minted.result["token"].as_str().unwrap().to_owned(); - let children = minted.result["children"].as_array().unwrap().clone(); - - // A lazy review: reads a.md and b.md, never opens c.md. - for child in children.iter().take(2) { - let token = child["token"].as_str().unwrap(); - handler - .dispatch( - "read", - &serde_json::json!({ "token": token }), - waggle_core::Timestamp::from_unix_ms(2), - &mut e, - ) - .await; - } - let audit = handler + let toc = handler .dispatch( - "coverage", + "read", &serde_json::json!({ "token": root }), - waggle_core::Timestamp::from_unix_ms(3), + waggle_core::Timestamp::from_unix_ms(2), &mut e, ) .await; - assert!(audit.hint.is_none(), "{audit:?}"); - assert_eq!(audit.result["read"], "2/3", "{audit:?}"); - assert_eq!(audit.result["complete"], false); - assert!( - audit.result["unread"][0]["target"] - .as_str() - .unwrap() - .ends_with("c.md"), - "the miss is NAMED: {audit:?}" - ); - assert_eq!(audit.next[0].tool, "read", "next closes the gap"); + let x = toc.result["dirs"] + .as_array() + .unwrap() + .iter() + .find(|d| d["name"] == "x") + .unwrap()["token"] + .as_str() + .unwrap() + .to_owned(); - // A root deep-search touches every file — honest per-child reads. + // Read x's file only. Coverage: some nodes read, not all → incomplete. handler .dispatch( - "search", - &serde_json::json!({ "token": root, "pattern": "content" }), - waggle_core::Timestamp::from_unix_ms(4), + "read", + &serde_json::json!({ "token": x, "file": "a.md" }), + waggle_core::Timestamp::from_unix_ms(3), &mut e, ) .await; - let after = handler + let audit = handler .dispatch( "coverage", &serde_json::json!({ "token": root }), - waggle_core::Timestamp::from_unix_ms(5), + waggle_core::Timestamp::from_unix_ms(4), &mut e, ) .await; - assert_eq!(after.result["read"], "3/3", "{after:?}"); - assert_eq!(after.result["complete"], true); + assert!(audit.hint.is_none(), "{audit:?}"); + assert_eq!(audit.result["kind"], "tree"); + assert_eq!(audit.result["total_files"], 2); + assert_eq!(audit.result["complete"], false, "y not read yet: {audit:?}"); + assert!(!audit.result["unread_nodes"].as_array().unwrap().is_empty()); - // The STRONG bar: run stays honest — nothing recorded use yet. - assert_eq!(after.result["run"], "0/3"); - let child0 = children[0]["token"].as_str().unwrap(); + // A search across the tree serves both files → every node read. handler .dispatch( - "record", - &serde_json::json!({ "token": child0, "stage": "run" }), - waggle_core::Timestamp::from_unix_ms(6), + "search", + &serde_json::json!({ "token": root, "pattern": "content" }), + waggle_core::Timestamp::from_unix_ms(5), &mut e, ) .await; - let strong = handler + let after = handler .dispatch( "coverage", &serde_json::json!({ "token": root }), - waggle_core::Timestamp::from_unix_ms(7), + waggle_core::Timestamp::from_unix_ms(6), &mut e, ) .await; - assert_eq!(strong.result["run"], "1/3"); + assert_eq!( + after.result["complete"], true, + "search touched every node: {after:?}" + ); }); } diff --git a/crates/waggle-mcp/tests/gap_fixes.rs b/crates/waggle-mcp/tests/gap_fixes.rs index d6bb33c..b249056 100644 --- a/crates/waggle-mcp/tests/gap_fixes.rs +++ b/crates/waggle-mcp/tests/gap_fixes.rs @@ -90,13 +90,27 @@ fn tree_mint_denies_generated_and_vendored_dirs() { ) .await; assert!(minted.hint.is_none(), "{minted:?}"); - let children = minted.result["children"].as_array().unwrap(); - assert_eq!(children.len(), 1, "only src/lib.rs survives: {children:?}"); - // Separator-agnostic: Windows targets carry backslashes. - assert!(children[0]["target"] - .as_str() + // node_modules/ and target/ are denied — only src/lib.rs survives. + assert_eq!( + minted.result["tree"]["files"], 1, + "only src/lib.rs: {minted:?}" + ); + let root = minted.result["token"].as_str().unwrap().to_owned(); + let toc = h + .dispatch( + "read", + &json!({ "token": root }), + Timestamp::from_unix_ms(2), + &mut e, + ) + .await; + // The one surviving file lives under src/, reachable through the projection. + assert_eq!(toc.result["total_files"], 1); + let src = toc.result["dirs"] + .as_array() .unwrap() - .replace('\\', "/") - .ends_with("src/lib.rs")); + .iter() + .find(|d| d["name"] == "src"); + assert!(src.is_some(), "src/ is the only subdir: {toc:?}"); }); } From 092313f7a5159c04b517cd10321c1ee7b206d86a Mon Sep 17 00:00:00 2001 From: Chetan Conikee Date: Mon, 13 Jul 2026 15:00:43 -0700 Subject: [PATCH 05/12] =?UTF-8?q?core+mcp:=20per-file=20tree=20coverage=20?= =?UTF-8?q?via=20Event.entry=20=E2=80=94=20a=20position,=20not=20a=20paylo?= =?UTF-8?q?ad?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Coverage on an indexed tree was node-granular ("the folder was touched"). Too coarse for the product's own promise — "see exactly what your AI read" should mean WHICH FILES. This makes it per-file without breaking I-1. The unlock: I-1 forbids bytes in the log, not positions into a signed manifest. The log already carries two such integers, both I-1-safe by their own docstrings — `variant` (which manifest variant served) and `regions` (which contract line-ranges a read touched). A tree node's DirIndex is a sorted, signed, immutable file list, so "file #k of this node was read" is the same species of object: meaningless without the signed manifest, therefore leaks nothing. The only blocker was the 8-bit width of `regions` — a width limit, not an I-1 limit. core Event.entry: Option — the ordinal, in the owning node's signed DirIndex file order, of the one file a read served. Additive and serde(default, skip): old wire frames keep parsing, ordinary non-tree traffic never carries it, and records persist as JSON payload so there is NO schema migration. EntryTouchFold unions the touched ordinals per token; union is commutative + idempotent, the same algebra that gives RegionTouchFold its replay tolerance, so R-1/R-3/C-8 come for free. mcp read --file stamps the file's ordinal; search stamps each confirmed match's ordinal (carried on waggle_tree::Hit.entry, a stable position in DirIndex.files()). Tree coverage replays each file-bearing node (the same scan_token + replay path contract_coverage uses), counts |touched ∩ [0, local_files)|, and sums across the lineage → files read / total at true per-file granularity, with per-node unread counts and the first missing file NAMED. Deliberately out of scope: soa.rs (the 7-column fixed-width funnel/region accelerator) gains no entry column — per-file coverage rides the replay path, not the columnar one, and a u32 ordinal would muddy that structure's narrow-width invariant for no gain. Test rewritten: a node with two files, one read, reports "1/3" and names the file nobody opened; a search that serves every file closes it to "3/3". Fold unit test mirrors the region test (union commutative + duplicate-immune). No token-per-file, so the 200-file cap stays lifted. All workspace tests pass; clippy clean; every file <= 750 lines. Design log: waggle-tree-scale/DECISIONS.md. --- bench/src/determinism.rs | 1 + bench/src/tier2.rs | 1 + crates/waggle-core/benches/hot_paths.rs | 1 + crates/waggle-core/src/event.rs | 9 ++ crates/waggle-core/src/fold.rs | 55 +++++++++- crates/waggle-core/src/lib.rs | 3 +- crates/waggle-core/src/reconstruct.rs | 1 + crates/waggle-core/src/soa.rs | 1 + crates/waggle-core/tests/event_sourcing.rs | 2 + crates/waggle-mcp/src/content_handlers.rs | 15 +++ crates/waggle-mcp/src/handlers.rs | 4 +- crates/waggle-mcp/src/lineage.rs | 102 ++++++++++++------ crates/waggle-mcp/src/record.rs | 1 + crates/waggle-mcp/src/tree_read.rs | 24 +++-- crates/waggle-mcp/tests/content_access.rs | 39 ++++--- crates/waggle-store-cloudflare/src/engine.rs | 2 + .../tests/e_matrix_native.rs | 2 + crates/waggle-store-fs-jsonl/src/lib.rs | 2 + .../tests/conformance.rs | 1 + .../benches/store_paths.rs | 1 + crates/waggle-store-sqlite/src/store.rs | 8 +- .../waggle-store-sqlite/tests/conformance.rs | 1 + .../waggle-store-sqlite/tests/crash_matrix.rs | 2 + crates/waggle-store-sqlite/tests/g_suite.rs | 1 + crates/waggle-store/src/conformance.rs | 1 + crates/waggle-store/src/memory.rs | 2 + crates/waggle-store/src/types.rs | 5 + crates/waggle-tree/src/search.rs | 5 + edge-worker/tests/miniflare.rs | 1 + 29 files changed, 236 insertions(+), 57 deletions(-) diff --git a/bench/src/determinism.rs b/bench/src/determinism.rs index 268f708..65ec23a 100644 --- a/bench/src/determinism.rs +++ b/bench/src/determinism.rs @@ -76,6 +76,7 @@ fn build_log(k: usize, events_per: usize) -> Vec { seq: Seq((e + 1) as u32), variant: None, regions: None, + entry: None, })); } } diff --git a/bench/src/tier2.rs b/bench/src/tier2.rs index 1fc926f..384cb25 100644 --- a/bench/src/tier2.rs +++ b/bench/src/tier2.rs @@ -111,6 +111,7 @@ fn permille_for(contract: &Contract, bits: u8, token: Token) -> u16 { seq: Seq(1), variant: None, regions: Some(1u8 << i), + entry: None, }) }) .collect(); diff --git a/crates/waggle-core/benches/hot_paths.rs b/crates/waggle-core/benches/hot_paths.rs index 3331545..811b287 100644 --- a/crates/waggle-core/benches/hot_paths.rs +++ b/crates/waggle-core/benches/hot_paths.rs @@ -165,6 +165,7 @@ fn bench_fold_1m(c: &mut Criterion) { seq: Seq(i), variant: None, regions: None, + entry: None, }, &mut tables, ); diff --git a/crates/waggle-core/src/event.rs b/crates/waggle-core/src/event.rs index 0c1a13e..1d20199 100644 --- a/crates/waggle-core/src/event.rs +++ b/crates/waggle-core/src/event.rs @@ -146,6 +146,14 @@ pub struct Event { /// contract-free tokens and on pre-contract logs. #[serde(default, skip_serializing_if = "Option::is_none")] pub regions: Option, + /// For a `read` of one file inside an indexed tree node: the ordinal of + /// that file in the node's signed [`crate::TreeNode`] directory index + /// (`DirIndex.files()` order). Manifest-referencing exactly like + /// [`Self::regions`] — a *position* into a signed, immutable declaration, + /// never bytes — so it stays I-1-compatible while giving per-file coverage. + /// Absent on non-tree traffic and on pre-tree logs. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub entry: Option, } #[cfg(test)] @@ -212,6 +220,7 @@ mod tests { seq: Seq(4), variant: Some(2), regions: Some(0b101), + entry: Some(7), }; let json = serde_json::to_string(&e).unwrap(); let back: Event = serde_json::from_str(&json).unwrap(); diff --git a/crates/waggle-core/src/fold.rs b/crates/waggle-core/src/fold.rs index e2956f4..fded1b7 100644 --- a/crates/waggle-core/src/fold.rs +++ b/crates/waggle-core/src/fold.rs @@ -4,7 +4,7 @@ //! pass** over the log; adding an analytic is a new `Fold` impl, never a //! new scan. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use crate::log::LogRecord; use crate::manifest::AttributionManifest; @@ -141,6 +141,28 @@ impl Fold for RegionTouchFold { } } +/// Accumulates the set of tree-node file ordinals a token's reads have +/// touched: the UNION of every event's [`crate::Event::entry`]. Union is +/// commutative and idempotent — like the OR in [`RegionTouchFold`] — so +/// R-1/R-3 (order- and duplicate-tolerance) and C-8 hold for free. Tree +/// coverage evaluates `|touched ∩ [0, node.files)|` against each node's +/// signed directory index for a true per-file receipt. +#[derive(Debug, Default)] +pub struct EntryTouchFold { + /// Token → the set of `DirIndex` file ordinals its reads have reached. + pub per_token: BTreeMap>, +} + +impl Fold for EntryTouchFold { + fn apply(&mut self, record: &LogRecord) { + if let LogRecord::Event(e) = record { + if let Some(ordinal) = e.entry { + self.per_token.entry(e.token).or_default().insert(ordinal); + } + } + } +} + /// The delegation forest: parent → children, from `Minted` records /// (doc `06 §4` — coordination trace, attribution roll-up, cascade path). #[derive(Debug, Default)] @@ -200,6 +222,7 @@ mod tests { seq: Seq(seq), variant: None, regions: None, + entry: None, }) } @@ -308,6 +331,7 @@ mod tests { seq: Seq(seq), variant: None, regions: Some(bits), + entry: None, }) }; let records = vec![ @@ -324,6 +348,35 @@ mod tests { assert_eq!(a.per_token, b.per_token, "OR is commutative — R-1"); } + #[test] + fn entry_touches_union_together_shuffle_and_duplicate_immune() { + let m = minted(9, None); + let touch = |ordinal: u32, seq: u32| { + LogRecord::Event(Event { + token: m.token, + stage: Stage::read(), + actor: ActorClass::from_context(&ResolverContext::anonymous_agent()), + at: Timestamp::from_unix_ms(2), + seq: Seq(seq), + variant: None, + regions: None, + entry: Some(ordinal), + }) + }; + let records = vec![ + touch(2, 1), + event(m.token, Stage::read(), 2), // no entry: whole-node access + touch(0, 3), + touch(2, 1), // duplicate record — the set absorbs it (R-3) + ]; + let mut shuffled = records.clone(); + shuffled.reverse(); + let a = replay(records, EntryTouchFold::default()); + let b = replay(shuffled, EntryTouchFold::default()); + assert_eq!(a.per_token[&m.token], BTreeSet::from([0, 2])); + assert_eq!(a.per_token, b.per_token, "union is commutative — R-1"); + } + #[test] fn mutation_before_mint_is_ignored_not_a_panic() { // Hostile/misordered input: folds are total (reconstruct orders diff --git a/crates/waggle-core/src/lib.rs b/crates/waggle-core/src/lib.rs index 033864d..b1707e1 100644 --- a/crates/waggle-core/src/lib.rs +++ b/crates/waggle-core/src/lib.rs @@ -60,7 +60,8 @@ pub use contract::{ pub use entropy::{Entropy, EntropyError}; pub use event::{ActorClass, Event, FamilyClass, HarnessClass, Seq}; pub use fold::{ - outcome_of, replay, Fold, FunnelFold, LineageFold, ManifestFold, Outcome, RegionTouchFold, + outcome_of, replay, EntryTouchFold, Fold, FunnelFold, LineageFold, ManifestFold, Outcome, + RegionTouchFold, }; pub use log::{Change, LogRecord}; pub use manifest::{ diff --git a/crates/waggle-core/src/reconstruct.rs b/crates/waggle-core/src/reconstruct.rs index 617c37d..4c914db 100644 --- a/crates/waggle-core/src/reconstruct.rs +++ b/crates/waggle-core/src/reconstruct.rs @@ -131,6 +131,7 @@ mod tests { seq: Seq(0), // permissive producer variant: None, regions: None, + entry: None, }), LogRecord::Minted { manifest: Box::new(m), diff --git a/crates/waggle-core/src/soa.rs b/crates/waggle-core/src/soa.rs index 0058a61..b97aa5f 100644 --- a/crates/waggle-core/src/soa.rs +++ b/crates/waggle-core/src/soa.rs @@ -146,6 +146,7 @@ mod tests { seq: Seq(seq), variant: if seq % 2 == 0 { Some(1) } else { None }, regions: None, + entry: None, } } diff --git a/crates/waggle-core/tests/event_sourcing.rs b/crates/waggle-core/tests/event_sourcing.rs index 557b684..b19c631 100644 --- a/crates/waggle-core/tests/event_sourcing.rs +++ b/crates/waggle-core/tests/event_sourcing.rs @@ -54,6 +54,7 @@ fn world(n_tokens: u32, events_per_token: u32) -> Vec { seq: Seq(i + 1), variant: Some(u8::try_from(i % 3).unwrap()), regions: None, + entry: None, })); } } @@ -158,6 +159,7 @@ fn fold_funnel_1m_shape() { seq: Seq(i), variant: None, regions: None, + entry: None, }, &mut tables, ); diff --git a/crates/waggle-mcp/src/content_handlers.rs b/crates/waggle-mcp/src/content_handlers.rs index 1cde92f..8931fc2 100644 --- a/crates/waggle-mcp/src/content_handlers.rs +++ b/crates/waggle-mcp/src/content_handlers.rs @@ -265,6 +265,20 @@ impl Handler { /// served bytes touched (`None` when contract-free or untouched — /// the field never appears for ordinary traffic). pub(crate) async fn record_read(&self, token: Token, now: Timestamp, regions: Option) { + self.record_read_entry(token, now, regions, None).await; + } + + /// Record a `read`, additionally stamping the tree-node file ordinal this + /// read served (`entry`) so per-file coverage can roll it up. `entry` is a + /// position into the node's signed directory index — I-1-safe like + /// `regions` — and is `None` for all non-tree traffic. + pub(crate) async fn record_read_entry( + &self, + token: Token, + now: Timestamp, + regions: Option, + entry: Option, + ) { let _ = self .store .append(AppendIntent::Event { @@ -273,6 +287,7 @@ impl Handler { actor: ActorClass::from_context(&ResolverContext::anonymous_agent()), variant: None, regions, + entry, at: now, }) .await; diff --git a/crates/waggle-mcp/src/handlers.rs b/crates/waggle-mcp/src/handlers.rs index 4a1ed01..9e9bb67 100644 --- a/crates/waggle-mcp/src/handlers.rs +++ b/crates/waggle-mcp/src/handlers.rs @@ -413,8 +413,7 @@ impl Handler { let variant_index = resolution.variant.as_ref().map(|s| s.index); let body = resolution.variant.as_ref().map(|s| &s.variant.body); - // Record the resolve — the host's separate act (I-4), done here at - // the transport layer, never inside core resolve(). + // Record the resolve — the host's separate act (I-4), at the transport layer. let recorded = self .store .append(AppendIntent::Event { @@ -423,6 +422,7 @@ impl Handler { actor: ActorClass::from_context(&ctx), variant: variant_index, regions: None, + entry: None, at: now, }) .await; diff --git a/crates/waggle-mcp/src/lineage.rs b/crates/waggle-mcp/src/lineage.rs index 8a47e39..70bd83b 100644 --- a/crates/waggle-mcp/src/lineage.rs +++ b/crates/waggle-mcp/src/lineage.rs @@ -100,11 +100,10 @@ impl Handler { Err(e) => return store_err(&e), }; // An indexed tree (design doc: tree-scale) is a hierarchy of directory - // NODES, not per-file tokens, so its coverage is node-granular: how many - // directory nodes were touched, out of the total, plus the true file count - // from the root's index. (Per-file `files:all` coverage needs a served-set - // the payload-free log cannot express in an 8-bit region mask — a - // documented follow-up.) + // NODES, not per-file tokens. Coverage is nonetheless PER-FILE: each read + // stamps the file's ordinal in its node's signed directory index + // (`Event.entry` — a position, not a payload, so I-1-safe), and the rollup + // unions those ordinals per node and sums across the lineage. if view.as_ref().is_some_and(|v| v.manifest.tree.is_some()) { return self.tree_node_coverage(root, view.as_ref().unwrap()).await; } @@ -191,10 +190,12 @@ impl Handler { }) } - /// Node-granular coverage for an indexed tree: walk the directory-node - /// lineage and report how many nodes have been read, the total node count, and - /// the true file/byte totals from the root's index. A node is "read" once any - /// of its files was served (search hit or `read --file`). + /// Per-file coverage for an indexed tree: walk the directory-node lineage and, + /// for each node, union the file ordinals its reads have touched + /// (`EntryTouchFold` over the node's records) against its signed directory + /// index. Sum across the lineage for a true `files read / total` receipt, with + /// per-node unread counts and the first missing file names. Pure-container + /// nodes (subdirs only) hold nothing of their own to read and are skipped. async fn tree_node_coverage( &self, root: waggle_core::Token, @@ -208,7 +209,9 @@ impl Handler { let mut queue: std::collections::VecDeque<_> = [root] .into_iter() .collect::>(); - let (mut nodes, mut read, mut visited) = (0u64, 0u64, 0); + // nodes/read_nodes count file-bearing NODES; files/read count FILES. + let (mut nodes, mut read_nodes, mut visited) = (0u64, 0u64, 0); + let (mut files, mut read) = (0u64, 0u64); let mut unread: Vec = Vec::new(); while let Some(node) = queue.pop_front() { visited += 1; @@ -221,57 +224,88 @@ impl Handler { let Ok(Some(v)) = self.store.manifest(node).await else { continue; }; - let Some(tn) = v.manifest.tree.as_ref() else { + if v.manifest.tree.is_none() { continue; - }; - // Only file-bearing nodes count: a pure container (subdirs only) has - // nothing of its own to read, so it can neither be read nor block - // completeness — it is covered when its children are. - if !self.node_has_local_files(tn).await { + } + let Some(index) = self.load_dir_index(&v.manifest).await else { continue; + }; + let names: Vec<&str> = index.files().map(|f| f.name.as_str()).collect(); + if names.is_empty() { + continue; // pure container — covered when its children are } nodes += 1; - let (was_read, _) = self.child_consumption(node).await; - if was_read { - read += 1; - } else if unread.len() < 20 { + files += names.len() as u64; + let touched = self.entries_touched(node).await; + let here = touched + .iter() + .filter(|&&e| (e as usize) < names.len()) + .count() as u64; + read += here; + if here > 0 { + read_nodes += 1; + } + if here < names.len() as u64 && unread.len() < 20 { + let missing: Vec<&str> = names + .iter() + .enumerate() + .filter(|(i, _)| !touched.contains(&u32::try_from(*i).unwrap_or(u32::MAX))) + .map(|(_, n)| *n) + .take(8) + .collect(); unread.push(json!({ "token": node.as_str(), "target": v.manifest.target.as_str(), + "unread_files": names.len() as u64 - here, + "first_missing": missing, })); } } + let complete = files > 0 && read == files; Envelope::ok( json!({ "token": root.as_str(), "kind": "tree", - "nodes": format!("{read}/{nodes}"), + "files": format!("{read}/{files}"), + "nodes": format!("{read_nodes}/{nodes}"), "total_files": total_files, "total_bytes": total_bytes, - "complete": unread.is_empty(), - "unread_nodes": unread, + "complete": complete, + "unread": unread, }), vec![NextCall { tool: "search".into(), args: json!({ "token": root.as_str(), "pattern": "" }), - why: "reading through search stamps the nodes it serves".into(), + why: "reading files (search hits or read --file) stamps per-file coverage".into(), }], ) .with_stats(Stats { - records: Some(nodes), + records: Some(files), seq: None, }) } - /// Does this node have files of its own (not just subdirectories)? Reads the - /// node's directory index; a fetch failure conservatively counts as "yes" so a - /// node is never silently dropped from coverage. - async fn node_has_local_files(&self, node: &waggle_core::TreeNode) -> bool { - match self.blobs.get(&node.index).await { - Ok(bytes) => serde_json::from_slice::(&bytes) - .map_or(true, |idx| idx.files().next().is_some()), - Err(_) => true, - } + /// Fetch and decode a tree node's signed directory index, if it is a tree + /// node whose index blob is present. + async fn load_dir_index( + &self, + manifest: &waggle_core::AttributionManifest, + ) -> Option { + let node = manifest.tree.as_ref()?; + let bytes = self.blobs.get(&node.index).await.ok()?; + serde_json::from_slice::(&bytes).ok() + } + + /// The set of file ordinals a node's reads have touched — the union fold of + /// `Event.entry` over the node's own records (I-1-safe positions, not bytes). + async fn entries_touched(&self, node: waggle_core::Token) -> std::collections::BTreeSet { + let Ok(records) = self.store.scan_token(node, waggle_core::Seq(0)).await else { + return std::collections::BTreeSet::new(); + }; + waggle_core::replay(records, waggle_core::EntryTouchFold::default()) + .per_token + .remove(&node) + .unwrap_or_default() } /// Single-token contract coverage (19 §4.2): fold the region-touch diff --git a/crates/waggle-mcp/src/record.rs b/crates/waggle-mcp/src/record.rs index bdd302a..b13fa03 100644 --- a/crates/waggle-mcp/src/record.rs +++ b/crates/waggle-mcp/src/record.rs @@ -34,6 +34,7 @@ impl Handler { actor: ActorClass::from_context(&ResolverContext::anonymous_agent()), variant: None, regions: None, + entry: None, at: now, }) .await; diff --git a/crates/waggle-mcp/src/tree_read.rs b/crates/waggle-mcp/src/tree_read.rs index 6cb03b1..faf5a47 100644 --- a/crates/waggle-mcp/src/tree_read.rs +++ b/crates/waggle-mcp/src/tree_read.rs @@ -96,7 +96,7 @@ impl Handler { Ok(i) => i, Err(e) => return e, }; - let Some(entry) = index.files().find(|f| f.name == name) else { + let Some((ordinal, entry)) = index.files().enumerate().find(|(_, f)| f.name == name) else { let names: Vec<&str> = index.files().map(|f| f.name.as_str()).take(20).collect(); return Envelope::err( format!("tree: no file `{name}` in this directory — files: {names:?}"), @@ -111,9 +111,16 @@ impl Handler { Ok(b) => b, Err(e) => return Envelope::err(e.to_string(), vec![]), }; - // A file read is real consumption — stamp it against this node so a - // files:all coverage can roll it up. - self.record_read(token, now, None).await; + // A file read is real consumption — stamp it against this node WITH the + // file's ordinal, so per-file coverage rolls it up (a position into the + // signed directory index, not a payload — I-1-safe). + self.record_read_entry( + token, + now, + None, + Some(u32::try_from(ordinal).unwrap_or(u32::MAX)), + ) + .await; let text = String::from_utf8_lossy(&bytes); Envelope::ok( json!({ @@ -151,10 +158,12 @@ impl Handler { let mut visited = 0usize; Box::pin(self.search_node(root, "", pattern, &re, &mut hits, &mut visited)).await; let ranked = tsearch::rank(hits, limit); - // Each confirmed match served that file's bytes — record it once. + // Each confirmed match served that file's bytes — record it once, with + // the file's ordinal in its node's directory index so per-file coverage + // rolls it up. for h in &ranked { if let Ok(t) = Token::parse(&h.token) { - self.record_read(t, now, None).await; + self.record_read_entry(t, now, None, Some(h.entry)).await; } } let matches: Vec = ranked @@ -242,6 +251,9 @@ impl Handler { hits.push(Hit { path: format!("{prefix}{}", entry.name), token: token.as_str().to_owned(), + // `i` is the file's position in this node's `DirIndex.files()` + // order — the same ordinal per-file coverage counts against. + entry: u32::try_from(i).unwrap_or(u32::MAX), line: line_no, text: line_text, matches: count, diff --git a/crates/waggle-mcp/tests/content_access.rs b/crates/waggle-mcp/tests/content_access.rs index f22f315..30fed93 100644 --- a/crates/waggle-mcp/tests/content_access.rs +++ b/crates/waggle-mcp/tests/content_access.rs @@ -753,16 +753,20 @@ fn tags_and_find_discovery() { }); } -/// Node-granular coverage on a tree: reading one subtree's file marks that node -/// read; a search across the tree marks every node it served. +/// Per-file coverage on a tree: a node with two files, only one of which is read, +/// reports `1/3` — not "the folder was touched" — and NAMES the file nobody +/// opened. A search that serves every file then closes it to `3/3`. #[test] -fn coverage_is_node_granular_over_a_tree() { +fn coverage_is_per_file_over_a_tree() { pollster::block_on(async { let dir = tempfile::tempdir().unwrap(); let docs = dir.path().join("review_me"); std::fs::create_dir_all(docs.join("x")).unwrap(); std::fs::create_dir_all(docs.join("y")).unwrap(); + // Node x holds TWO files; node y holds one. All share "content" so a + // search matches every file; each has a unique word too. std::fs::write(docs.join("x/a.md"), "content alpha\n").unwrap(); + std::fs::write(docs.join("x/c.md"), "content gamma\n").unwrap(); std::fs::write(docs.join("y/b.md"), "content beta\n").unwrap(); let handler = handler_with_blobs(dir.path()); let mut e = entropy(); @@ -793,7 +797,8 @@ fn coverage_is_node_granular_over_a_tree() { .unwrap() .to_owned(); - // Read x's file only. Coverage: some nodes read, not all → incomplete. + // Read ONE of x's two files. Per-file coverage: 1 of 3 read, incomplete, + // and the miss is named (c.md in node x, plus y/b.md untouched). handler .dispatch( "read", @@ -812,11 +817,23 @@ fn coverage_is_node_granular_over_a_tree() { .await; assert!(audit.hint.is_none(), "{audit:?}"); assert_eq!(audit.result["kind"], "tree"); - assert_eq!(audit.result["total_files"], 2); - assert_eq!(audit.result["complete"], false, "y not read yet: {audit:?}"); - assert!(!audit.result["unread_nodes"].as_array().unwrap().is_empty()); + assert_eq!(audit.result["total_files"], 3); + assert_eq!(audit.result["files"], "1/3", "one file read: {audit:?}"); + assert_eq!(audit.result["complete"], false); + // Node x is partially read: one of its two files is still missing, named. + let unread = audit.result["unread"].as_array().unwrap(); + let x_gap = unread + .iter() + .find(|u| u["token"] == x) + .expect("x is partially read"); + assert_eq!(x_gap["unread_files"], 1, "c.md still missing: {audit:?}"); + assert!(x_gap["first_missing"] + .as_array() + .unwrap() + .iter() + .any(|n| n == "c.md")); - // A search across the tree serves both files → every node read. + // A search across the tree serves every file → 3/3, complete. handler .dispatch( "search", @@ -833,9 +850,7 @@ fn coverage_is_node_granular_over_a_tree() { &mut e, ) .await; - assert_eq!( - after.result["complete"], true, - "search touched every node: {after:?}" - ); + assert_eq!(after.result["files"], "3/3", "search served all: {after:?}"); + assert_eq!(after.result["complete"], true); }); } diff --git a/crates/waggle-store-cloudflare/src/engine.rs b/crates/waggle-store-cloudflare/src/engine.rs index 4be49fd..60be6b2 100644 --- a/crates/waggle-store-cloudflare/src/engine.rs +++ b/crates/waggle-store-cloudflare/src/engine.rs @@ -231,6 +231,7 @@ impl AppendStore for EdgeStore { actor, variant, regions, + entry, at, } => { if self.load_manifest(token).await?.is_none() { @@ -246,6 +247,7 @@ impl AppendStore for EdgeStore { seq, variant, regions, + entry, })) .await?; Ok(Appended::Event { seq }) diff --git a/crates/waggle-store-cloudflare/tests/e_matrix_native.rs b/crates/waggle-store-cloudflare/tests/e_matrix_native.rs index 9d81fd1..71eef00 100644 --- a/crates/waggle-store-cloudflare/tests/e_matrix_native.rs +++ b/crates/waggle-store-cloudflare/tests/e_matrix_native.rs @@ -95,6 +95,7 @@ fn e6_native_edge_equals_local() { ), variant: None, regions: None, + entry: None, at: waggle_core::Timestamp::from_unix_ms(u64::from(step)), }; let a = edge.append(mk()).await; @@ -218,6 +219,7 @@ fn e4_native_replay_migration_rehearsal() { ), variant: None, regions: None, + entry: None, at: waggle_core::Timestamp::from_unix_ms(2), }) .await diff --git a/crates/waggle-store-fs-jsonl/src/lib.rs b/crates/waggle-store-fs-jsonl/src/lib.rs index 6cf10ed..b1603e8 100644 --- a/crates/waggle-store-fs-jsonl/src/lib.rs +++ b/crates/waggle-store-fs-jsonl/src/lib.rs @@ -117,6 +117,7 @@ fn record_of(intent: &AppendIntent, receipt: &Appended) -> Option { actor, variant, regions, + entry, at, }, Appended::Event { seq }, @@ -128,6 +129,7 @@ fn record_of(intent: &AppendIntent, receipt: &Appended) -> Option { seq: *seq, variant: *variant, regions: *regions, + entry: *entry, })), // Mint replays journal nothing (the original line already exists); // mismatched intent/receipt pairs cannot occur but journal nothing. diff --git a/crates/waggle-store-fs-jsonl/tests/conformance.rs b/crates/waggle-store-fs-jsonl/tests/conformance.rs index 1547f8c..a9b2f58 100644 --- a/crates/waggle-store-fs-jsonl/tests/conformance.rs +++ b/crates/waggle-store-fs-jsonl/tests/conformance.rs @@ -61,6 +61,7 @@ fn reopen_replays_the_journal() { ), variant: None, regions: None, + entry: None, at: waggle_core::Timestamp::from_unix_ms(2), }) .await diff --git a/crates/waggle-store-sqlite/benches/store_paths.rs b/crates/waggle-store-sqlite/benches/store_paths.rs index 081855a..7188045 100644 --- a/crates/waggle-store-sqlite/benches/store_paths.rs +++ b/crates/waggle-store-sqlite/benches/store_paths.rs @@ -60,6 +60,7 @@ fn bench_append(c: &mut Criterion) { actor, variant: None, regions: None, + entry: None, at: waggle_core::Timestamp::from_unix_ms(2), })) .unwrap() diff --git a/crates/waggle-store-sqlite/src/store.rs b/crates/waggle-store-sqlite/src/store.rs index 6b23fdd..0893503 100644 --- a/crates/waggle-store-sqlite/src/store.rs +++ b/crates/waggle-store-sqlite/src/store.rs @@ -170,8 +170,9 @@ impl AppendStore for SqliteStore { actor, variant, regions, + entry, at, - } => event_tx(&tx, token, &stage, actor, variant, regions, at)?, + } => event_tx(&tx, token, &stage, actor, variant, regions, entry, at)?, }; tx.commit().map_err(sql)?; // Cache maintenance strictly after commit: never serve uncommitted. @@ -324,6 +325,9 @@ fn mutate_tx( )) } +// The args mirror the `AppendIntent::Event` variant's fields 1:1; bundling them +// into a throwaway struct would only relocate the same fields. +#[allow(clippy::too_many_arguments)] fn event_tx( tx: &Connection, token: Token, @@ -331,6 +335,7 @@ fn event_tx( actor: waggle_core::ActorClass, variant: Option, regions: Option, + entry: Option, at: waggle_core::Timestamp, ) -> Result<(Appended, Touched), StoreError> { if load_manifest(tx, token)?.is_none() { @@ -353,6 +358,7 @@ fn event_tx( seq, variant, regions, + entry, }), )?; Ok((Appended::Event { seq }, Vec::new())) diff --git a/crates/waggle-store-sqlite/tests/conformance.rs b/crates/waggle-store-sqlite/tests/conformance.rs index a599a77..e6816e8 100644 --- a/crates/waggle-store-sqlite/tests/conformance.rs +++ b/crates/waggle-store-sqlite/tests/conformance.rs @@ -39,6 +39,7 @@ fn event_intent(token: waggle_core::Token, stage: Stage) -> AppendIntent { ), variant: None, regions: None, + entry: None, at: waggle_core::Timestamp::from_unix_ms(4), } } diff --git a/crates/waggle-store-sqlite/tests/crash_matrix.rs b/crates/waggle-store-sqlite/tests/crash_matrix.rs index a2aa4f3..ee41dbc 100644 --- a/crates/waggle-store-sqlite/tests/crash_matrix.rs +++ b/crates/waggle-store-sqlite/tests/crash_matrix.rs @@ -64,6 +64,7 @@ fn crash_child_workload() { actor, variant: None, regions: None, + entry: None, at: waggle_core::Timestamp::from_unix_ms(2), })) .unwrap(); @@ -149,6 +150,7 @@ fn killed_writers_never_lose_an_acked_write() { ), variant: None, regions: None, + entry: None, at: waggle_core::Timestamp::from_unix_ms(3), })) .unwrap(); diff --git a/crates/waggle-store-sqlite/tests/g_suite.rs b/crates/waggle-store-sqlite/tests/g_suite.rs index 8811c6a..85e9417 100644 --- a/crates/waggle-store-sqlite/tests/g_suite.rs +++ b/crates/waggle-store-sqlite/tests/g_suite.rs @@ -49,6 +49,7 @@ fn event(token: waggle_core::Token) -> AppendIntent { ), variant: None, regions: None, + entry: None, at: waggle_core::Timestamp::from_unix_ms(2), } } diff --git a/crates/waggle-store/src/conformance.rs b/crates/waggle-store/src/conformance.rs index c767876..2094a94 100644 --- a/crates/waggle-store/src/conformance.rs +++ b/crates/waggle-store/src/conformance.rs @@ -59,6 +59,7 @@ fn event_intent(token: waggle_core::Token, stage: Stage) -> AppendIntent { actor: ActorClass::from_context(&ResolverContext::anonymous_agent()), variant: None, regions: None, + entry: None, at: Timestamp::from_unix_ms(7), } } diff --git a/crates/waggle-store/src/memory.rs b/crates/waggle-store/src/memory.rs index d54e2d0..263b8b1 100644 --- a/crates/waggle-store/src/memory.rs +++ b/crates/waggle-store/src/memory.rs @@ -160,6 +160,7 @@ impl Inner { actor, variant, regions, + entry, at, } = intent else { @@ -186,6 +187,7 @@ impl Inner { seq, variant: *variant, regions: *regions, + entry: *entry, })); Ok(Appended::Event { seq }) } diff --git a/crates/waggle-store/src/types.rs b/crates/waggle-store/src/types.rs index 5bd7d16..b50dca9 100644 --- a/crates/waggle-store/src/types.rs +++ b/crates/waggle-store/src/types.rs @@ -56,6 +56,11 @@ pub enum AppendIntent { /// frames keep parsing. #[serde(default)] regions: Option, + /// For a one-file read inside an indexed tree node: the file's ordinal + /// in the node's signed directory index (a position, not a payload — + /// I-1-safe like `regions`). Defaulted so pre-tree frames keep parsing. + #[serde(default)] + entry: Option, /// When it happened. at: Timestamp, }, diff --git a/crates/waggle-tree/src/search.rs b/crates/waggle-tree/src/search.rs index 3bd3713..3efaa76 100644 --- a/crates/waggle-tree/src/search.rs +++ b/crates/waggle-tree/src/search.rs @@ -50,6 +50,10 @@ pub struct Hit { /// The token of the subtree node that directly contains the file — the /// handle a consumer resolves to read more. pub token: String, + /// The file's ordinal in its node's `DirIndex.files()` order — a stable + /// position into the signed directory index. Carried so the caller can + /// stamp a per-file read (I-1-safe like a region touch) on the owning node. + pub entry: u32, /// 1-based line number of the first match in the file. pub line: u32, /// The matching line's text (already budget-trimmed by the caller). @@ -91,6 +95,7 @@ mod tests { Hit { path: path.into(), token: "t".into(), + entry: 0, line: 1, text: "x".into(), matches, diff --git a/edge-worker/tests/miniflare.rs b/edge-worker/tests/miniflare.rs index f7b2282..3eb165f 100644 --- a/edge-worker/tests/miniflare.rs +++ b/edge-worker/tests/miniflare.rs @@ -621,6 +621,7 @@ fn seed_local_store() -> (Vec, String) { ), variant: None, regions: None, + entry: None, at: waggle_core::Timestamp::from_unix_ms(2), }) .await From 153aff838d7fac4a06402e3510ccbce66b7f70eb Mon Sep 17 00:00:00 2001 From: Chetan Conikee Date: Mon, 13 Jul 2026 15:16:30 -0700 Subject: [PATCH 06/12] bench+cli: conformance suite for the indexed tree; wire read --file into the CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Tier-3 conformance suite (zero-LLM scripted-consumer detector) encoded the OLD flat-tree contract — per-file child tokens, --section fan-out with a --from cursor, a files:all gate with a met verdict. On the indexed tree it crashed (read's `files` is now a count, not a list). Rewritten to the indexed contract it actually ships: projection read is a table of contents — total_files (true tree size), local files, and subdirs each with a token to descend. Asserted on reasoning (11), folder (12), bigtree (180, nested). per-file read --file serves one file and stamps exactly it: opening 1 of 11 reports files "1/11", names the other 10 unread, and clears its own name. This is the headline the indexing work stands on — a receipt of WHICH file, not "the folder was touched". search one call spans the lineage, every match names its file (path + token); an absent literal visits ZERO nodes (Bloom prune bites); a zero-match search moves coverage not at all. the close a perfect consumer closes per-file coverage cheaply: one search serves the bulk, coverage names the stragglers, read --file opens each — reasoning closes in 2 calls. budget the bounded responses (TOC, search) fit max-bytes. 53 checks, all green on the real corpora against the indexed binary. Wiring gap this surfaced: read --file existed in the MCP handler but was never exposed on the CLI — there was no command-line way to read one file of a tree, so the per-file close path was unreachable. Added `read --file ` to the CLI and the matching ops-catalog ArgSpec (parity holds). Known follow-up (flagged, not silently changed): the READ/coverage catalog DESCRIPTIONS still narrate the old fan-out / files:all-gate model, which the indexed tree no longer implements — that is model-facing prose and a wording call. Per-file coverage remains the shipped receipt; the paper's flat-tree gate/fan-out numbers stay pinned to the pre-index backend (decision 2). All workspace tests pass; clippy clean; every file <= 750 lines. --- bench/tier3/conformance.py | 355 +++++++++++++++++++--------------- crates/waggle-cli/src/main.rs | 5 + crates/waggle-ops/src/lib.rs | 1 + 3 files changed, 205 insertions(+), 156 deletions(-) diff --git a/bench/tier3/conformance.py b/bench/tier3/conformance.py index f097d75..1418872 100644 --- a/bench/tier3/conformance.py +++ b/bench/tier3/conformance.py @@ -2,26 +2,43 @@ We spent a day discovering bugs with a language model as the detector. Every one of them was DETERMINISTIC — a zero-match search forging a receipt, a projection -truncating in silence, a fan-out dropping two files of eleven, a response -overrunning the caller's byte budget, the harness showing the model half of what -waggle served while the receipt certified all of it. Not one of those needed a -model to find. Each cost twenty minutes and a sweep to notice, and each was -noticed alone, so the next sweep found the next one. +lying about its size, a search that could not say WHICH file matched, a response +overrunning the caller's byte budget. Not one of those needed a model to find. +Each cost twenty minutes and a sweep to notice, and each was noticed alone, so +the next sweep found the next one. This is the detector that should have existed first. It drives the substrate with a SCRIPTED consumer — no model, no tokens, no nondeterminism — and asserts the properties the benchmark depends on. It runs in seconds and reports EVERY failure at once, so the fixes land in one batch instead of one per expensive run. +The substrate under test serves a big corpus as an INDEXED TREE (design doc: +tree-scale): `mint --tree` builds one directory node per folder — Merkle-addressed +content, a trigram index, and a Bloom summary per node — instead of one token per +file. That lifts the old per-file cap (a thousand files mint in one call) and +changes the interaction contract, which is what this suite now asserts: + + a folder READ (no address) is a table of contents — local files (name/size/type) + and subdirectories (name/token/totals), bounded, no truncation; + + a folder READ --file serves ONE file from its content-addressed blob and + stamps it as a per-file read; + + a SEARCH spans the whole lineage in ONE call — Bloom-pruned, trigram-narrowed, + regex-confirmed, ranked — and every match names its file (path + owning token); + + COVERAGE is per-file: `files: "read/total"`, `complete`, and the unread files + NAMED. Reading part of a tree reports part; a search that matches nothing moves + it not at all. + Two consumers are simulated: the ORACLE — plays perfectly: uses the affordances waggle advertises, in the - order waggle advertises them. If the oracle cannot satisfy a - contract in a few calls, no model will, and the contract is a - trap rather than a check. + order it advertises them. If the oracle cannot close per-file + coverage in a few calls, no model will. - the LAZY one — answers having read nothing. The gate MUST refuse it. A gate that - passes this is not a gate. + the LAZY one — certifies nothing it has not read. Coverage on an untouched tree + must report incomplete; a consumer that read nothing is refused. Green here is the precondition for spending money on a sweep. """ @@ -59,192 +76,218 @@ def result(args: list[str]) -> dict: return (wag(args).get("result") or {}) -def mint_tree(path: str, require_all: bool) -> str: - a = ["mint", "--target", path, "--tree"] - if require_all: - a += ["--require", "files:all"] - return result(a).get("token", "") +def mint_tree(path: str) -> str: + return result(["mint", "--target", path, "--tree"]).get("token", "") + + +def cov_ratio(tok: str) -> tuple[int, int]: + """Coverage's `files: "read/total"` parsed to (read, total).""" + s = result(["coverage", "--token", tok]).get("files", "0/0") + r, _, n = str(s).partition("/") + return int(r or 0), int(n or 0) # ---------------------------------------------------------------- invariants -def inv_budget_respected(tok: str, kind: str, val: str) -> None: - """A byte budget is a CONTRACT WITH THE CALLER. +def inv_projection_knows_its_denominator(path: str, n_files: int) -> None: + """A folder's table of contents must state the true size of the whole tree. + + The denominator is what tells a consumer whether it has seen everything. A + projection that under- or over-counts the tree is a projection that cannot be + trusted to say `complete`. + """ + tok = mint_tree(path) + d = wag(["read", "--token", tok]).get("result") or {} + check(d.get("kind") == "tree", "projection-is-a-tree", + f"read of a --tree token returned kind={d.get('kind')!r}") + check(d.get("total_files") == n_files, "projection-knows-its-denominator", + f"total_files={d.get('total_files')} but the tree has {n_files}") + + +def inv_projection_hands_back_the_way_down(path: str) -> None: + """Every subdirectory in a listing must carry the token to descend into it, + and every local file a name to open it — or the tree is a dead end.""" + tok = mint_tree(path) + d = wag(["read", "--token", tok]).get("result") or {} + for sub in d.get("dirs") or []: + check(bool(sub.get("token")), "subdir-carries-a-token", + f"subdir {sub.get('name')!r} has no token — unreachable") + for f in d.get("children") or []: + check(bool(f.get("name")), "file-carries-a-name", + "a listed file has no name — cannot be opened with --file") + + +def inv_search_identifies_its_files(path: str, pattern: str) -> None: + """A tree search must say WHICH file matched, or the matches are unusable.""" + d = result(["search", "--token", mint_tree(path), "--pattern", pattern]) + matches = d.get("matches") or [] + check(bool(matches), "search-finds-matches", f"no matches for {pattern!r}") + for m in matches[:3]: + check(bool(m.get("path")), "search-identifies-its-files", + "a match carries no path — the consumer cannot tell which file") + check(bool(m.get("token")), "search-hands-back-a-token", + "a match carries no token — the consumer cannot open it") + + +def inv_absent_pattern_prunes_the_tree(path: str) -> None: + """A literal pattern the Bloom excludes everywhere must visit ZERO nodes. - We returned 9,255 bytes against a budget of 8,000. The client showed its model - the first 4,500 and dropped the rest — while our receipt certified all ten - files as read. A response that overruns the budget is a receipt that lies, - because we cannot know what the client truncates. + The prune gate is what makes one-call search over thousands of files viable. + If an absent needle still walks the whole tree, the index is decoration. """ - env = wag(["read", "--token", tok, f"--{kind}", val, "--max-bytes", str(VIEW_BYTES)]) - n = len(json.dumps(env.get("result") or {})) - check(n <= VIEW_BYTES, "budget-respected", - f"asked {VIEW_BYTES}B, response {n}B — the consumer will not see {n - VIEW_BYTES}B " - f"of what our receipt claims it read") + d = result(["search", "--token", mint_tree(path), + "--pattern", "ZZQQ_NO_SUCH_STRING_ZZQQ"]) + check(d.get("total_matches") == 0, "absent-pattern-finds-nothing", + f"a nonsense pattern matched {d.get('total_matches')} files") + check(d.get("nodes_visited") == 0, "absent-pattern-prunes-the-tree", + f"an absent literal still walked {d.get('nodes_visited')} nodes — " + f"the Bloom prune gate did not bite") def inv_zero_match_forges_nothing(path: str) -> None: """A grep that matches nothing must not stamp `read` on the files it swept. - One zero-match search once took a folder from 0/11 met=false to 11/11 - met=true. Bytes touched by the matcher and content served to the consumer are - different ledgers; only the second is a receipt. + One zero-match search once took a folder from 0/11 to 11/11. Bytes touched by + the matcher and content served to the consumer are different ledgers; only the + second is a receipt. """ - tok = mint_tree(path, True) - before = result(["coverage", "--token", tok]).get("read") + tok = mint_tree(path) + before = cov_ratio(tok) wag(["search", "--token", tok, "--pattern", "ZZQQ_NO_SUCH_STRING_ZZQQ"]) - after = result(["coverage", "--token", tok]).get("read") + after = cov_ratio(tok) check(before == after, "zero-match-forges-nothing", - f"coverage moved {before} -> {after} on a search that matched NOTHING") + f"coverage moved {before[0]}/{before[1]} -> {after[0]}/{after[1]} " + f"on a search that matched NOTHING") -def inv_projection_is_honest(path: str, n_files: int) -> None: - """A truncated listing must say it is truncated, and hand back the way on.""" - tok = mint_tree(path, False) - env = wag(["read", "--token", tok]) - d = env.get("result") or {} - check(d.get("total_files") == n_files, "projection-knows-its-denominator", - f"total_files={d.get('total_files')} but the tree has {n_files}") - if not d.get("complete"): - check(bool(d.get("hint")), "projection-truncation-is-loud", - "listing is incomplete and says nothing about it") - cursors = [x for x in (env.get("next") or []) if x["args"].get("from") is not None] - check(bool(cursors), "projection-truncation-is-resumable", - "listing is incomplete and hands back no cursor") - - -def inv_fanout_covers_or_pages(path: str, n_files: int, section: str, fact: str) -> None: - """The fan-out must serve WHOLE files, never drop them silently, and page. - - `max_bytes / 6` — a hardcoded six-file assumption — served nine of eleven and - dropped two. That made `--require files:all` unsatisfiable in one call BY - CONSTRUCTION. And the naive repair (thin every file to fit) is worse: the - load-bearing sentence is at the END of a section, so a thin prefix of every - file serves everything and cuts the fact out of each. Full coverage, zero - information. +def inv_coverage_refuses_the_unread(path: str) -> None: + """A tree nobody has read must report incomplete. Else there is no receipt.""" + tok = mint_tree(path) + r, n = cov_ratio(tok) + complete = result(["coverage", "--token", tok]).get("complete") + check(r == 0 and n > 0 and complete is False, "coverage-refuses-the-unread", + f"an untouched tree shows files={r}/{n} complete={complete}") + + +def inv_per_file_receipt_is_precise(path: str, n_files: int) -> None: + """Reading ONE file of many must report exactly one read — and NAME the rest. + + This is the headline the indexing work stands on: a receipt that says the + agent opened this file and not that one, not merely "the folder was touched". """ - tok = mint_tree(path, True) - seen, page, guard = 0, 0, 0 - frm = None - while guard < 12: - guard += 1 - a = ["read", "--token", tok, "--section", section, "--max-bytes", str(VIEW_BYTES)] - if frm is not None: - a += ["--from", str(frm)] - env = wag(a) - d = env.get("result") or {} - files = d.get("files") or [] - seen += len(files) - page += 1 - # every served file keeps the fact — depth was not silently thinned away - intact = sum(1 for f in files if fact in (f.get("text") or "")) - check(intact == len(files), "fanout-keeps-the-fact", - f"{len(files) - intact} of {len(files)} served files had the load-bearing " - f"line truncated away — coverage without information") - if d.get("complete"): - break - nxt = [x for x in (env.get("next") or []) if x["args"].get("from") is not None] - if not check(bool(nxt), "fanout-truncation-is-resumable", - "fan-out incomplete and no `from` cursor offered"): - return - frm = nxt[0]["args"]["from"] - check(page <= 3, "fanout-completes-cheaply", - f"took {page} pages to fan out over {n_files} files — a contract this " - f"expensive is a trap, not a check") - - -def inv_contract_satisfiable_by_oracle(path: str, section: str) -> None: - """A perfect consumer must be able to CLOSE a files:all contract in few calls. - - If the oracle cannot, no model will: it will burn its turns and answer nothing, - and the gate will (correctly, uselessly) refuse it. That is what happened — - gpt-4o-mini HELD the right answer, was refused four times, and died at the cap - while the ungated arm simply gave the same answer and was believed. + tok = mint_tree(path) + d = wag(["read", "--token", tok]).get("result") or {} + names = [f["name"] for f in (d.get("children") or [])] + if not check(bool(names), "per-file-needs-a-flat-node", + f"{path} has no files directly on its root node to probe"): + return + opened = names[0] + served = wag(["read", "--token", tok, "--file", opened]).get("result") or {} + check(served.get("name") == opened, "read-file-serves-the-named-file", + f"read --file {opened!r} returned name={served.get('name')!r}") + r, n = cov_ratio(tok) + check((r, n) == (1, n_files), "per-file-receipt-is-precise", + f"opened 1 file, coverage says {r}/{n} (want 1/{n_files})") + cov = result(["coverage", "--token", tok]) + missing = [f for u in (cov.get("unread") or []) for f in (u.get("first_missing") or [])] + check(opened not in missing, "read-file-clears-its-own-name", + f"{opened!r} was read yet still listed unread") + check(any(nm in missing for nm in names[1:]), "unread-files-are-named", + "coverage named none of the files left unread") + + +def inv_search_stamps_every_match(path: str, pattern: str, expect: int) -> None: + """A search that serves K files must stamp exactly K as read — the one-call + equivalent of the old fan-out, and cheaper.""" + tok = mint_tree(path) + d = result(["search", "--token", tok, "--pattern", pattern]) + served = d.get("total_matches") + r, _ = cov_ratio(tok) + check(served == expect, "search-serves-the-expected-set", + f"search {pattern!r} matched {served} files, expected {expect}") + check(r == served, "search-stamps-every-match", + f"search served {served} files but coverage moved to {r} read") + + +def inv_oracle_closes_per_file(path: str, pattern: str, n_files: int) -> None: + """A perfect consumer must CLOSE per-file coverage in a few calls. + + The move the indexed tree advertises: one search serves the bulk; coverage + then NAMES the stragglers; read --file opens each by name. If the oracle + cannot finish cheaply, no model with ten turns and a question to answer will. """ - tok = mint_tree(path, True) + tok = mint_tree(path) calls = 0 - # play exactly what waggle advertises, in the order it advertises it - frm = None - for _ in range(6): - a = ["read", "--token", tok, "--section", section, "--max-bytes", str(VIEW_BYTES)] - if frm is not None: - a += ["--from", str(frm)] - env = wag(a) - calls += 1 - d = env.get("result") or {} - if d.get("complete"): - break - nxt = [x for x in (env.get("next") or []) if x["args"].get("from") is not None] - if not nxt: - break - frm = nxt[0]["args"]["from"] + # one search serves every file that carries the pattern + wag(["search", "--token", tok, "--pattern", pattern]) + calls += 1 - # files with no such section are never served, so read them directly - for _ in range(6): + # coverage names the files with no such match; open each by name on its node + for _ in range(n_files): cov = result(["coverage", "--token", tok]) - unread = cov.get("unread") or [] - if not unread: + if cov.get("complete"): + break + gap = next((u for u in (cov.get("unread") or []) if u.get("first_missing")), None) + if not gap: break - wag(["read", "--token", unread[0]["token"]]) + wag(["read", "--token", gap["token"], "--file", gap["first_missing"][0]]) calls += 1 cov = result(["coverage", "--token", tok]) - check(cov.get("met") is True, "oracle-can-close-the-contract", - f"a PERFECT consumer ended at read={cov.get('read')} met={cov.get('met')} — " - f"the contract is unsatisfiable, so the gate is a trap") - check(calls <= 5, "contract-closes-cheaply", - f"oracle needed {calls} calls to satisfy files:all — models have ~10 turns " - f"and must also reason; this will burn them out") - - -def inv_gate_refuses_the_lazy(path: str) -> None: - """A consumer that read nothing must not be believed. Else there is no gate.""" - tok = mint_tree(path, True) - cov = result(["coverage", "--token", tok]) - check(cov.get("met") is False, "gate-refuses-the-lazy", - f"a consumer that has read NOTHING shows met={cov.get('met')}") - - -def inv_search_identifies_its_files(path: str, pattern: str) -> None: - """A tree search must say WHICH file matched, or the matches are unusable.""" - tok = mint_tree(path, False) - d = result(["search", "--token", tok, "--pattern", pattern, "--context", "0"]) - files = d.get("files") or [] - check(bool(files), "search-finds-matches", f"no matches for {pattern!r}") - for f in files[:3]: - check(bool(f.get("target") or f.get("name")), "search-identifies-its-files", - "a matched file carries no name/target — the consumer cannot tell which") - check(bool(f.get("token")), "search-hands-back-a-token", - "a matched file carries no token — the consumer cannot open it") + check(cov.get("complete") is True, "oracle-can-close-per-file-coverage", + f"a PERFECT consumer ended at files={cov.get('files')} " + f"complete={cov.get('complete')} — the tree is not closable") + check(calls <= 4, "per-file-coverage-closes-cheaply", + f"oracle needed {calls} calls to read all {n_files} files — a corpus " + f"where one search + a couple of reads will not close is a trap") + + +def inv_budget_respected(path: str, pattern: str) -> None: + """The bounded responses — a table of contents, a search result — are a + CONTRACT WITH THE CALLER: what we return must fit the budget it asked for, + because we cannot know what the client truncates past it.""" + tok = mint_tree(path) + toc = wag(["read", "--token", tok, "--max-bytes", str(VIEW_BYTES)]) + n = len(json.dumps(toc.get("result") or {})) + check(n <= VIEW_BYTES, "toc-fits-the-budget", + f"table of contents was {n}B against a {VIEW_BYTES}B budget") + srch = wag(["search", "--token", tok, "--pattern", pattern, + "--max-matches", "10"]) + n = len(json.dumps(srch.get("result") or {})) + check(n <= VIEW_BYTES, "search-fits-the-budget", + f"search result was {n}B against a {VIEW_BYTES}B budget") def main() -> int: corpus2 = os.environ.get("TIER3_CORPUS2", "/tmp/tier3/corpus2") corpus3 = os.environ.get("TIER3_CORPUS3", "/tmp/tier3/corpus3") - reasoning = f"{corpus2}/reasoning_0" - folder = f"{corpus2}/folder_0" - bigtree = f"{corpus3}/bigtree_0" + reasoning = f"{corpus2}/reasoning_0" # flat: 11 files, "retry budget of" in 10 + folder = f"{corpus2}/folder_0" # flat: 12 files + bigtree = f"{corpus3}/bigtree_0" # nested: 180 files across subdirectories print(f"substrate under test: {WAGGLE}") h = subprocess.run(["shasum", "-a", "256", WAGGLE], capture_output=True, text=True) print(f" sha256 {h.stdout.split()[0][:16] if h.stdout else '?'}\n") - # --- the reasoning tree: 11 files, a files:all contract, the gate's home turf - inv_gate_refuses_the_lazy(reasoning) + # --- the reasoning tree: a flat folder, the per-file receipt's home turf + inv_coverage_refuses_the_unread(reasoning) inv_zero_match_forges_nothing(reasoning) - inv_budget_respected(mint_tree(reasoning, True), "section", "Retry Policy") - inv_fanout_covers_or_pages(reasoning, 11, "Retry Policy", "retry budget of") - inv_contract_satisfiable_by_oracle(reasoning, "Retry Policy") + inv_per_file_receipt_is_precise(reasoning, 11) + inv_search_stamps_every_match(reasoning, "retry budget of", 10) + inv_oracle_closes_per_file(reasoning, "retry budget of", 11) inv_search_identifies_its_files(reasoning, "retry budget of") + inv_budget_respected(reasoning, "retry budget of") - # --- a small folder - inv_projection_is_honest(folder, 12) + # --- a small folder: the projection must know its own size and hand the way on + inv_projection_knows_its_denominator(folder, 12) + inv_projection_hands_back_the_way_down(folder) - # --- the big tree: where the projection MUST truncate - inv_projection_is_honest(bigtree, 180) - inv_zero_match_forges_nothing(bigtree) + # --- the big nested tree: denominator, pruning, and named matches at scale + inv_projection_knows_its_denominator(bigtree, 180) + inv_projection_hands_back_the_way_down(bigtree) + inv_absent_pattern_prunes_the_tree(bigtree) inv_search_identifies_its_files(bigtree, "legacy_reconcile") - inv_budget_respected(mint_tree(bigtree, False), "section", "Overview") + inv_budget_respected(bigtree, "legacy_reconcile") print(f"{CHECKS[0]} checks, {len(FAILS)} FAILED\n") for f in FAILS: diff --git a/crates/waggle-cli/src/main.rs b/crates/waggle-cli/src/main.rs index 1147a83..5dd9f33 100644 --- a/crates/waggle-cli/src/main.rs +++ b/crates/waggle-cli/src/main.rs @@ -130,6 +130,9 @@ enum Cmd { /// Markdown heading whose section to read (text/markdown lens). #[arg(long)] section: Option, + /// For a FOLDER token (minted --tree): read ONE file by name, fetched from the content-addressed blob and stamped as a per-file read. The folder's `read` (no address) lists the file names. + #[arg(long)] + file: Option, /// Code symbol whose definition to read (symbol lens — tokens minted with a snapshot of source code); the overview's `symbols` lists what exists. #[arg(long)] symbol: Option, @@ -301,6 +304,7 @@ fn main() { token, lines, section, + file, symbol, path, from, @@ -311,6 +315,7 @@ fn main() { "token": token, "lines": lines, "section": section, + "file": file, "from": from, "symbol": symbol, "path": path, diff --git a/crates/waggle-ops/src/lib.rs b/crates/waggle-ops/src/lib.rs index 1517c0a..96b061b 100644 --- a/crates/waggle-ops/src/lib.rs +++ b/crates/waggle-ops/src/lib.rs @@ -206,6 +206,7 @@ pub const READ: OperationSpec = OperationSpec { ArgSpec { name: "token", required: true, doc: "The waggle token whose content to read." }, ArgSpec { name: "lines", required: false, doc: "Line window, 1-based inclusive (e.g. 120-180)." }, ArgSpec { name: "section", required: false, doc: "Markdown heading whose section to read (text/markdown lens)." }, + ArgSpec { name: "file", required: false, doc: "For a FOLDER token (minted --tree): read ONE file by name, fetched from the content-addressed blob and stamped as a per-file read. The folder's `read` (no address) lists the file names." }, ArgSpec { name: "symbol", required: false, doc: "Code symbol whose definition to read (symbol lens — tokens minted with a snapshot of source code); the overview's `symbols` lists what exists." }, ArgSpec { name: "path", required: false, doc: "JSON pointer into parsed content (application/json lens), e.g. /dependencies/react." }, ArgSpec { name: "from", required: false, doc: "For a FOLDER token with a lens: continue the fan-out from this file index. A truncated tree-lens names the cursor to resume from — a partial folder read that looks like a whole one is how a confident wrong answer gets made." }, From e135dc93362eb3dd2898f3586699a448afdf6c9e Mon Sep 17 00:00:00 2001 From: Chetan Conikee Date: Mon, 13 Jul 2026 15:20:08 -0700 Subject: [PATCH 07/12] paper: frame indexing as scale + per-file receipts, pinned to the measured backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Limitations gains a scope note on scale (decision: option 2). The evaluated system mints one token per file — per-file coverage for free, but bounded past a few hundred files. Subsequent work replaces the per-file forest with an indexed directory tree (content-addressed nodes, per-node trigram + Bloom): thousands of files in one mint, one search pruned and ranked across the lineage, coverage still per file but now WITHOUT a token per file — each read records the file's position in its node's signed index, a position not a payload, admissible under the same payload-free-log invariant (sec:receipts) as the region-touch field. Crucially honest: this is validated by the conformance suite (deterministic), NOT by re-running the answer-quality sweep, which is reported on the per-file system it was collected on. Whether indexing moves those numbers is future measurement, not a claim drawn from these. No results numbers change. Adds the Bloom (1970) citation. Compiles clean (tectonic) — no undefined refs or citations. --- paper/references.bib | 10 ++++++++++ paper/waggle.tex | 17 +++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/paper/references.bib b/paper/references.bib index ae2205d..b0c555e 100644 --- a/paper/references.bib +++ b/paper/references.bib @@ -69,6 +69,16 @@ @inproceedings{merkle1987 year = {1987} } +@article{bloom1970, + author = {Burton H. Bloom}, + title = {Space/Time Trade-offs in Hash Coding with Allowable Errors}, + journal = {Communications of the ACM}, + volume = {13}, + number = {7}, + pages = {422--426}, + year = {1970} +} + @article{grasse1959, author = {Pierre-Paul Grass{\'e}}, title = {La reconstruction du nid et les coordinations interindividuelles chez \emph{Bellicositermes natalensis} et \emph{Cubitermes} sp.\ La th{\'e}orie de la stigmergie}, diff --git a/paper/waggle.tex b/paper/waggle.tex index e8331c4..1767f9b 100644 --- a/paper/waggle.tex +++ b/paper/waggle.tex @@ -1105,6 +1105,23 @@ \section{Limitations and Outlook} precisely where the arms diverge and where our own mechanism misfires. A harder corpus is the obvious next measurement, and we expect it to be less flattering. +A word on scale, and on what measures it. The evaluated system mints one token +per file. That serves a folder faithfully and makes coverage per file for free--- +each file is its own token---but it bounds the folder: past a few hundred files a +single mint is impractical. Subsequent work replaces the per-file forest with an +indexed directory tree, each folder one content-addressed node carrying a trigram +index and a Bloom summary~\citep{bloom1970}, so a corpus of thousands of files +mints in one call and a single \tok{search} prunes and ranks across the whole +lineage. Coverage \emph{stays} per file, now without a token per file: each read +records the file's position in its node's signed index---a position, not a +payload, admissible under the same invariant (\S\ref{sec:receipts}) that keeps +the log free of recipient bytes, exactly as the region-touch field is. We hold +this to the standard the rest of the substrate is held to before a sweep---the +conformance suite, extended to the indexed tree's contract, deterministically--- +rather than re-running the answer-quality measurement, which is reported here on +the per-file system it was collected on. Whether indexing moves those numbers is +future measurement, not a claim we draw from these. + The outlook follows from the three-radii result. As harnesses routinely fan work across machines and across vendors, the copy discipline's costs compound superlinearly---more copies, more billing boundaries, more stale From 2000a29596e43eea2078dcb0a2f6eabccf1dbf27 Mon Sep 17 00:00:00 2001 From: Chetan Conikee Date: Mon, 13 Jul 2026 15:44:12 -0700 Subject: [PATCH 08/12] =?UTF-8?q?mcp:=20wire=20files:all=20onto=20the=20in?= =?UTF-8?q?dexed=20tree=20=E2=80=94=20the=20completeness=20gate,=20proven?= =?UTF-8?q?=20per=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mint --tree --require files:all silently ignored the contract: you asked for a gate and got none. Now the root node carries the files:all contract (tree_mint parses it via the shared contract_args path; a folder has no single text, so a section: requirement is rejected — only files:all is meaningful over a tree), and tree coverage emits `met`/`requires` alongside the per-file `files: read/total`. `met` flips false -> true as the last file is read. A plain --tree carries no contract and reports `complete` as a fact, never a verdict. This restores the gate the flat tree had — and per-file coverage makes it STRONGER: met is proven file by file, not node by node. The paper's gate results are reproducible on the shipping backend again. Verified live on corpus2/reasoning_0 (11 files, files:all): met=false at 0/11, met=true at 11/11 after one search + one read --file. Also fixes a pre-existing test flake: two extraction tests shared the temp dir name `waggle-extract-{pid}`, so in parallel one's cleanup wiped the other's blobs (binary_target_...pdf_story failed intermittently). Given distinct html/pdf names, the suite is green multi-threaded. All workspace tests pass; clippy clean. --- crates/waggle-mcp/src/lineage.rs | 29 ++++-- crates/waggle-mcp/src/tree_mint.rs | 15 +++ crates/waggle-mcp/tests/content_access.rs | 112 +++++++++++++++++++++- 3 files changed, 144 insertions(+), 12 deletions(-) diff --git a/crates/waggle-mcp/src/lineage.rs b/crates/waggle-mcp/src/lineage.rs index 70bd83b..79fa051 100644 --- a/crates/waggle-mcp/src/lineage.rs +++ b/crates/waggle-mcp/src/lineage.rs @@ -262,17 +262,26 @@ impl Handler { } } let complete = files > 0 && read == files; + let mut result = json!({ + "token": root.as_str(), + "kind": "tree", + "files": format!("{read}/{files}"), + "nodes": format!("{read_nodes}/{nodes}"), + "total_files": total_files, + "total_bytes": total_bytes, + "complete": complete, + "unread": unread, + }); + // A tree minted `--require files:all` carries a COMPLETENESS contract on its + // root. Without it, `complete` is a fact an orchestrator may consult; with + // it, `met` is a verdict it can refuse an answer on — the same gate the flat + // path applies, now proven per file rather than per node. + if requires_all_files(Some(view)) { + result["met"] = json!(complete); + result["requires"] = json!(crate::contract_args::TREE_ALL); + } Envelope::ok( - json!({ - "token": root.as_str(), - "kind": "tree", - "files": format!("{read}/{files}"), - "nodes": format!("{read_nodes}/{nodes}"), - "total_files": total_files, - "total_bytes": total_bytes, - "complete": complete, - "unread": unread, - }), + result, vec![NextCall { tool: "search".into(), args: json!({ "token": root.as_str(), "pattern": "" }), diff --git a/crates/waggle-mcp/src/tree_mint.rs b/crates/waggle-mcp/src/tree_mint.rs index 50ed280..e053569 100644 --- a/crates/waggle-mcp/src/tree_mint.rs +++ b/crates/waggle-mcp/src/tree_mint.rs @@ -338,6 +338,21 @@ impl Handler { } if let Some(args) = root_args { spec = crate::discovery::apply_tags(spec, args); + // A completeness contract rides on the ROOT node: `--require files:all` + // declares that this delegation needs the WHOLE tree, and `coverage` + // then refuses to call it met while any file is unread. A folder has no + // single text, so a `section:` requirement (which would need one) is + // rejected here — only `files:all` is meaningful over a tree. + match crate::contract_args::parse_contract(args, || { + Err(Envelope::err( + "require: a folder has no single text — only `files:all` applies to a --tree mint", + vec![], + )) + }) { + Ok(Some(contract)) => spec = spec.contract(contract), + Ok(None) => {} + Err(e) => return Err(e), + } } let mut manifest = waggle_core::mint(spec, &MintOptions::default(), &mut *entropy, now) .map_err(|e| Envelope::err(e.to_string(), vec![]))?; diff --git a/crates/waggle-mcp/tests/content_access.rs b/crates/waggle-mcp/tests/content_access.rs index 30fed93..de33a9e 100644 --- a/crates/waggle-mcp/tests/content_access.rs +++ b/crates/waggle-mcp/tests/content_access.rs @@ -61,7 +61,7 @@ fn check_next(env: &waggle_mcp::Envelope) { /// carries the searchable text, not the harness. #[test] fn opaque_html_is_extracted_and_searchable_with_provenance() { - let dir = std::env::temp_dir().join(format!("waggle-extract-{}", std::process::id())); + let dir = std::env::temp_dir().join(format!("waggle-html-extract-{}", std::process::id())); std::fs::remove_dir_all(&dir).ok(); std::fs::create_dir_all(&dir).unwrap(); let file = dir.join("briefing.html"); @@ -378,7 +378,7 @@ fn binary_target_with_extracted_content_the_pdf_story() { // Doc 18 §7: the harness extracted the PDF once at mint; the token // serves surgical access to the extraction forever, while the target // stays the original binary. - let dir = std::env::temp_dir().join(format!("waggle-extract-{}", std::process::id())); + let dir = std::env::temp_dir().join(format!("waggle-pdf-extract-{}", std::process::id())); std::fs::remove_dir_all(&dir).ok(); std::fs::create_dir_all(&dir).unwrap(); let pdf = dir.join("q3-report.pdf"); @@ -854,3 +854,111 @@ fn coverage_is_per_file_over_a_tree() { assert_eq!(after.result["complete"], true); }); } + +/// `mint --tree --require files:all` puts a completeness GATE on the tree: coverage +/// reports `met` (a verdict an orchestrator can refuse on), flipping false→true as +/// the last file is read. A plain `--tree` carries no such verdict. +#[test] +fn files_all_gate_on_a_tree_flips_met_when_the_last_file_is_read() { + pollster::block_on(async { + let dir = tempfile::tempdir().unwrap(); + let docs = dir.path().join("review_me"); + std::fs::create_dir_all(&docs).unwrap(); + std::fs::write(docs.join("a.md"), "content alpha\n").unwrap(); + std::fs::write(docs.join("b.md"), "content beta\n").unwrap(); + let handler = handler_with_blobs(dir.path()); + let mut e = entropy(); + let root = handler + .dispatch( + "mint", + &json!({ "target": format!("file://{}", docs.display()), + "tree": true, "require": "files:all" }), + Timestamp::from_unix_ms(1), + &mut e, + ) + .await + .result["token"] + .as_str() + .unwrap() + .to_owned(); + + // Nothing read: the contract is declared and unmet. + let before = handler + .dispatch( + "coverage", + &json!({ "token": root }), + Timestamp::from_unix_ms(2), + &mut e, + ) + .await; + assert_eq!(before.result["requires"], "files:all", "{before:?}"); + assert_eq!( + before.result["met"], false, + "unread tree cannot be met: {before:?}" + ); + + // Read one file — still short, still unmet. + handler + .dispatch( + "read", + &json!({ "token": root, "file": "a.md" }), + Timestamp::from_unix_ms(3), + &mut e, + ) + .await; + let mid = handler + .dispatch( + "coverage", + &json!({ "token": root }), + Timestamp::from_unix_ms(4), + &mut e, + ) + .await; + assert_eq!(mid.result["met"], false, "one of two read: {mid:?}"); + + // Read the second — the gate flips to met. + handler + .dispatch( + "read", + &json!({ "token": root, "file": "b.md" }), + Timestamp::from_unix_ms(5), + &mut e, + ) + .await; + let done = handler + .dispatch( + "coverage", + &json!({ "token": root }), + Timestamp::from_unix_ms(6), + &mut e, + ) + .await; + assert_eq!(done.result["met"], true, "whole tree read: {done:?}"); + + // A plain --tree (no contract) reports completeness as a fact, never a verdict. + let plain = handler + .dispatch( + "mint", + &json!({ "target": format!("file://{}", docs.display()), "tree": true }), + Timestamp::from_unix_ms(7), + &mut e, + ) + .await + .result["token"] + .as_str() + .unwrap() + .to_owned(); + let cov = handler + .dispatch( + "coverage", + &json!({ "token": plain }), + Timestamp::from_unix_ms(8), + &mut e, + ) + .await; + assert!( + cov.result.get("met").is_none(), + "no contract, no verdict: {cov:?}" + ); + }); +} From 1a3c94cac88ef9e2163e64271b4ae1f186666f3f Mon Sep 17 00:00:00 2001 From: Chetan Conikee Date: Mon, 13 Jul 2026 15:47:42 -0700 Subject: [PATCH 09/12] =?UTF-8?q?ops+cli:=20correct=20the=20tree=20model?= =?UTF-8?q?=20in=20the=20catalog=20=E2=80=94=20read/search/coverage/tree,?= =?UTF-8?q?=20drop=20--from?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The operations catalog (the source of truth for CLI help, MCP tool descriptions, and COMMANDS.md) described the OLD flat model: mint --tree "mints every file as a child", read "FANS OUT a lens across every file" with an examined/total_files truncation and a --from cursor, search groups matches "each with that file's own token". None of that is how the indexed tree works. Corrected to the shipping model: --tree builds an indexed directory tree — one content-addressed node per folder (trigram + Bloom), thousands of files in one mint. read a folder token returns its table of contents (files by name, subdirs by token); read --file serves one file. search spans the whole tree in ONE call — pruned, narrowed, ranked; each match names its file path + owning node token. coverage a per-file receipt (files: read/total, complete, unread NAMED); --require files:all adds the met verdict. Removed the --from arg (CLI + catalog): the fan-out it paged no longer exists, so it was a dead, misleading flag. --require's files:all clause is unchanged — it is correct again now that the gate is wired. COMMANDS.md regenerated; parity green. --- COMMANDS.md | 14 +++++++------- crates/waggle-cli/src/main.rs | 5 ----- crates/waggle-ops/src/lib.rs | 13 ++++++------- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/COMMANDS.md b/COMMANDS.md index 3646ebc..161f4fa 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -14,7 +14,7 @@ Create an attributed reference (a waggle token) for an artifact instead of pasti | `--parent` | false | Parent token: forms the delegation tree at mint; revoking the parent tombstones this child. | | `--snapshot` | false | Pin the target's bytes content-addressed at mint: read/search then work anywhere the blobs replicate, immutable by hash. If the target is a PDF or HTML document, its text layer is extracted here too (deterministically, provenance recorded) so read/search work over the artifact itself. Audio/video carry no text layer: their bytes are pinned and read tells the consumer to perceive them with its own model. | | `--private` | false | Mint a capability URL: a 16-char unguessable token (possession IS the credential); public unfurls and social renders refuse it. | -| `--tree` | false | For a DIRECTORY target: also mint every file inside (recursive, snapshot-pinned) as children of this token — one revocation covers the whole tree, and the folder's funnel rolls its children up. | +| `--tree` | false | For a DIRECTORY target: build an INDEXED directory tree — one content-addressed node per folder, each carrying a trigram index and a Bloom summary, snapshot-pinned. Thousands of files mint in one call. `read` a folder token for its table of contents, `read --file ` for one file, `search` to span the whole tree in one call, `coverage` for a per-file receipt. One revocation covers the whole tree. | | `--tag` | false | Name the token for humans (repeatable, k=v or a bare name): cosmetic labels that `find` matches on. A tag is a convenience, never identity — resolution stays token-only. | | `--content` | false | Path to text you extracted yourself for a binary target — becomes the searchable content while the target stays the original. Rarely needed: `snapshot` now extracts PDF and HTML text layers automatically. Use this only for a format the substrate does not read, or to override its extraction. Mutually exclusive with snapshot. | | `--attach` | false | Path to media (image/audio) stored content-addressed; vision/audio consumers receive it, others get the catch-all. | @@ -80,26 +80,26 @@ A token's funnel: stage counts (impression → resolve → run → repeat) plus ## `read` — CLI + MCP tool -Read the token's CONTENT surgically: a line window, a markdown section, a code symbol, or a JSON pointer path — never the whole artifact. With no address: the overview (size, content type, available lenses, outline; source code carries its symbol table of contents). If the token names a FOLDER (minted --tree), read DESCRIBES it — every file, its own token, size, type and outline: the folder's table of contents — and a lens applied to the folder token FANS OUT across every file at once, so you ask once, not once per file. A fan-out that exhausts its budget reports complete=false with examined/total_files and a `from` cursor to resume: never conclude anything about a tree you have not finished reading. Every response fits max-bytes and names the bytes you avoided. +Read the token's CONTENT surgically: a line window, a markdown section, a code symbol, or a JSON pointer path — never the whole artifact. With no address: the overview (size, content type, available lenses, outline; source code carries its symbol table of contents). If the token names a FOLDER (minted --tree), read with no address returns its table of contents — the folder's own files by name (size, type) and its subdirectories, each with a token to descend — and `read --file ` serves one of those files. To grep a folder, use `search`: it spans the whole tree in one call. Coverage stays per-file. Every response fits max-bytes and names the bytes you avoided. | arg | required | doc | |---|---|---| | `--token` | true | The waggle token whose content to read. | | `--lines` | false | Line window, 1-based inclusive (e.g. 120-180). | | `--section` | false | Markdown heading whose section to read (text/markdown lens). | +| `--file` | false | For a FOLDER token (minted --tree): read ONE file by name, fetched from the content-addressed blob and stamped as a per-file read. The folder's `read` (no address) lists the file names. | | `--symbol` | false | Code symbol whose definition to read (symbol lens — tokens minted with a snapshot of source code); the overview's `symbols` lists what exists. | | `--path` | false | JSON pointer into parsed content (application/json lens), e.g. /dependencies/react. | -| `--from` | false | For a FOLDER token with a lens: continue the fan-out from this file index. A truncated tree-lens names the cursor to resume from — a partial folder read that looks like a whole one is how a confident wrong answer gets made. | | `--max-bytes` | false | Response budget in bytes (default 4096, floor 64). | -- forward → `read`: continue the window, follow the outline deeper, or resume a truncated folder fan-out from its `from` cursor -- forward → `search`: grep the artifact — or, on a folder token, every file in the tree at once +- forward → `read`: continue the window, follow the outline deeper, or open a folder's file by name with --file +- forward → `search`: grep the artifact — or, on a folder token, the whole tree in one call - forward → `coverage`: on a folder: which files you have actually been served, and which you have not - forward → `record`: report run when the content did its job ## `search` — CLI + MCP tool -Grep the token's CONTENT: regex matches with line numbers and context, capped and budgeted — the matches travel, the artifact stays put. total_matches is counted in full even when the list is truncated. A FOLDER token (minted --tree) greps as a TREE: every file is searched and matches come back grouped per file, each with that file's own token so you can open it surgically. Works wherever the content's blobs replicate. +Grep the token's CONTENT: regex matches with line numbers and context, capped and budgeted — the matches travel, the artifact stays put. total_matches is counted in full even when the list is truncated. A FOLDER token (minted --tree) greps as a TREE in ONE call: Bloom-pruned and trigram-narrowed across the whole lineage, then ranked, each match naming its file path and the node token that owns it, so you can open it with read --file. Works wherever the content's blobs replicate. | arg | required | doc | |---|---|---| @@ -135,7 +135,7 @@ Find tokens by what humans remember: matches the query against target basenames, ## `coverage` — CLI + MCP tool -For a lineage root (a folder or bundle): which descendants were actually consumed? Three honest levels per file — unread (never touched), read (bytes served: a resolve, read, or search reached it), run (the consumer recorded using it). For a single token minted with a contract (mint --require): which required regions did the served bytes reach — met/unmet against the declared threshold. Either way, misses are NAMED: the unread list is the proof of what a review skipped. +For a FOLDER (minted --tree): a PER-FILE receipt — how many of its files were actually served (`files: read/total`, a read or a search hit counts), whether that is `complete`, and the unread files NAMED. A tree minted `--require files:all` also carries a verdict: `met` stays false while any file is unread. For a single token minted with a contract (mint --require lines/section/symbol): which required regions the served bytes reached — met/unmet against the declared threshold. Either way, misses are NAMED: the unread list is the proof of what a review skipped. | arg | required | doc | |---|---|---| diff --git a/crates/waggle-cli/src/main.rs b/crates/waggle-cli/src/main.rs index 5dd9f33..06dd21c 100644 --- a/crates/waggle-cli/src/main.rs +++ b/crates/waggle-cli/src/main.rs @@ -139,9 +139,6 @@ enum Cmd { /// JSON pointer into parsed content (application/json lens), e.g. /dependencies/react. #[arg(long)] path: Option, - /// For a FOLDER token with a lens: continue the fan-out from this file index. A truncated tree-lens names the cursor to resume from — a partial folder read that looks whole is how a confident wrong answer gets made. - #[arg(long)] - from: Option, /// Response budget in bytes (default 4096, floor 64). #[arg(long)] max_bytes: Option, @@ -307,7 +304,6 @@ fn main() { file, symbol, path, - from, max_bytes, } => run::tool_call( "read", @@ -316,7 +312,6 @@ fn main() { "lines": lines, "section": section, "file": file, - "from": from, "symbol": symbol, "path": path, "max-bytes": max_bytes, diff --git a/crates/waggle-ops/src/lib.rs b/crates/waggle-ops/src/lib.rs index 96b061b..7df58ca 100644 --- a/crates/waggle-ops/src/lib.rs +++ b/crates/waggle-ops/src/lib.rs @@ -107,7 +107,7 @@ pub const MINT: OperationSpec = OperationSpec { ArgSpec { name: "parent", required: false, doc: "Parent token: forms the delegation tree at mint; revoking the parent tombstones this child." }, ArgSpec { name: "snapshot", required: false, doc: "Pin the target's bytes content-addressed at mint: read/search then work anywhere the blobs replicate, immutable by hash. If the target is a PDF or HTML document, its text layer is extracted here too (deterministically, provenance recorded) so read/search work over the artifact itself. Audio/video carry no text layer: their bytes are pinned and read tells the consumer to perceive them with its own model." }, ArgSpec { name: "private", required: false, doc: "Mint a capability URL: a 16-char unguessable token (possession IS the credential); public unfurls and social renders refuse it." }, - ArgSpec { name: "tree", required: false, doc: "For a DIRECTORY target: also mint every file inside (recursive, snapshot-pinned) as children of this token — one revocation covers the whole tree, and the folder's funnel rolls its children up." }, + ArgSpec { name: "tree", required: false, doc: "For a DIRECTORY target: build an INDEXED directory tree — one content-addressed node per folder, each carrying a trigram index and a Bloom summary, snapshot-pinned. Thousands of files mint in one call. `read` a folder token for its table of contents, `read --file ` for one file, `search` to span the whole tree in one call, `coverage` for a per-file receipt. One revocation covers the whole tree." }, ArgSpec { name: "tag", required: false, doc: "Name the token for humans (repeatable, k=v or a bare name): cosmetic labels that `find` matches on. A tag is a convenience, never identity — resolution stays token-only." }, ArgSpec { name: "content", required: false, doc: "Path to text you extracted yourself for a binary target — becomes the searchable content while the target stays the original. Rarely needed: `snapshot` now extracts PDF and HTML text layers automatically. Use this only for a format the substrate does not read, or to override its extraction. Mutually exclusive with snapshot." }, ArgSpec { name: "attach", required: false, doc: "Path to media (image/audio) stored content-addressed; vision/audio consumers receive it, others get the catch-all." }, @@ -201,7 +201,7 @@ pub const READ: OperationSpec = OperationSpec { name: "read", surface: Surface::Both, kind: OpKind::RelaxedWrite, - description: "Read the token's CONTENT surgically: a line window, a markdown section, a code symbol, or a JSON pointer path — never the whole artifact. With no address: the overview (size, content type, available lenses, outline; source code carries its symbol table of contents). If the token names a FOLDER (minted --tree), read DESCRIBES it — every file, its own token, size, type and outline: the folder's table of contents — and a lens applied to the folder token FANS OUT across every file at once, so you ask once, not once per file. A fan-out that exhausts its budget reports complete=false with examined/total_files and a `from` cursor to resume: never conclude anything about a tree you have not finished reading. Every response fits max-bytes and names the bytes you avoided.", + description: "Read the token's CONTENT surgically: a line window, a markdown section, a code symbol, or a JSON pointer path — never the whole artifact. With no address: the overview (size, content type, available lenses, outline; source code carries its symbol table of contents). If the token names a FOLDER (minted --tree), read with no address returns its table of contents — the folder's own files by name (size, type) and its subdirectories, each with a token to descend — and `read --file ` serves one of those files. To grep a folder, use `search`: it spans the whole tree in one call. Coverage stays per-file. Every response fits max-bytes and names the bytes you avoided.", args: &[ ArgSpec { name: "token", required: true, doc: "The waggle token whose content to read." }, ArgSpec { name: "lines", required: false, doc: "Line window, 1-based inclusive (e.g. 120-180)." }, @@ -209,12 +209,11 @@ pub const READ: OperationSpec = OperationSpec { ArgSpec { name: "file", required: false, doc: "For a FOLDER token (minted --tree): read ONE file by name, fetched from the content-addressed blob and stamped as a per-file read. The folder's `read` (no address) lists the file names." }, ArgSpec { name: "symbol", required: false, doc: "Code symbol whose definition to read (symbol lens — tokens minted with a snapshot of source code); the overview's `symbols` lists what exists." }, ArgSpec { name: "path", required: false, doc: "JSON pointer into parsed content (application/json lens), e.g. /dependencies/react." }, - ArgSpec { name: "from", required: false, doc: "For a FOLDER token with a lens: continue the fan-out from this file index. A truncated tree-lens names the cursor to resume from — a partial folder read that looks like a whole one is how a confident wrong answer gets made." }, ArgSpec { name: "max-bytes", required: false, doc: "Response budget in bytes (default 4096, floor 64)." }, ], forward: &[ - EdgeSpec { to: "read", why: "continue the window, follow the outline deeper, or resume a truncated folder fan-out from its `from` cursor" }, - EdgeSpec { to: "search", why: "grep the artifact — or, on a folder token, every file in the tree at once" }, + EdgeSpec { to: "read", why: "continue the window, follow the outline deeper, or open a folder's file by name with --file" }, + EdgeSpec { to: "search", why: "grep the artifact — or, on a folder token, the whole tree in one call" }, EdgeSpec { to: "coverage", why: "on a folder: which files you have actually been served, and which you have not" }, EdgeSpec { to: "record", why: "report run when the content did its job" }, ], @@ -227,7 +226,7 @@ pub const SEARCH: OperationSpec = OperationSpec { name: "search", surface: Surface::Both, kind: OpKind::RelaxedWrite, - description: "Grep the token's CONTENT: regex matches with line numbers and context, capped and budgeted — the matches travel, the artifact stays put. total_matches is counted in full even when the list is truncated. A FOLDER token (minted --tree) greps as a TREE: every file is searched and matches come back grouped per file, each with that file's own token so you can open it surgically. Works wherever the content's blobs replicate.", + description: "Grep the token's CONTENT: regex matches with line numbers and context, capped and budgeted — the matches travel, the artifact stays put. total_matches is counted in full even when the list is truncated. A FOLDER token (minted --tree) greps as a TREE in ONE call: Bloom-pruned and trigram-narrowed across the whole lineage, then ranked, each match naming its file path and the node token that owns it, so you can open it with read --file. Works wherever the content's blobs replicate.", args: &[ ArgSpec { name: "token", required: true, doc: "The waggle token whose content to search." }, ArgSpec { name: "pattern", required: true, doc: "Regex (Rust syntax; (?i) prefix for case-insensitive)." }, @@ -263,7 +262,7 @@ pub const COVERAGE: OperationSpec = OperationSpec { name: "coverage", surface: Surface::Both, kind: OpKind::Read, - description: "For a lineage root (a folder or bundle): which descendants were actually consumed? Three honest levels per file — unread (never touched), read (bytes served: a resolve, read, or search reached it), run (the consumer recorded using it). For a single token minted with a contract (mint --require): which required regions did the served bytes reach — met/unmet against the declared threshold. Either way, misses are NAMED: the unread list is the proof of what a review skipped.", + description: "For a FOLDER (minted --tree): a PER-FILE receipt — how many of its files were actually served (`files: read/total`, a read or a search hit counts), whether that is `complete`, and the unread files NAMED. A tree minted `--require files:all` also carries a verdict: `met` stays false while any file is unread. For a single token minted with a contract (mint --require lines/section/symbol): which required regions the served bytes reached — met/unmet against the declared threshold. Either way, misses are NAMED: the unread list is the proof of what a review skipped.", args: &[ ArgSpec { name: "token", required: true, doc: "The lineage root (or contract-bearing token) to audit." }, ], From a1fa0d26a6e4cc769b3ded322763f244f202ff26 Mon Sep 17 00:00:00 2001 From: Chetan Conikee Date: Mon, 13 Jul 2026 15:52:39 -0700 Subject: [PATCH 10/12] docs: sweep the stale flat-tree model out of the site, guides, and design logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catalog fix (prior commit) corrected the generated surfaces; this corrects the hand-written ones to the indexed tree + per-file model. Site (docs/pages/index.html): the tutorial's step 3 listed a token per file and step 4 fanned a --section lens "across every file at once". Now step 3 shows the table of contents (files by name) and points at `read --file`; step 4 is the one-call tree `search`, pruned and ranked. Steps 2 and 5 (--require files:all, coverage with met) are unchanged — correct now that the gate is wired. Guides: 04-lifecycle-and-query read --section fan-out -> read --file + one-call search; coverage key read -> files. 07-surgical-content per-child funnels -> per-file receipts; run level -> per-file files: read/total + complete + met. 11-tmux-switchboard per-file child tokens -> indexed tree; resolve INDEX of file-tokens -> read table of contents + read --file. Design logs (historical decision records): added "superseded by waggle-tree-scale" banners to 18 §9 and 22 §4 rather than erasing the fan-out history — the failures they name were real; only the mechanics changed. Fixed 18 §9.1 (per-file token -> file by name) and 20-symbol-lens's "≤ 200 files" -> indexed, thousands. Core: TreeNode doc-comment claimed a per-file token is minted lazily — no such minting exists; files are addressed by name. Corrected. The paper stays historical (measured pre-index) and needs no change. No code behavior changed here; workspace builds, fmt clean. --- crates/waggle-core/src/manifest.rs | 5 +++-- docs/design/18-content-access.md | 29 +++++++++++++++++++-------- docs/design/20-symbol-lens.md | 2 +- docs/design/22-benchmark.md | 8 ++++++++ docs/guide/04-lifecycle-and-query.md | 8 ++++---- docs/guide/07-surgical-content.md | 16 +++++++-------- docs/guide/11-tmux-switchboard.md | 21 ++++++++++--------- docs/pages/index.html | 30 ++++++++++++++-------------- 8 files changed, 70 insertions(+), 49 deletions(-) diff --git a/crates/waggle-core/src/manifest.rs b/crates/waggle-core/src/manifest.rs index b9d3b41..9e7ffc2 100644 --- a/crates/waggle-core/src/manifest.rs +++ b/crates/waggle-core/src/manifest.rs @@ -238,8 +238,9 @@ pub struct Extraction { /// addressed by their own subtree token), the **trigram index** over its files, /// and a **Bloom summary** of every trigram beneath it, inlined so a search can /// prune the subtree from the manifest alone. Files pin their bytes eagerly (so a -/// deleted file still reads — C-1); a per-file token is minted only lazily, when a -/// consumer needs a standalone reference to one file. +/// deleted file still reads — C-1) and are addressed by name within their node +/// (`read --file`); a file is not itself a token, so a folder of thousands mints +/// as a handful of nodes, not thousands of tokens. /// /// The `bloom` is the lowercase-hex wire form of a fixed-size filter (parsed into /// the typed structure by the consumer, `waggle-tree::Bloom`); keeping it a string diff --git a/docs/design/18-content-access.md b/docs/design/18-content-access.md index 5ba00b1..18b71c7 100644 --- a/docs/design/18-content-access.md +++ b/docs/design/18-content-access.md @@ -171,10 +171,21 @@ blobs (audio segments, resumable pulls — ranges, never decoding). ## 9 · The directory affordances (added; see doc 22 §4) -A token may name a **folder** (`mint --tree`: one parent, every file a child). -These three affordances were not designed at a whiteboard — each exists -because the cross-modality benchmark caught an agent needing it and not -having it, and the traces name the failure exactly. +> **Superseded in part (design doc: `waggle-tree-scale`).** This section records +> the *original* flat tree — one token per file, and a lens that *fanned out* +> across a folder with a `--from` cursor (§9.2–9.3). That model capped a folder at +> a few hundred files. It was later replaced by an **indexed directory tree** — one +> content-addressed node per folder (trigram + Bloom), thousands of files in one +> mint. In the current model: `read` a folder for its table of contents, +> `read --file ` for one file, and `search` to span the whole tree in one +> call (there is no lens fan-out and no `--from`). Coverage is per-file. The +> *reasons* below still hold — each affordance answered a real failure — but the +> mechanics are as described in the tree-scale design doc. + +A token may name a **folder** (`mint --tree`). These three affordances were not +designed at a whiteboard — each exists because the cross-modality benchmark +caught an agent needing it and not having it, and the traces name the failure +exactly. ### 9.1 Describe (`read` on a tree) @@ -184,10 +195,12 @@ makes with a shared directory is to ask what is in it. Both models we traced opened with an overview call and received 83 bytes of nulls, then had to guess a regex blind; one guessed wrong and got zero matches. -`read` on a tree now returns **the tree**: each file, its own token, size, -content type, and outline. The folder's table of contents. Listing is *not* -consumption — no `read` stage is stamped for the children, because a table of -contents tells you what exists and does not serve you the bytes. +`read` on a tree returns **the tree's table of contents**: its files by name +(size, content type, outline) and its subdirectories, each with a token to +descend. (Original design: each file its own token; the indexed tree addresses +a file by name via `read --file` instead.) Listing is *not* consumption — no +`read` stage is stamped, because a table of contents tells you what exists and +does not serve you the bytes. ### 9.2 Lens (`read --section/--symbol/--lines` on a tree) diff --git a/docs/design/20-symbol-lens.md b/docs/design/20-symbol-lens.md index a63a211..7e75926 100644 --- a/docs/design/20-symbol-lens.md +++ b/docs/design/20-symbol-lens.md @@ -123,7 +123,7 @@ Performance is a design input here, not an afterthought. The budget: |---|---|---| | serve (`read` overview, edge or local) | **zero parsing; one CAS get** — within noise of today | outline precomputed at mint; render is a budget-fit over an already-flat structure | | mint, single file | extraction ≤ **5 ms / kLOC** on one core; total mint overhead < 15% over snapshot-only | one parse, one query pass, arena output; tree dropped before return | -| mint `--tree` (≤ 200 files) | wall-clock ≈ snapshot cost, not snapshot + N·parse | extraction parallel across a bounded blocking pool; appends stay sequential (the committer owns order) | +| mint `--tree` (indexed; thousands of files) | wall-clock ≈ snapshot cost, not snapshot + N·parse | extraction parallel across a bounded blocking pool; appends stay sequential (the committer owns order) | | memory | peak = one tree + one arena per in-flight file; **no retained trees, no global index** | parse → extract → drop; outline ≈ 40 B/symbol + one shared name buffer | | startup | zero until first code mint | grammars linked, queries compiled lazily, once per process | diff --git a/docs/design/22-benchmark.md b/docs/design/22-benchmark.md index 29e11c6..eff238a 100644 --- a/docs/design/22-benchmark.md +++ b/docs/design/22-benchmark.md @@ -364,6 +364,14 @@ recorded consumers guessing, and that is what sent us to the traces. An evaluation that looked only at answers would have scored the model down and shipped the defect. +> **Note (design doc: `waggle-tree-scale`).** The last two rows describe the +> original *lens fan-out* over a folder and its `complete`/`examined`/`from` +> truncation. The indexed directory tree later replaced the fan-out: to read +> across a tree you now use `search` (one call, Bloom-pruned and ranked) or +> `read --file `, and coverage is per-file — there is no fan-out cursor. +> The failures the rows name were real; the mechanics are now as in the +> tree-scale design doc. + ### 10.1 A harness failure worth naming The harness was **dropping waggle's `next` hints** — the substrate's diff --git a/docs/guide/04-lifecycle-and-query.md b/docs/guide/04-lifecycle-and-query.md index a03f6ff..fd8ddd4 100644 --- a/docs/guide/04-lifecycle-and-query.md +++ b/docs/guide/04-lifecycle-and-query.md @@ -77,11 +77,11 @@ you can't make of a filesystem path: ```bash waggle mint --target file://$PWD/src --tree --require files:all # → PxL8mQ2v -waggle read --token PxL8mQ2v # the projection: one child token per file -waggle read --token PxL8mQ2v --section Retry # fan the lens out over every child -waggle search --token PxL8mQ2v --pattern "def retry" # or grep the tree; hits stamp their file +waggle read --token PxL8mQ2v # the table of contents: files by name, subdirs by token +waggle read --token PxL8mQ2v --file retry.py # open one file by name +waggle search --token PxL8mQ2v --pattern "def retry" # or grep the whole tree in one call; hits stamp their file waggle coverage --token PxL8mQ2v -# → { "read": "3/4", "met": false, "requires": "files:all" } +# → { "files": "3/4", "met": false, "requires": "files:all" } ``` `read` is the **consumer's** ledger, not the reader's: a search that diff --git a/docs/guide/07-surgical-content.md b/docs/guide/07-surgical-content.md index 7d032d5..49150ad 100644 --- a/docs/guide/07-surgical-content.md +++ b/docs/guide/07-surgical-content.md @@ -179,19 +179,19 @@ pointer returns the valid roots; a token with no readable content says ## Coverage: proof the tree was read -For any lineage root (a `--tree` mint or a bundle), the per-child -funnels already know which files were consumed — `coverage` turns that -into proof, with misses NAMED: +For a `--tree` mint, each read records which file it served, so +`coverage` turns that into a **per-file** proof, with misses NAMED: ```sh waggle coverage --token -# read 2/3 · run 0/3 · complete: false +# files 2/3 · complete: false # unread: [.../notes.md] <- what the review skipped ``` -Three honest levels: `unread` / `read` (bytes served — a deep search -over the root counts, because it really reads every file) / `run` (the -consumer recorded use — the strong, intentional bar). It's receipts, -not surveillance (payload-free, I-1), and receipts turn +A file counts as read when its bytes were served — a `read --file`, or a +`search` hit that returned from it (a broad grep that sweeps a file but +returns nothing does not stamp it). A tree minted `--require files:all` +also carries a `met` verdict: false while any file is unread. It's +receipts, not surveillance (payload-free, I-1), and receipts turn enforcement-grade when the handoff is SEALED (`waggle-tmux mint --seal`) or remote — where the token is the only door. diff --git a/docs/guide/11-tmux-switchboard.md b/docs/guide/11-tmux-switchboard.md index aa82076..92c2128 100644 --- a/docs/guide/11-tmux-switchboard.md +++ b/docs/guide/11-tmux-switchboard.md @@ -174,21 +174,20 @@ standing): > Put your outputs in `handoff/` — the diff, the test log, notes. When > done: `waggle mint --target handoff --snapshot --channel tmux/claude-code` -A directory target mints as a **tree**: the folder token is the root, -and every file inside becomes a snapshot-pinned CHILD. One token -travels; behind it: +A directory target mints as an **indexed tree**: the folder token is the +root of a content-addressed tree — one node per folder, snapshot-pinned, +thousands of files in one mint. One token travels; behind it: ```sh -waggle resolve --token b2uQyZUC # the root serves its INDEX: -# children: [ {qR8s → handoff/diff.patch}, -# {wN2k → handoff/test.log}, -# {pV5m → handoff/notes.md} ] +waggle read --token b2uQyZUC # the root's table of contents: +# files: [ handoff/diff.patch, handoff/test.log, handoff/notes.md ] +# (subdirectories, if any, each carry a token to descend) -waggle search --token b2uQyZUC --pattern "FAILED" # DEEP search: greps -# every file in the tree, matches grouped per file, with a -# read-this-next chain into the first hit +waggle search --token b2uQyZUC --pattern "FAILED" # ONE call greps the +# whole tree — Bloom-pruned, ranked — each match naming its file path +# and owning node token, with a read-this-next chain into the first hit -waggle read --token wN2k --lines 40-80 # one file, one window +waggle read --token b2uQyZUC --file test.log # open one file by name ``` **Step 3 — lineage.** `waggle-tmux mint` chains each outcome to the diff --git a/docs/pages/index.html b/docs/pages/index.html index 4b74567..e616e4b 100644 --- a/docs/pages/index.html +++ b/docs/pages/index.html @@ -95,14 +95,14 @@

The subagent resolves it — and sees a map, not 20 KB.

$ waggle read --token 4x82KC93
- - - - + + + +
00_escalation_policy.mdWgcYKCbaEscalation Policy · Ceiling
runbook_01.md2UbiByHsRetry Policy · Stage 2
runbook_02.mdDvnUKymYRetry Policy · Stage 2
… 11 files
00_escalation_policy.mdEscalation Policy · Ceiling
runbook_01.mdRetry Policy · Stage 2
runbook_02.mdRetry Policy · Stage 2
… 11 files
-

A table of contents, with headings. Every file gets its own - token too — so the agent can address one file directly, without ever +

A table of contents, with headings. Open any file directly by + name — read --file runbook_01.md — without ever pulling the other ten.

@@ -111,19 +111,19 @@

The subagent resolves it — and sees a map, not 20 KB.

4

Now query through the token.

- 
$ waggle read --token 4x82KC93 --section "Retry Policy"
+ 
$ waggle search --token 4x82KC93 --pattern "retry budget"
- - + +
runbook_01.mdL3-8“…a retry budget of 2 attempts…”
runbook_02.mdL3-8“…a retry budget of 4 attempts…”
runbook_01.mdL5“…a retry budget of 2 attempts…”
runbook_02.mdL5“…a retry budget of 4 attempts…”
-
8 files served · 6,616 bytes - of 20,825 — only what it asked for
+
10 matches across the tree + — one call, pruned and ranked
-

One question, asked of every file at once. Not eleven - calls — one. The same works with --symbol for code, - --lines for a window, or search for a pattern across the - whole tree.

+

One search spans the whole tree at once — not + eleven greps, one — and every match names its file, so you open it with + read --file. The same token takes --symbol for code or + --lines for a window.

From 94cc76872b1519e8e1303dfac5cf83378fc72e48 Mon Sep 17 00:00:00 2001 From: Chetan Conikee Date: Mon, 13 Jul 2026 16:07:28 -0700 Subject: [PATCH 11/12] release: bump workspace to 0.5.0 Indexed directory tree (cap lifted to thousands of files), per-file coverage receipts, files:all gate rewired onto trees, and the CLI read --file affordance. Breaking CLI change: the dead --from fan-out cursor is removed. --- Cargo.lock | 34 +++++++++++------------ Cargo.toml | 16 +++++------ crates/waggle-cli/Cargo.toml | 6 ++-- crates/waggle-mcp/Cargo.toml | 8 +++--- crates/waggle-store-cloudflare/Cargo.toml | 4 +-- crates/waggle-store-fs-jsonl/Cargo.toml | 2 +- crates/waggle-store-sqlite/Cargo.toml | 2 +- crates/waggle-tmux/Cargo.toml | 2 +- edge-worker/Cargo.toml | 10 +++---- 9 files changed, 42 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f6e5e5b..3555a22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2142,7 +2142,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "waggle" -version = "0.4.0" +version = "0.5.0" dependencies = [ "waggle-core", "waggle-ops", @@ -2150,7 +2150,7 @@ dependencies = [ [[package]] name = "waggle-agent" -version = "0.4.0" +version = "0.5.0" dependencies = [ "serde", "serde_json", @@ -2159,7 +2159,7 @@ dependencies = [ [[package]] name = "waggle-bench" -version = "0.4.0" +version = "0.5.0" dependencies = [ "serde_json", "waggle-core", @@ -2167,7 +2167,7 @@ dependencies = [ [[package]] name = "waggle-cli" -version = "0.4.0" +version = "0.5.0" dependencies = [ "base64", "clap", @@ -2186,7 +2186,7 @@ dependencies = [ [[package]] name = "waggle-core" -version = "0.4.0" +version = "0.5.0" dependencies = [ "criterion", "ed25519-dalek", @@ -2198,7 +2198,7 @@ dependencies = [ [[package]] name = "waggle-edge-worker" -version = "0.4.0" +version = "0.5.0" dependencies = [ "base64", "console_error_panic_hook", @@ -2220,7 +2220,7 @@ dependencies = [ [[package]] name = "waggle-lens-code" -version = "0.4.0" +version = "0.5.0" dependencies = [ "criterion", "serde", @@ -2237,7 +2237,7 @@ dependencies = [ [[package]] name = "waggle-mcp" -version = "0.4.0" +version = "0.5.0" dependencies = [ "criterion", "ed25519-dalek", @@ -2259,14 +2259,14 @@ dependencies = [ [[package]] name = "waggle-ops" -version = "0.4.0" +version = "0.5.0" dependencies = [ "serde", ] [[package]] name = "waggle-social" -version = "0.4.0" +version = "0.5.0" dependencies = [ "qrcodegen", "serde", @@ -2275,7 +2275,7 @@ dependencies = [ [[package]] name = "waggle-store" -version = "0.4.0" +version = "0.5.0" dependencies = [ "pollster", "serde", @@ -2286,7 +2286,7 @@ dependencies = [ [[package]] name = "waggle-store-cloudflare" -version = "0.4.0" +version = "0.5.0" dependencies = [ "pollster", "serde", @@ -2298,7 +2298,7 @@ dependencies = [ [[package]] name = "waggle-store-fs-jsonl" -version = "0.4.0" +version = "0.5.0" dependencies = [ "pollster", "serde_json", @@ -2309,7 +2309,7 @@ dependencies = [ [[package]] name = "waggle-store-sqlite" -version = "0.4.0" +version = "0.5.0" dependencies = [ "criterion", "loom", @@ -2324,7 +2324,7 @@ dependencies = [ [[package]] name = "waggle-tmux" -version = "0.4.0" +version = "0.5.0" dependencies = [ "clap", "pollster", @@ -2340,7 +2340,7 @@ dependencies = [ [[package]] name = "waggle-tree" -version = "0.4.0" +version = "0.5.0" dependencies = [ "serde", "serde_json", @@ -2671,7 +2671,7 @@ checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "xtask" -version = "0.4.0" +version = "0.5.0" dependencies = [ "ed25519-dalek", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 4af89e4..0972bca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["crates/*", "edge-worker", "xtask", "bench"] [workspace.package] -version = "0.4.0" +version = "0.5.0" edition = "2021" rust-version = "1.85" license = "MIT OR Apache-2.0" @@ -33,13 +33,13 @@ loom = "0.7" ed25519-dalek = { version = "2", default-features = false, features = ["std"] } tokio = { version = "1", default-features = false, features = ["rt-multi-thread", "net", "io-util", "macros", "time", "sync"] } -waggle-core = { path = "crates/waggle-core", version = "0.4.0" } -waggle-tree = { path = "crates/waggle-tree", version = "0.4.0" } -waggle-ops = { path = "crates/waggle-ops", version = "0.4.0" } -waggle-agent = { path = "crates/waggle-agent", version = "0.4.0" } -waggle-social = { path = "crates/waggle-social", version = "0.4.0" } -waggle-store = { path = "crates/waggle-store", version = "0.4.0" } -waggle-store-sqlite = { path = "crates/waggle-store-sqlite", version = "0.4.0" } +waggle-core = { path = "crates/waggle-core", version = "0.5.0" } +waggle-tree = { path = "crates/waggle-tree", version = "0.5.0" } +waggle-ops = { path = "crates/waggle-ops", version = "0.5.0" } +waggle-agent = { path = "crates/waggle-agent", version = "0.5.0" } +waggle-social = { path = "crates/waggle-social", version = "0.5.0" } +waggle-store = { path = "crates/waggle-store", version = "0.5.0" } +waggle-store-sqlite = { path = "crates/waggle-store-sqlite", version = "0.5.0" } [workspace.lints.rust] missing_docs = "deny" diff --git a/crates/waggle-cli/Cargo.toml b/crates/waggle-cli/Cargo.toml index 7272dfb..e4d63e9 100644 --- a/crates/waggle-cli/Cargo.toml +++ b/crates/waggle-cli/Cargo.toml @@ -19,9 +19,9 @@ path = "src/main.rs" [dependencies] waggle-core = { workspace = true } waggle-ops = { workspace = true } -waggle-mcp = { path = "../waggle-mcp", version = "0.4.0", features = ["code-lens", "doc-extract"] } -waggle-store = { path = "../waggle-store", version = "0.4.0" } -waggle-store-sqlite = { path = "../waggle-store-sqlite", version = "0.4.0" } +waggle-mcp = { path = "../waggle-mcp", version = "0.5.0", features = ["code-lens", "doc-extract"] } +waggle-store = { path = "../waggle-store", version = "0.5.0" } +waggle-store-sqlite = { path = "../waggle-store-sqlite", version = "0.5.0" } clap = { workspace = true } serde_json = { workspace = true } pollster = { workspace = true } diff --git a/crates/waggle-mcp/Cargo.toml b/crates/waggle-mcp/Cargo.toml index 18c4695..9c51c47 100644 --- a/crates/waggle-mcp/Cargo.toml +++ b/crates/waggle-mcp/Cargo.toml @@ -16,7 +16,7 @@ homepage.workspace = true waggle-core = { workspace = true } waggle-tree = { workspace = true } waggle-ops = { workspace = true } -waggle-store = { path = "../waggle-store", version = "0.4.0" } +waggle-store = { path = "../waggle-store", version = "0.5.0" } serde = { workspace = true } serde_json = { workspace = true } getrandom = { workspace = true } @@ -26,7 +26,7 @@ ed25519-dalek = { workspace = true } # feature-gated off by default — the edge worker compiles this crate to # wasm and serves precomputed outlines as data (outline_wire.rs, always # compiled). The CLI/daemon turn the feature on. -waggle-lens-code = { path = "../waggle-lens-code", version = "0.4.0", optional = true } +waggle-lens-code = { path = "../waggle-lens-code", version = "0.5.0", optional = true } # Deterministic PDF text-layer extraction (doc 18 §7). Pure Rust, no C — but a # heavy parser, so feature-gated off by default and off for the wasm/edge build. # The CLI/daemon turn it on; HTML extraction needs no crate and is always present. @@ -39,9 +39,9 @@ doc-extract = ["dep:pdf-extract"] [dev-dependencies] pollster = { workspace = true } -waggle-store-sqlite = { path = "../waggle-store-sqlite", version = "0.4.0" } +waggle-store-sqlite = { path = "../waggle-store-sqlite", version = "0.5.0" } waggle-social = { workspace = true } -waggle-lens-code = { path = "../waggle-lens-code", version = "0.4.0" } +waggle-lens-code = { path = "../waggle-lens-code", version = "0.5.0" } criterion = { workspace = true } tempfile = "3" diff --git a/crates/waggle-store-cloudflare/Cargo.toml b/crates/waggle-store-cloudflare/Cargo.toml index f5366e6..dd8bf16 100644 --- a/crates/waggle-store-cloudflare/Cargo.toml +++ b/crates/waggle-store-cloudflare/Cargo.toml @@ -14,13 +14,13 @@ homepage.workspace = true [dependencies] waggle-core = { workspace = true } -waggle-store = { path = "../waggle-store", version = "0.4.0" } +waggle-store = { path = "../waggle-store", version = "0.5.0" } serde = { workspace = true } serde_json = { workspace = true } [dev-dependencies] pollster = { workspace = true } -waggle-store-sqlite = { path = "../waggle-store-sqlite", version = "0.4.0" } +waggle-store-sqlite = { path = "../waggle-store-sqlite", version = "0.5.0" } [lints] workspace = true diff --git a/crates/waggle-store-fs-jsonl/Cargo.toml b/crates/waggle-store-fs-jsonl/Cargo.toml index 88173e9..fdb2086 100644 --- a/crates/waggle-store-fs-jsonl/Cargo.toml +++ b/crates/waggle-store-fs-jsonl/Cargo.toml @@ -14,7 +14,7 @@ homepage.workspace = true [dependencies] waggle-core = { workspace = true } -waggle-store = { path = "../waggle-store", version = "0.4.0" } +waggle-store = { path = "../waggle-store", version = "0.5.0" } serde_json = { workspace = true } thiserror = { workspace = true } pollster = { workspace = true } diff --git a/crates/waggle-store-sqlite/Cargo.toml b/crates/waggle-store-sqlite/Cargo.toml index 72c6ddc..2bacedf 100644 --- a/crates/waggle-store-sqlite/Cargo.toml +++ b/crates/waggle-store-sqlite/Cargo.toml @@ -14,7 +14,7 @@ homepage.workspace = true [dependencies] waggle-core = { workspace = true } -waggle-store = { path = "../waggle-store", version = "0.4.0" } +waggle-store = { path = "../waggle-store", version = "0.5.0" } thiserror = { workspace = true } rusqlite = { workspace = true } serde_json = { workspace = true } diff --git a/crates/waggle-tmux/Cargo.toml b/crates/waggle-tmux/Cargo.toml index 68b072c..f196717 100644 --- a/crates/waggle-tmux/Cargo.toml +++ b/crates/waggle-tmux/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "waggle-tmux" -version = "0.4.0" +version = "0.5.0" edition = "2021" rust-version = "1.85" description = "The seamless switchboard: move outcomes between Claude Code, Codex, and other tmux-resident harnesses as waggle tokens — minted in one gesture, resolved by the act of switching." diff --git a/edge-worker/Cargo.toml b/edge-worker/Cargo.toml index b550723..9ebd9cd 100644 --- a/edge-worker/Cargo.toml +++ b/edge-worker/Cargo.toml @@ -21,9 +21,9 @@ console_error_panic_hook = "0.1" base64 = "0.22" waggle-core = { workspace = true } waggle-ops = { workspace = true } -waggle-store = { path = "../crates/waggle-store", version = "0.4.0" } -waggle-store-cloudflare = { path = "../crates/waggle-store-cloudflare", version = "0.4.0" } -waggle-mcp = { path = "../crates/waggle-mcp", version = "0.4.0" } +waggle-store = { path = "../crates/waggle-store", version = "0.5.0" } +waggle-store-cloudflare = { path = "../crates/waggle-store-cloudflare", version = "0.5.0" } +waggle-mcp = { path = "../crates/waggle-mcp", version = "0.5.0" } waggle-social = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -37,6 +37,6 @@ base64 = "0.22" ureq = "2" serde_json = { workspace = true } waggle-core = { workspace = true } -waggle-store = { path = "../crates/waggle-store", version = "0.4.0" } -waggle-store-sqlite = { path = "../crates/waggle-store-sqlite", version = "0.4.0" } +waggle-store = { path = "../crates/waggle-store", version = "0.5.0" } +waggle-store-sqlite = { path = "../crates/waggle-store-sqlite", version = "0.5.0" } pollster = { workspace = true } From d14bec6e736aee9886475bebefea67f79e49da75 Mon Sep 17 00:00:00 2001 From: Chetan Conikee Date: Mon, 13 Jul 2026 16:13:36 -0700 Subject: [PATCH 12/12] fix: derive a tree node's subdir name from the filesystem, not its URL On Windows, a directory node's dir_url was built with dir.display() (backslash separators), and the parent derived each subdir's name by splitting that URL on '/', which returned the whole path instead of the leaf. The projection's subdir names were wrong, so a consumer could not find a subdirectory by name. Carry the real base name in NodeData (from entry.file_name(), correct on every platform) and drop the URL-parsing dir_name helper. Fixes the Windows-only content_access::coverage_is_per_file_over_a_tree failure; the other platforms were unaffected because '/' is their native separator. --- crates/waggle-mcp/src/tree_mint.rs | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/crates/waggle-mcp/src/tree_mint.rs b/crates/waggle-mcp/src/tree_mint.rs index e053569..6aec957 100644 --- a/crates/waggle-mcp/src/tree_mint.rs +++ b/crates/waggle-mcp/src/tree_mint.rs @@ -40,6 +40,10 @@ const DEFAULT_BUDGET: u64 = 256 * 1024 * 1024; /// already exist in the store, which post-order minting cannot provide. struct NodeData { token: Token, + /// This directory's own base name (the leaf), taken from the filesystem + /// entry — never parsed back out of a URL, so it is correct on every + /// platform's path separator. + name: String, /// `file://` URL of this directory. dir_url: String, /// The `tree` field this node's manifest will carry. @@ -200,7 +204,7 @@ impl Handler { .collect(); for sub in &children { entries.push(Entry::Dir(SubdirEntry { - name: dir_name(&sub.dir_url), + name: sub.name.clone(), token: sub.token.as_str().to_owned(), files: sub.files, bytes: sub.bytes, @@ -218,6 +222,10 @@ impl Handler { Ok(NodeData { token, + name: dir + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(), dir_url: format!("file://{}", dir.display()), tree_node: waggle_core::TreeNode { index: index_ref, @@ -402,17 +410,6 @@ fn index_text(content_type: &str, bytes: &[u8]) -> String { .unwrap_or_default() } -/// The final path component of a `file://…/dir` URL — a subdir's name within its -/// parent. -fn dir_name(dir_url: &str) -> String { - dir_url - .trim_end_matches('/') - .rsplit('/') - .next() - .unwrap_or(dir_url) - .to_owned() -} - /// Directory entries, sorted by name for deterministic node bytes. fn read_sorted(dir: &std::path::Path) -> Result, Envelope> { let mut entries: Vec<_> = std::fs::read_dir(dir)