From bffccdb5b7ea8797bcb1e12f4ff6ba52188a7bdf Mon Sep 17 00:00:00 2001 From: aaltshuler Date: Tue, 21 Jul 2026 05:05:07 +0300 Subject: [PATCH 1/2] Select unbounded MemWAL retention and close fold boundary --- .github/workflows/ci.yml | 52 + AGENTS.md | 4 +- WAL-thinking.md | 266 ++-- crates/omnigraph/src/db/omnigraph.rs | 12 +- .../src/db/omnigraph/stream_ingest.rs | 66 +- crates/omnigraph/tests/memwal_stream_cost.rs | 1076 ++++++++++++++++- docs/dev/canon.md | 91 +- docs/dev/ci.md | 4 +- docs/dev/index.md | 2 +- docs/dev/invariants.md | 73 +- docs/dev/lance.md | 133 +- docs/dev/rfc-022-027-architecture-review.md | 238 ++-- docs/dev/testing.md | 102 +- docs/dev/versioning.md | 14 +- docs/dev/writes.md | 72 +- docs/dev/writing-path-state-of-affairs.md | 210 ++-- docs/rfcs/0026-memwal-streaming-ingest.md | 1048 +++++++++------- docs/user/concepts/storage.md | 2 +- lance-memwal-pr.md | 448 +++++++ wal-options.md | 592 +++++++++ 20 files changed, 3590 insertions(+), 915 deletions(-) create mode 100644 lance-memwal-pr.md create mode 100644 wal-options.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e9efa55..42e2c4a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -211,6 +211,48 @@ jobs: # omnigraph/failpoints, so each line rebuilds only its crate. run: | cargo test --locked -p omnigraph-engine --features failpoints --test failpoints + output=$(cargo test --locked -p omnigraph-engine --features failpoints --test memwal_stream_cost gate_r0_ -- --nocapture 2>&1); status=$? + echo "$output" + [ "$status" -eq 0 ] || exit "$status" + echo "$output" | grep -Eq "test result: ok\. [1-9][0-9]* passed" \ + || { echo "::error::Gate R0 filter matched no tests — vacuous pass"; exit 1; } + echo "$output" | grep -Fq "Gate R0 widest local:" \ + || { echo "::error::Gate R0 local closure cell did not produce evidence"; exit 1; } + for test_name in \ + gate_r0_current_inventory_path_classifier_is_canonical_and_fail_closed \ + gate_r0_source_audit_revision_tripwire_matches_surveyed_lance \ + gate_r0_retain_all_current_object_growth_is_swept_explicitly \ + gate_r0_referenced_cut_retry_reuses_the_same_generation_root_local \ + gate_r0_widest_generation_closes_and_records_retain_all_growth_local + do + echo "$output" | grep -Eq "test ${test_name} \.\.\. ok" \ + || { echo "::error::required positive Gate R0 cell '${test_name}' did not execute normally"; exit 1; } + done + for test_name in \ + gate_r0_current_inventory_refuses_manifest_for_another_shard \ + gate_r0_current_inventory_refuses_duplicate_generation_authority \ + gate_r0_current_inventory_refuses_zero_current_generation \ + gate_r0_current_inventory_refuses_non_advanced_current_generation \ + gate_r0_current_inventory_refuses_manifest_filename_body_version_mismatch \ + gate_r0_current_inventory_refuses_gapped_manifest_version_chain \ + gate_r0_current_inventory_refuses_listed_shard_without_manifest + do + echo "$output" | grep -Eq "test ${test_name} - should panic \.\.\. ok" \ + || { echo "::error::required negative Gate R0 cell '${test_name}' did not execute as should-panic"; exit 1; } + done + + # This is a remeasurement tripwire for the current single-exclusive- + # fold shape, not a runtime allocator limit. Keep it outside the + # gate_r0_ filter because the subprocess owner has a stable older name. + output=$(cargo test --locked -p omnigraph-engine --features failpoints --test memwal_stream_cost widest_legal_generation_records_no_roll_estimates_and_peak_rss -- --exact --nocapture 2>&1); status=$? + echo "$output" + [ "$status" -eq 0 ] || exit "$status" + echo "$output" | grep -Eq "test result: ok\. 1 passed" \ + || { echo "::error::Gate R0 RSS filter matched no tests — vacuous pass"; exit 1; } + echo "$output" | grep -Eq "test widest_legal_generation_records_no_roll_estimates_and_peak_rss \.\.\. ok" \ + || { echo "::error::Gate R0 RSS remeasurement cell did not execute normally"; exit 1; } + echo "$output" | grep -Fq "fold_delta_remeasure_bytes=402653184" \ + || { echo "::error::Gate R0 RSS evidence line or 384-MiB tripwire changed"; exit 1; } cargo test --locked -p omnigraph-cluster --features failpoints --test failpoints - name: Commit regenerated openapi.json to PR branch @@ -472,6 +514,16 @@ jobs: echo "$output" | grep -Eq "test result: ok\. [1-9][0-9]* passed" \ || { echo "::error::filter 's3_' matched no tests — vacuous pass"; exit 1; } + output=$(cargo test --locked -p omnigraph-engine --features failpoints --test memwal_stream_cost gate_r0_widest_generation_closes_and_records_retain_all_growth_on_configured_rustfs -- --exact --nocapture 2>&1); status=$? + echo "$output" + [ "$status" -eq 0 ] || exit "$status" + echo "$output" | grep -Eq "test result: ok\. 1 passed" \ + || { echo "::error::RustFS Gate R0 filter matched no tests — vacuous pass"; exit 1; } + echo "$output" | grep -Eq "test gate_r0_widest_generation_closes_and_records_retain_all_growth_on_configured_rustfs \.\.\. ok" \ + || { echo "::error::RustFS Gate R0 cell did not execute with positive-test polarity"; exit 1; } + echo "$output" | grep -Fq "Gate R0 widest rustfs:" \ + || { echo "::error::RustFS Gate R0 test skipped instead of measuring configured storage"; exit 1; } + - name: Dump RustFS logs on failure if: failure() run: docker logs rustfs diff --git a/AGENTS.md b/AGENTS.md index 872f07ee..cb7f4128 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,7 +33,7 @@ OmniGraph is a typed property-graph engine built as a coordination layer over ma - **Multi-modal querying**: vector ANN (`nearest`), full-text (`search`/`fuzzy`/`match_text`/`bm25`), Reciprocal Rank Fusion (`rrf`), and graph traversal (`Expand`, anti-join `not { … }`) in one runtime. - **Branches and commits across the whole graph**: Git-style — every successful publish appends to a commit DAG; merges are three-way at the row level. - **Atomic per-query writes**: `mutate_as` and `load` accumulate insert/update batches into an in-memory `MutationStaging.pending` per touched table. Strict insert and upsert both route through the sealed exact-`id`, filter-bearing adapter; bare Lance Append is test-only. Their RFC-022 adapter resolves or rejects relevant recovery intents before base capture, captures `(native branch id, exact graph_head, schema identity)`, then rechecks recovery and authority under schema → branch → table gates. Mutation/Load keeps one keyed transaction per table and rejects more than 8,192 rows or 32 MiB before recovery arm; mutation update scans stream into the remaining table budget after pending-key shadowing, with blob sizes checked before payload reads. It then arms an identity-bearing recovery-v9 sidecar containing exact Lance transaction identities + pre-minted lineage, commits each table with zero transparent conflict retries, confirms the achieved effects, and publishes under the same token. Unrelated retryable pre-effect authority movement may fully reprepare without changing logical mode. A proven effect-free strict conflict returns `KeyConflict` only after a fresh exact-ID probe; no exact match triggers bounded full strict-mode reprepare, while an effect-free upsert conflict also fully reprepares. Any earlier effect or ambiguity returns `RecoveryRequired` with the sidecar retained. Strict read-set conflicts return `ReadSetChanged`. Deletes stage through the same path. D₂ at parse time remains the constructive (insert/update) XOR destructive (delete) boundary. -- **RFC-026 private streaming core**: Phase A's main-only adapter enrolls one exact-`id` table into one empty unsharded Lance MemWAL under recovery-v10, then publishes its physical binding and `OPEN` lifecycle. Phase B1 adds a root-singleflight worker for one no-roll generation (8,192 rows / 32 MiB), watcher-only durable acknowledgement, conservative replay/fold-only recovery, and one recovery-v11 strict exact-`id` fold whose sole `__manifest` CAS advances the table pointer, lifecycle witness/epoch floor, and graph lineage together. Cheap raw caller row/byte bounds reject obviously over-cap input before recovery I/O; raw-fit input then receives exact post-tombstone validation at that same pre-recovery boundary. After any recovery/authority prelude, every put follows exact charge → shared admission → same-key input queue → worker mode; the permit transfers into the resident generation. Cold replay installs exact accounting plus a fold-only marker before releasing the opener queue; previously charged callers may create a recorded transient overlap above the nominal 32-MiB root budget while they drain, but new charge is refused. A cold fold reserves the full generation/resident/pending envelope before its owned opener and retains it with exclusive authority across the original seal deadline. Shared admission is retained through put/watcher; exclusive admission spans seal, drain proof, table effect, and manifest publication. Post-invocation ambiguity is typed `AckUnknown`; unresolved recovery blocks progress. The topology remains main-only, unsharded, one resident writer and one live writer process, with no fresh reads or generation GC. The core remains crate-private; its only callable integration seam is a feature-gated, doc-hidden test method. B2-0 specifies explicit enrollment, compare-and-chain tokens, trusted attribution, a manifest-selected graph-global current-token participant, monotonic lifecycle revisions with bounded management receipts, strict correction, Lance-owned reclamation with an enforced retained-storage admission watermark, and a manifest-authoritative graph-global `GraphHistoryBudget` checked by every manifest publisher. That budget reserves each publication's source-bounded physical-growth envelope and preserves dynamic per-stream correction/quiesce/`SEALED` closure reserves; none of B2 is implemented. Two RC.1 guards prove generic cleanup does not reclaim `_mem_wal` and raw deletion of the successor's empty WAL fence sentinel is unsafe, so OmniGraph never deletes those paths. Public B2 additionally waits for durable reclaim attempt/receipt recovery, bounded history checkpointing, strong inventory or Lance-owned accounting, and source-derived enforced physical-growth reservations. No schema-v9, SDK, CLI, HTTP, Cedar, or OpenAPI streaming contract ships until the Lance patch and all B2 evidence gates pass. +- **RFC-026 private streaming core**: Phase A's main-only adapter enrolls one exact-`id` table into one empty unsharded Lance MemWAL under recovery-v10, then publishes its physical binding and `OPEN` lifecycle. Phase B1 adds a root-singleflight worker for one no-roll generation (8,192 rows / 32 MiB), watcher success plus the same writer's post-durability `check_fenced()` before clean acknowledgement, conservative replay/fold-only recovery, and one recovery-v11 strict exact-`id` fold whose sole `__manifest` CAS advances the table pointer, lifecycle witness/epoch floor, and graph lineage together. Cheap raw bounds and exact post-tombstone validation run before recovery; put ownership follows charge → shared admission → same-key queue → worker mode; cold replay installs exact fold-only accounting; exclusive admission spans seal, drain proof, table effect, and publication. Fold now charges the scanner's logical slices against the same 32-MiB generation cap and copies each scanner emission into dense owned arrays before retaining it, so the 8,192-row high-entropy near-cap closure cell folds and publishes successfully; its isolated fold RSS delta was 284,934,144 bytes (about 272 MiB), below the 384-MiB remeasurement tripwire. Post-invocation ambiguity is `AckUnknown`; unresolved recovery blocks progress. The topology remains main-only, unsharded, one resident writer and one live writer process, with no fresh reads or generation GC. The selected next profile is **unbounded retain-all**: OmniGraph imposes no retained-byte, object-count, file-count, or history quota and never deletes raw `_mem_wal` paths. Provider exhaustion is a loud storage failure and may halt admission, fold, or recovery progress; it never permits silent loss or publication of partial state. Row count, Arrow memory, deadlines, retries, and ambiguous outcomes remain bounded. RC.1's missing cross-open materialization-attempt receipt and complete physical-output envelope are therefore not blockers for this profile; they remain relevant to the future B2b managed-reclamation design. Compare-and-chain tokens, trusted attribution, lifecycle receipts, correction, authorization, and product parity remain required and inactive. The core remains crate-private behind one feature-gated, doc-hidden test seam; no schema-v9, SDK, CLI, HTTP, Cedar, or OpenAPI streaming contract ships from this result. - **HTTP server**: Axum + utoipa OpenAPI, bearer auth (SHA-256 hashed, optional AWS Secrets Manager). Cedar policy enforcement is engine-wide — every `_as` writer calls `Omnigraph::enforce(action, scope, actor)`, so HTTP, CLI, and embedded SDK consumers all hit the same gate. **Cluster-only boot** (RFC-011): the server always boots from a cluster directory (`--cluster `, RFC-005) and serves N graphs (N ≥ 1) under multi-graph routes (`/graphs/{graph_id}/...` + read-only `GET /graphs` enumeration); there are no single-graph flat routes and no positional-URI boot. Per-graph + server-level Cedar policies. Runtime add/remove (`POST /graphs`, `DELETE /graphs/{id}`) is not exposed — operators run `cluster apply` and restart. - **CLI** with two-surface config (RFC-007/008): the team-owned cluster directory (`cluster.yaml`) plus the per-operator `~/.omnigraph/config.yaml` (servers, clusters, credentials, actor, profiles, aliases, defaults). Graphs are addressed via `--store`/`--server`/`--cluster`/`--profile`/operator defaults (RFC-011). Multi-format output (json/jsonl/csv/kv/table). @@ -280,7 +280,7 @@ omnigraph policy explain --cluster ./company-brain --graph knowledge --actor act | Cleanup (`cleanup_old_versions`) | ✅ | `omnigraph cleanup` derives requested `--keep` / `--older-than` cutoffs from each table's available versions; Lance refs plus OmniGraph's live-lazy-branch and recovery floors may retain additional versions. It fails closed on unopenable pins, recovery intent, or uncovered main-table HEAD drift | | BTREE / inverted (FTS) / vector indexes | ✅ | `@index`/`@key` declares intent; the physical index is derived state that never fails a logical op. Built per column through one chokepoint (`build_indices_on_dataset_for_catalog`, type-dispatched by `node_prop_index_kind`: enum + orderable scalar → BTREE, free-text String → FTS, Vector → vector); idempotent; lazy across branches. **Schema apply and mutation/load build no indexes inline**: they publish only logical data/schema effects, leaving physical intent pending. `ensure_indices` first runs the roll-forward-only recovery barrier before base capture or planning, then materializes every declared-but-missing artifact for one table through one staged mixed CreateIndex transaction under its identity-bearing recovery-v9 authority/lineage/delta envelope; it continues to report untrainable Vector columns as pending. `Armed` is rollback-only, `EffectsConfirmed` rolls forward only while captured authority holds, and first-touch refs carry exact identity. `optimize` separately restores fragment coverage through its bounded identity-bearing v9 maintenance envelope. | | Strict insert / upsert ingestion | ✅ transaction conflict filters + uncommitted fragment staging | Internal schema v6 introduced the explicit logical mode and v8 preserves it. General strict insert and upsert use the sealed exact-`id`, forced-v2 MergeInsert adapter; strict insert exact-probes its pinned parent before minting `omnigraph.insert_absence=v1`, while an all-new upsert may mint the same optional certificate only from its completed effect statistics. Mutation/Load remains one transaction per table, capped before arm at 8,192 rows / 32 MiB. BranchMerge's proven all-new route accepts only a complete certificate chain plus final source/target native-incarnation checks; it stages bounded fragments with `InsertBuilder`, commits them as exact-`id` filtered `Update` transactions, and performs zero target preflights, target merge joins, or committed Appends. Missing or malformed proof falls back to the general ordered diff. Raw Lance writers are outside the supported graph-writer topology, and the certificate is an internal, non-cryptographic capability. The final five-pair production gate passed at 10K (3.875× median; 24,297,472-byte max paired RSS overhead) and 100K (~3.886×; 32,604,160 bytes). | -| MemWAL streaming foundation | ✅ MemWAL system index, durable WAL generations, epoch-fenced shard writer | Internal schema v8 carries exact identity-keyed physical binding, current-HEAD witness, `OPEN`/`DRAINING`/`SEALED` state, per-shard epoch floor, and stream-config v2. Phase A's recovery-v10 adapter enrolls one empty unsharded shard. Private B1 adds one root-singleflight, no-roll generation capped at 8,192 rows / 32 MiB; watcher-only durable ack; detached ownership across cancellation/timeouts; fold-only replay using the pinned BatchStore watermark bridge; explicit seal/drain proof; and recovery-v11 exact-transaction fold followed by one graph-visibility CAS. Root-shared admission and lifecycle guards exclude competing table effects; a claimed/warm ambiguity retires and aborts before reuse. B1 performs no generation GC and exposes no fresh reads. B2-0 specifies the inactive schema-v9/config-v3/state-v2/recovery-v12 explicit-enrollment, token, attribution, lifecycle-revision/management-receipt, correction, Lance-owned reclamation, per-binding admission-watermark, and graph-global `GraphHistoryBudget` contracts. The latter is checked by every manifest writer and keeps independent per-stream closure reserves. There is still no public stream schema, enrollment, put/fold/status API, Cedar action, CLI, HTTP route, or OpenAPI contract; those remain B2. | +| MemWAL streaming foundation | ✅ MemWAL system index, durable WAL generations, epoch-fenced shard writer | Internal schema v8 carries exact identity-keyed physical binding, current-HEAD witness, `OPEN`/`DRAINING`/`SEALED` state, per-shard epoch floor, and stream-config v2. Phase A's recovery-v10 adapter enrolls one empty unsharded shard. Private B1 adds one root-singleflight, no-roll generation capped at 8,192 rows / 32 MiB; watcher success plus same-writer post-durability epoch check; detached ownership; fold-only replay; seal/drain proof; and recovery-v11 exact-transaction fold followed by one graph-visibility CAS. The fold charges logical slices and densifies each scanner emission, so the deterministic 8,192-row/near-32-MiB high-entropy generation now closes; the measured isolated fold RSS delta is 284,934,144 bytes (about 272 MiB), below the 384-MiB remeasurement tripwire. It performs no generation GC or fresh reads. The selected unbounded retain-all profile has no OmniGraph byte/object/file/history quota and no raw `_mem_wal` deletion; provider exhaustion fails loudly and may stop progress. Row/logical-Arrow admission, deadline, retry, and ambiguity bounds remain. B2b retains the future Lance-owned reclamation design. Common schema-v9/config-v3/state-v2/recovery-v12 enrollment/token/attribution/lifecycle/correction contracts remain inactive. There is no public stream schema, enrollment, put/fold/status API, Cedar action, CLI, HTTP route, or OpenAPI contract. | | Vector search | ✅ | `nearest()` query op; embedding pipeline (Gemini / OpenAI clients); `@embed` in schema | | Full-text search | ✅ | `search/fuzzy/match_text/bm25` query ops | | Hybrid ranking | — | `rrf(...)` Reciprocal Rank Fusion in one runtime | diff --git a/WAL-thinking.md b/WAL-thinking.md index 12e26ffa..cdcd275c 100644 --- a/WAL-thinking.md +++ b/WAL-thinking.md @@ -1,14 +1,21 @@ # WAL Thinking -Working notes, updated 2026-07-19. Plain-language grounding for the WAL/streaming +Working notes, updated 2026-07-21. Plain-language grounding for the WAL/streaming discussion ([RFC-018](docs/rfcs/0018-ingest-wal.md) → [RFC-026](docs/rfcs/0026-memwal-streaming-ingest.md)). Three parts: the contract difference between an interactive graph commit and durable stream admission, the expected performance shape and evidence still required, and the build inventory. -Current boundary: RFC-026 Phase A and the evidence-green Phase-B1 private core -are built, but public streaming is not. Current internal schema v8 preserves +Current boundary: RFC-026 Phase A and the Phase-B1 private core are built, but +public streaming is not. Gate R0 found and the follow-up fixed the one known +all-shape closure failure: a legal high-entropy near-cap generation is durably +acknowledged, materialized, folded, and published without lowering the +8,192-row/32-MiB admission cap. The fold now charges logical Arrow slices and +densifies selected rows before retaining them. The measured full-fold RSS delta +was 284,934,144 bytes (about 272 MiB); CI carries a 384-MiB remeasurement +tripwire for one exclusive fold, not a runtime allocator limit. Current +internal schema v8 preserves Phase A's historical v7 foundation: exact empty main-only MemWAL enrollment recovery, identity-keyed lifecycle authority, exclusion of ordinary local table/schema/ maintenance/repair/recovery effects for a lifecycle-bound table, and @@ -17,17 +24,24 @@ because they do not move table HEAD. V8 adds stream-config v2 and recovery-v11 `StreamFold`. One feature-gated, doc-hidden engine seam can privately admit an already-normalized physical -batch, acknowledge only its successful Lance durability watcher, route replay -or one flushed-unmerged generation fold-only, and publish one exact fold at the +batch, acknowledge only after its Lance durability watcher and the same +writer's post-durability epoch check both succeed, route replay or one +flushed-unmerged generation fold-only, and publish one exact fold at the `__manifest` CAS. It prevents MemTable rollover and retires the writer before a successor generation can put. This is implementation/evidence machinery, not a product surface. There is still no `@stream`, public enrollment or row admission, SDK/HTTP/CLI/OpenAPI route, operator drain/resume workflow, or fresh -read. RFC-026 remains Draft. Phase B2-0 now specifies the compare-and-chain -token, trusted attribution, persistent lifecycle/correction, and Lance-owned -reclamation contracts, but activates none of them. Phase B2 is the later -implementation and public strict activation, and v7/config-v1 is never adopted -in place. +read. RFC-026 remains Draft. + +RFC-026 now selects **unbounded retain-all on stock Lance** as the first storage +profile. OmniGraph deletes no `_mem_wal` object and advertises no file-count or +retained-byte limit. RC.1's missing durable materialization-attempt receipt and +complete physical-output envelope therefore remain useful facts, but no longer +block activation: the selected contract deliberately accepts monotonic storage +and loud provider-capacity exhaustion. Managed reclamation and the Lance patch +are deferred optimizations. Common compare-and-chain token, trusted +attribution, persistent lifecycle/correction, authorization, and product- +parity contracts remain specified and inactive. --- @@ -120,8 +134,9 @@ exact charge is recomputed and reserved before the caller can wait in the same-key queue, acquire shared admission, detach, or cold-claim a writer. The acknowledgement says only that the submitted batch crossed its Lance -durability watcher. That successful watcher is the only acknowledgement -authority. The watcher returns success/failure, not a durable WAL row +durability watcher and the same writer then passed a post-durability epoch +check. Both are required acknowledgement authority. The watcher returns +success/failure, not a durable WAL row coordinate. Lance's returned batch positions are active-MemTable positions that reset after rollover, and its next-WAL-position statistic is a mutable hint, so neither may appear as a receipt. If invocation may have had an effect @@ -131,7 +146,7 @@ attempt. The attempt remains ambiguous and OmniGraph must never claim “not durable.” Cancellation cannot abandon the worker once invocation starts. The current private B1 seam has only cardinality-level same-key retry behavior: ambiguous `X(id)`, then durable `Y(id)`, then retry `X(id)` can make stale `X` -newest again. B2-0 closes that public-contract hole with an explicit +newest again. The common B2 contract inventory closes that public-contract hole with an explicit compare-and-chain token. A caller-stable `write_id` plus the opaque `predecessor_token` returned by the server derives one stable successor token; the predecessor must equal the complete current token before Lance is called. @@ -154,7 +169,7 @@ horizontal writer scaling comes later from deterministic key-based sharding. The strict one-generation fold is implemented privately in B1. Phase A established the binding, recovery, and exclusion preconditions it consumes; -B2-0 specifies the token participant and persistent operator lifecycle that +B2's common contract inventory specifies the token participant and persistent operator lifecycle that Phase B2 must implement before exposing the public fold. B1 deliberately chooses a smaller contract: one writer owns exactly one @@ -193,24 +208,20 @@ a fold: The fold is a new stream-specific RFC-022 effect adapter on the existing publication and recovery chassis. It reuses that chassis, but it adds stream -binding and generation authority, fold-time validation, merge progress, -reject/audit atomicity, and later GC eligibility. - -Folded data is not deleted from MemWAL immediately. WAL entries and flushed -generations become eligible for later GC only after the exact fold is -graph-visible, its recovery sidecar is resolved, maintained indexes have caught -up, retained versions no longer need the data, no fresh-read guard pins it, and -writer fencing remains safe. - -Stock RC.1 cannot perform that reclamation safely. Generic table-version -cleanup leaves `_mem_wal` untouched, and deleting the successor's empty WAL -fence sentinel can let a stale writer report a later WAL PUT as durable because -RC.1 lacks a post-success epoch check. OmniGraph therefore never deletes raw -MemWAL paths. The next substrate slice is Lance-owned durable -inspect/plan/execute with attempt/receipt recovery, bounded history -checkpointing, an enforceable physical-growth reservation, and the -post-success fence check, authored by us and pinned to an exact reviewed fork -commit without waiting for an upstream release. +binding and generation authority, fold-time validation, merge progress, and +reject/audit atomicity. + +The selected retain-all profile never deletes folded MemWAL data. WAL entries, +flushed generations, random failed-attempt subtrees, and fence sentinels remain +in the root even after the exact fold is graph-visible and recovery is settled. +That is deliberate: generic table-version cleanup leaves `_mem_wal` untouched, +and deleting the successor's empty WAL fence sentinel can let a stale writer +report a later WAL PUT as durable because RC.1 lacks a post-success epoch +check. OmniGraph therefore never deletes raw MemWAL paths. + +A Lance-owned inspect/plan/execute protocol can add managed reclamation later. +Its attempt receipts, inventory, fencing, and accounting are no longer the next +activation slice because the first profile promises no retained-storage bound. ### Side by side @@ -236,9 +247,9 @@ commit without waiting for an upstream release. 3. **More lifecycle machinery.** Enrollment, owner routing, backpressure, drain/seal/resume, replay, fold health, and cleanup become operator-visible responsibilities. -4. **Write and storage amplification.** Data may exist in a WAL entry, a - flushed Lance generation, the base dataset, and maintained indexes before - obsolete copies can be reclaimed. +4. **Write and monotonic storage amplification.** Data may exist in a WAL + entry, a flushed Lance generation, the base dataset, and maintained indexes. + Under retain-all, those MemWAL copies are not reclaimed in place. One line for the team: @@ -317,15 +328,17 @@ complete end-to-end call count: they cannot be added into a truthful “12 calls per write” result. See [`write_cost.rs`](crates/omnigraph/tests/write_cost.rs) and -[`memwal_stream_cost.rs`](crates/omnigraph/tests/memwal_stream_cost.rs). B1 now -has accepted component evidence locally and on configured RustFS: warm -already-claimed acknowledgement is flat at the measured compacted-history -endpoints, while cold replay, retained generation metadata, selected-generation -scan, widest-generation RSS, and the known non-flat uncompacted-manifest fold -term are reported separately. There is still no matched end-to-end comparison -of one-row interactive latency, sustained branch throughput, and a public -streaming workload. RFC-018's historical single-digit language and upstream -prototype results are not measurements of that product comparison. +[`memwal_stream_cost.rs`](crates/omnigraph/tests/memwal_stream_cost.rs). B1 has +accepted component evidence for its measured fixture set: the current +post-containment warm already-claimed acknowledgement result is local, while +the configured-RustFS figures are a 2026-07-19 pre-containment baseline that +must be rerun before a current object-store claim. Cold replay, retained +generation metadata, selected-generation scan, widest-generation RSS, and the +known non-flat uncompacted-manifest fold term are reported separately. There is +still no matched end-to-end comparison of one-row interactive latency, +sustained branch throughput, and a public streaming workload. RFC-018's +historical single-digit language and upstream prototype results are not +measurements of that product comparison. Consequently, the following are hypotheses rather than results: @@ -338,8 +351,9 @@ Consequently, the following are hypotheses rather than results: ### Required benchmark before quoting a multiplier -The private B1 admission component instrument is already accepted locally and -on configured RustFS. It does not supply the missing matched product comparison. +The private B1 admission component instrument has current local evidence and a +historical pre-containment configured-RustFS baseline. It does not supply the +missing matched product comparison or a current object-store result. Before quoting a multiplier, build and accept a separate end-to-end comparison: 1. fix schema, row shape, row bytes, table count, and index configuration; @@ -351,8 +365,8 @@ Before quoting a multiplier, build and accept a separate end-to-end comparison: latency, and fold visibility lag; 5. count GET/HEAD/LIST/PUT/DELETE and multipart operations separately for the acknowledgement and background paths; -6. record serial critical-path hops, uploaded bytes, retained bytes before and - after GC, fold rows/bytes/generations, and every affected dataset; +6. record serial critical-path hops, uploaded bytes, monotonic retained bytes, + fold rows/bytes/generations, and every affected dataset; 7. run multiple fresh trials and preserve the raw records. The comparison must state the semantic difference: interactive success is @@ -382,7 +396,7 @@ These are real substrate capabilities, not a turnkey graph-streaming product: | **`merged_generations`** | Merge progress updated atomically with a base-table merge. It supplies the marker needed for idempotent per-table folding; RFC-022 still owns graph publication and partial-effect recovery. | | **WAL replay** | Durable entries are replayed to reconstruct the MemTable under the next valid claimant. The guarantee depends on respecting fencing, lifecycle, compatibility, and safe GC. | | **LSM merging reads** | A scanner over the base table, selected flushed generations not yet safely replaceable by the base read plan, and optionally same-process active/frozen MemTables. OmniGraph must build and retain a coherent graph-level fresh-read cut. It does not query raw WAL files as a normal read source. | -| **GC eligibility rules** | Upstream specifies when generations may be obsolete and warns that deleting WAL files can weaken fencing. Stock RC.1 does not provide a safe graph-aware MemWAL collector: generic cleanup ignores `_mem_wal`, and there is no post-success WAL epoch check. B2 consumes a Lance-owned opaque reclamation primitive; OmniGraph never deletes raw paths. | +| **Retention behavior** | Upstream describes when generations may be obsolete and warns that deleting WAL files can weaken fencing. The selected profile does not turn eligibility into deletion: generic cleanup ignores `_mem_wal`, OmniGraph never deletes raw paths, and retained storage grows monotonically. A later managed profile may consume a Lance-owned opaque reclamation primitive. | | **Staged merge-insert** | `execute_uncommitted` plus atomic `merged_generations` fits the RFC-022 staged shape. There is no one-call MemWAL fold API. | | **Shared sessions/store parameters in RC.1** | Writer-created generations and base-backed scanner paths reuse the base dataset's access context; fresh-only construction still receives that context from its caller. OmniGraph owns long-lived writer/session lifecycle and scanner integration. | | **Key-based sharding** *(later)* | Horizontal scale when every occurrence of one key deterministically maps to one shard. | @@ -391,12 +405,13 @@ OmniGraph already has reusable chassis: the manifest publisher, generic recovery-sidecar framework, graph lineage, stable table identity, keyed write adapter, and validation components. “Reusable” does not mean “unchanged”: Phase A added stream lifecycle authority and enrollment recovery. Private B1 -now adds one bounded admission worker, watcher-only acknowledgement, replay and -flushed-generation classification, and one recovery-owned fold. Later phases -still need implementation of B2-0's token/attribution, persistent operator -lifecycle/correction, Lance-owned reclamation and enforced admission watermark; -then the public -caller, reject participants, fresh cuts, and later cleanup coordination. +now adds one nominally bounded admission worker, watcher success plus the same +writer's post-durability epoch check, replay and flushed-generation +classification, and one recovery-owned fold. The all-shape closure repair is +implemented. Later phases still need the common token/attribution, persistent +operator lifecycle/correction, authorization, and product contracts; then the +public caller, reject participants, and fresh cuts. Managed cleanup is optional +future work rather than an activation prerequisite. ### What Phase A added @@ -428,24 +443,24 @@ workflow is implemented. | Responsibility | Required contract | |---|---| -| **Format capability and refusal** | **Phase A historical foundation:** v7 stamp, strict v6↔v7 refusal, export/init/load rebuild, exact lifecycle validation, and uncovered-partial-format refusal. **Private B1 current format:** schema v8, stream-config v2, and recovery-v11 `StreamFold`; genuine v7↔v8 old/new-binary refusal and rebuild evidence is green. **Specified B2 format:** schema v9, config-v3, state-v2, and recovery-v12 add hidden token/attribution state plus the manifest-authoritative graph-global `GraphHistoryBudget`; they remain inactive until genuine v8↔v9 refusal/rebuild and budget-init/refusal evidence pass. | +| **Format capability and refusal** | **Phase A historical foundation:** v7 stamp, strict v6↔v7 refusal, export/init/load rebuild, exact lifecycle validation, and uncovered-partial-format refusal. **Private B1 current format:** schema v8, stream-config v2, and recovery-v11 `StreamFold`; genuine v7↔v8 old/new-binary refusal and rebuild evidence remains green, and the widest legal fold now closes. **Specified common B2 format:** schema v9, config-v3, state-v2, and recovery-v12 add hidden token/attribution/lifecycle state; they remain inactive until their genuine v8↔v9 refusal/rebuild evidence passes. Retain-all requires no storage-budget format. | | **`@stream` intent and enrollment** | **Foundation only:** Phase A binds stable table/incarnation identity, location/main ref, never-reused enrollment ID, one empty shard, fixed configuration, and the mutable current-HEAD witness under recovery. B1 is implemented but remains private. B2 makes `@stream` declaration leave the type `UNENROLLED`; an explicit request-idempotent enroll operation creates the logical stream incarnation and physical binding. Rebind remains later. | | **Public surface** | **Phase B2, after private evidence:** explicit enroll, `POST /graphs/{graph_id}/streams/{type_name}/ingest`, minimum status/block-inspection/fold/quiesce/resume/abort-drain/correct/rebuild-preflight controls, `omnigraph stream …` commands, Cedar, and OpenAPI parity. Every mutating management call after enrollment compares a lifecycle revision and durably returns its bounded terminal receipt. Existing `/ingest` remains the deprecated load alias. | -| **Writer registry and routing** | **Phase B1 implemented privately:** one root-scoped, cross-handle registry owns the full binding and reuses the common table-identity plus resolved-physical-ref admission key; one serialized owner serves the initial `(table, main)` profile. One no-rollover generation is capped in total at 8,192 rows / 32 MiB. Puts use exact charge → shared admission → same-key input queue → worker-mode inspection. Claim/replay starts under the shared admission lease. Empty reopen may admit; non-empty replay and one flushed-unmerged generation are fold-only, with exact accounting and the refusal marker installed before the opener releases its queue. Already-charged callers can overlap recovered replay transiently; the ledger records that over-cap overlap while refusing new charge. The exclusive fold validates replayed rows and uses the pinned public BatchStore watermark bridge before reseal. A cold fold reserves the full generation/resident/pending envelope before its owned opener and retains it with exclusive authority across the original seal deadline. Retirement stops puts before public `abort`; `close` is not durability evidence. The private evidence limits are fixed at one resident writer per root/table, a nominal 32-MiB root Arrow admission budget, 32 in-flight calls, four pending generations, 30-second put, 60-second seal/abort, and 60-second idle eviction. They are not public defaults: B2 must requalify public/configurable or multi-resident limits with matched RSS/latency evidence. | -| **Durability batching** | **Phase B1 implemented privately:** one admitted call is one non-empty, already-normalized physical `RecordBatch` and one Lance put. The worker owns the final check, invocation, and watcher; only watcher success acknowledges. Anything ambiguous after invocation is typed `AckUnknown`; replay preserves possible residue but never resolves that attempt. Automatic rollover is disabled and the writer retires before a successor-generation put. There is no hidden group-commit policy until an instrument justifies one. | +| **Writer registry and routing** | **Phase B1 implemented privately:** one root-scoped, cross-handle registry owns the full binding and reuses the common table-identity plus resolved-physical-ref admission key; one serialized owner serves the initial `(table, main)` profile. One no-rollover generation has an 8,192-row / 32-MiB admission cap. Puts use exact charge → shared admission → same-key input queue → worker-mode inspection. Claim/replay starts under the shared admission lease. Empty reopen may admit; non-empty replay and one flushed-unmerged generation are fold-only, with exact accounting and the refusal marker installed before the opener releases its queue. Already-charged callers can overlap recovered replay transiently; the ledger records that overlap while refusing new charge. The exclusive fold validates replayed rows and uses the pinned public BatchStore watermark bridge before reseal. A cold fold reserves the full generation/resident/pending envelope before its owned opener and retains exclusive authority across the original seal deadline. Retirement stops puts before public `abort`; `close` is not durability evidence. Fold scanning now charges logical slices and densifies selected rows, so the near-cap closure cell succeeds. These are not public defaults. | +| **Durability batching** | **Phase B1 implemented privately:** one admitted call is one non-empty, already-normalized physical `RecordBatch` and one Lance put. The worker owns the final check, invocation, watcher, and post-durability epoch check; only watcher success followed by same-writer `check_fenced()` success acknowledges. Anything ambiguous after invocation is typed `AckUnknown`; replay preserves possible residue but never resolves that attempt. Automatic rollover is disabled and the writer retires before a successor-generation put. There is no hidden group-commit policy until an instrument justifies one. | | **Pre-ack validation** | **Phase B1 implemented privately:** apply every rule that needs no graph/base-table read before WAL persistence. The private seam consumes physical vectors supplied in the normalized batch; it neither calls an external embedding provider nor invents unspecified fold-derived fields. | -| **Fold adapter** | **Phase B1 implemented privately:** capture one exact stream binding and post-drain shard snapshot, independently prove empty frozen refs plus the exact authoritative generation/cursor (RC.1's drain waiter alone is insufficient), run base-dependent validation, require the LWW output to fit 8,192 rows / 32 MiB, stage one merge-insert with `merged_generations`, and publish the exact table/lifecycle/lineage outcome through recovery schema v11 plus one `__manifest` CAS. Already-normalized physical vectors pass through unchanged. Seal/drain/abort stay background-owned across caller deadlines; recognized unreferenced generation subtrees are retained orphans until proved GC. | -| **Strict and dead-letter disposition** | **B1/B2:** strict only; a permanent deferred-validation failure leaves durable unmerged input and blocks progress loudly. B1 has no correction lane. B2-0 specifies bounded `REPLACE`/`WITHDRAW` correction over the immutable blocked cut, without a second generation or silent drop. **Phase C:** dead letters only after restart-stable reject identity and retention are defined. | -| **Policy, lineage, and audit** | B1 records only fixed mechanism lineage and has no public caller. B2-0 stores trusted contributor/write metadata with the row, publishes current token state with the base version, and commits a fixed winner summary; it promises durable attribution for visible winners/current withdrawals, not unbounded audit retention for every superseded acknowledgement. Phase C consumes that evidence for rejects. | -| **Quiescence and rebind** | B2-0 specifies durable `OPEN -> DRAINING -> SEALED -> OPEN`, strictly monotonic lifecycle revisions, bounded complete terminal management receipts, authoritative status, roll-forward-only resume/abort-drain recovery, rebuild preflight, and bounded correction. B2 must implement and prove them before public activation. Resume rechecks the bounded no-named-branch topology; an incompatible branch operation leaves the stream `SEALED`. `SEALED` permits export/rebuild, not in-place maintenance. Phase D integrates automatic operation drain and physical rebind. | +| **Fold adapter** | **Phase B1 implemented privately and closure-green:** capture one exact stream binding and post-drain shard snapshot, independently prove empty frozen refs plus the exact authoritative generation/cursor (RC.1's drain waiter alone is insufficient), run base-dependent validation, require the LWW output to fit 8,192 rows / 32 MiB, stage one merge-insert with `merged_generations`, and publish the exact table/lifecycle/lineage outcome through recovery schema v11 plus one `__manifest` CAS. Scanner slices are charged by logical size and rebuilt densely before retention. Already-normalized physical vectors pass through unchanged. Seal/drain/abort stay background-owned across caller deadlines; recognized unreferenced generation subtrees are retained forever under the selected profile. | +| **Strict and dead-letter disposition** | **B1/common B2:** strict only; a permanent deferred-validation failure leaves durable unmerged input and blocks progress loudly. B1 has no correction lane. The common B2 inventory specifies bounded `REPLACE`/`WITHDRAW` correction over the immutable blocked cut, without a second generation or silent drop. **Phase C:** dead letters only after restart-stable reject identity and retention are defined. | +| **Policy, lineage, and audit** | B1 records only fixed mechanism lineage and has no public caller. The common B2 inventory stores trusted contributor/write metadata with the row, publishes current token state with the base version, and commits a fixed winner summary; it promises durable attribution for visible winners/current withdrawals, not unbounded audit retention for every superseded acknowledgement. Phase C consumes that evidence for rejects. | +| **Quiescence and rebind** | The common B2 inventory specifies durable `OPEN -> DRAINING -> SEALED -> OPEN`, strictly monotonic lifecycle revisions, bounded complete terminal management receipts, authoritative status, roll-forward-only resume/abort-drain recovery, rebuild preflight, and bounded correction. Retain-all is selected, but these lifecycle contracts still must be implemented and proved before public activation. Resume rechecks the bounded no-named-branch topology; an incompatible branch operation leaves the stream `SEALED`. `SEALED` permits export/rebuild, not in-place maintenance. Phase D integrates automatic operation drain and physical rebind. | | **Fresh reads** *(later)* | Explicit committed/fresh IR mode, exact base-plus-MemWAL cut, merged-generation exclusion, retention guards, and documented lack of cross-table atomicity. | -| **Cleanup** | B1 performs no WAL/generation GC. B2-0 proves stock RC.1 generic cleanup is not the answer and specifies Lance-owned durable inspect/plan/execute, post-success fencing, whole-cut eligibility, strong inventory or durable accounting, bounded history checkpointing, and an enforceable retained-object/byte admission watermark. B2 production activation waits for that patch and local/RustFS crash/bound evidence. Later graph-aware GC remains integrated with visibility, recovery, time travel, indexes, fresh-read guards, and fencing safety. | -| **Graph-manifest lifetime** | The per-binding MemWAL watermark cannot bound shared `__manifest` history. B2-0 therefore specifies one graph-global `GraphHistoryBudget` initialized with schema v9 and checked by every manifest writer. Each publication uses a reserve-first, source-bounded physical-growth envelope; the initial profile holds that global gate from sidecar arm through effect/CAS/finalization, and pending recovery charges settle exactly once. Dynamic per-stream closure reserves cannot be borrowed by ordinary work or another stream, and `GraphRebuildRequired` stops ordinary work at the shared floor while correction/quiesce/`SEALED` rebuild remains funded. The canonical order is relevant stream admission → graph history → schema → main → token → tables. This contract is specified and inactive. | -| **Evidence and operations** | **B1 green privately:** surface guards, genuine v7↔v8 refusal/rebuild, all 24 cells in the graph-level B1 behavior/crash/race suite, qualified local cost/PK-index/RSS evidence, and configured-RustFS cost evidence pass. The instrument keeps warm ack, retained-WAL replay, selected-generation data scan, accumulated retained shard-generation metadata, whole-process RSS, and the non-flat graph-manifest publication-history term separate. **B2-0:** two checked-in guards prove generic cleanup non-ownership and the deleted-successor-sentinel stale-writer hazard. B2 still needs the positive Lance patch matrix, source-derived/enforced growth reservation plus validating instrument, public metrics/status, API tests, CLI parity, and operator evidence. | +| **Retention profile** | **Unbounded retain-all is selected:** no OmniGraph MemWAL GC, no file-count or retained-byte admission limit, and monotonic storage for the root. Provider exhaustion is an accepted loud operational risk. RC.1's missing durable attempt receipt and complete output envelope remain documented but do not block a contract that promises neither limit. **Managed reclamation** retains the Lance-owned durable inspect/plan/execute design as an optional later optimization. OmniGraph never deletes raw `_mem_wal` paths. | +| **Graph-manifest lifetime** | Retain-all also places no finite-lifetime bound on base/token or shared `__manifest` history. `GraphHistoryBudget` is not part of the first profile. If a later product promises a hard whole-root bound, it must earn a separate RFC and physical-growth evidence across every manifest publisher. | +| **Evidence and operations** | **B1 closure green:** surface guards, genuine v7↔v8 refusal/rebuild, the graph-level behavior/crash/race cells, and qualified cost/PK-index/RSS evidence pass. The legal high-entropy near-cap cell now acknowledges, materializes, folds, publishes exactly once, and retains every listed path. The full-fold RSS delta measured 284,934,144 bytes (~272 MiB); the 384-MiB check is a CI remeasurement tripwire for one exclusive fold, not a runtime allocator limit. The revision-pinned source audit and current-object sweeps remain useful descriptions of monotonic storage, not a rejected-gate disposition. Two guards continue to prove why generic cleanup and raw fence deletion are unsafe. Public metrics/status, API tests, CLI parity, and operator evidence remain absent by design. | The table intentionally omits “small/medium” estimates. Atomic rejection, -quiescence, fresh-read retention, and GC are correctness protocols; size them -only after their implementation spikes expose the real work. +quiescence, fresh-read retention, and any future GC are correctness protocols; +size them only after their implementation spikes expose the real work. ### The RC.1 enrollment gap, stated precisely @@ -472,13 +487,13 @@ covering both: objects. An upstream exact receipt and public admission lifecycle would simplify this -substantially. We should propose that change, but its review and release timing -is not a prerequisite for the bounded-profile decision. Gate E0 passed; the +substantially. We may propose that change, but its review and release timing +is not a prerequisite for retain-all. Gate E0 passed; the upstream shape still gates overlapping-process topology. Bounded Phase A and -the private, evidence-green B1 core are now implemented; B2's product and -operational gates remain before public row activation. +the private B1 core are implemented, and the Gate R0 closure gap is repaired. +The common product and operational gates remain before public row activation. -### The RC.1 reclamation gap, stated precisely +### The RC.1 reclamation gap, stated precisely (B2b) RC.1 exposes evidence-level raw listing plus manifest reads, but not a complete Lance-owned classified inventory. Its generic @@ -490,38 +505,19 @@ demonstrates that the stale writer can still complete its WAL PUT and receive watcher success even though an explicit check sees `PeerClaimedEpoch`. Deleting the sentinel weakened the fence. -The no-wait path is to author the missing capability in Lance ourselves: an -opaque inspect/plan/execute contract with exact shard/base/history witnesses, -durable attempt/receipt recovery, whole-cut cursor proof, manifest-version and -writer-epoch advancement before deletion, bounded history checkpointing, -conservative orphan classification, an enforceable source-derived growth -reservation, and a post-success WAL epoch check. Exact inventory requires -strong HEAD/GET/LIST visibility after PUT/DELETE plus incomplete-multipart -accounting/abort, or Lance-owned durable accounting. Every epoch claim is -sentinel-first and manifest-named; ordinary open/quiesce/resume/checkpoint -claims preserve the prior replay cursor, while only a proved no-data-tail -whole-cut reclaim advances it. Enrollment creates a genesis bootstrap before a -new MemWAL details kind that stock RC.1 rejects. Checkpointing bounds both -ordinary-claim and reclaim receipts, and admission reserves durable -materialization attempts plus control headroom. One bootstrap-selected reserve- -first ledger serializes the physical budget per binding before WAL/upload -effects and settles only from exact inventory. Versioned/soft-delete/Object- -Lock stores are refused unless every retained byte is countable and eligible -versions are permanently removable. We open that patch upstream but pin -OmniGraph to the exact reviewed fork commit immediately. This respects the -substrate boundary without making an upstream merge or release a calendar -dependency. - -That is a MemWAL bound, not a whole-root retention claim. Base-table, token- -table, and shared manifest histories still grow while in-place maintenance is -refused. B2-0 therefore adds a separate manifest-authoritative, graph-global -`GraphHistoryBudget`. Every manifest writer reserves its exact publication and -source-bounded physical-growth envelope before effect. Dynamic per-stream -closure reserves cannot be consumed by ordinary work or by another stream, so -the shared floor refuses new ordinary work with `GraphRebuildRequired` while -preserving enough operations to fold, correct, quiesce, and rebuild each stream -from `SEALED`. Truly indefinite in-place streaming remains a later maintenance -design. +Managed reclamation remains a coherent future path: author a Lance-owned opaque +inspect/plan/execute contract with exact shard/base/history witnesses, durable +attempt/receipt recovery, whole-cut cursor proof, writer fencing before +deletion, conservative orphan classification, and backend-appropriate +inventory/accounting. That is deliberately deferred. The retain-all profile +does not need to prove those deletion or storage-bound contracts because it +performs no deletion and advertises no hard retained-storage limit. + +The same applies to whole-root history. Base-table, token-table, and shared +manifest histories may grow. The first profile observes that growth but does +not pretend to cap it. A future `GraphHistoryBudget` or other finite-lifetime +promise is a separate format and evidence decision, not hidden prerequisite +work for retain-all. ### Gate E0: a no-wait decision, not activation @@ -618,11 +614,13 @@ acknowledgements, so Phase A separately proves initialization, shard claim, fixed binding publication, admission exclusion, restart, and foreign-state refusal. Private B1 now implements first put, durability/lost-response semantics, replay/fold-only reopening, and strict folding. Its genuine -cross-version/rebuild, the 24-cell graph-level B1 suite, local cost/RSS, and -configured-RustFS cost gates are green. B2-0 now specifies the separate token, -attribution, lifecycle/correction, and reclamation contracts. Their -implementation and evidence gates must close before B2 may expose a single row -acknowledgement. +cross-version/rebuild, the graph-level B1 suite, and qualified cost/RSS cells +remain green. Gate R0's legal high-entropy near-cap failure is now repaired and +the closure cell publishes successfully. The missing stock-RC.1 attempt receipt +and complete-output envelope remain factual limits on any future bounded- +storage claim, but do not block the selected unbounded retain-all profile. The +common token, attribution, lifecycle/correction, authorization, and product +contracts must still close before any public row acknowledgement. ### Bottom line @@ -640,38 +638,36 @@ The plan is: exclusion, narrow `SEALED` native-branch exception, lifecycle state, admission lease, and refusal/rebuild gates as the foundation preserved by current v8; -3. retain the implemented private B1 root-scoped, single-generation worker: - prevent rollover, hard-cap the complete generation, retire/reopen between - generations, hold admission from before epoch claim, preserve ambiguous - attempts through replay, route non-empty replay fold-only through the pinned - BatchStore watermark bridge, prove drain from refs plus authoritative - manifest state, and fold through one recovery-v11 keyed transaction and one - exact manifest publication; keep its now-green cross-version, crash, and - cost/RSS instruments as regression gates; -4. retain B2-0's specified compare-and-chain token, trusted attribution, +3. retain the repaired logical-slice/dense fold and its local/configured-RustFS + near-cap closure plus full-fold RSS remeasurement instruments as regression + gates; do not silently shrink the 8,192-row/32-MiB admission cap; +4. implement the selected unbounded retain-all profile: delete no MemWAL path, + add no file/byte admission limit, expose observed growth only as advisory, + and surface provider-capacity failure loudly through recovery; +5. retain the common B2 compare-and-chain token, trusted attribution, manifest-selected token participant, protocol-v2 lifecycle, and bounded `REPLACE`/`WITHDRAW` correction as the implementation contract; -5. author and review the Lance-owned reclamation primitive and post-success - fence check, durable attempt/receipt and bounded-history machinery, pin the - exact fork commit without waiting for release, and pass its local/RustFS - crash matrix plus source-derived/enforced growth reservation and validating - physical retained-storage instrument; -6. implement schema v9/config-v3/state-v2/recovery-v12 privately and keep every - product surface absent until the token, lifecycle, correction, - enforced-watermark, cross-version, and crash matrices are green; -7. add schema/SDK/HTTP/CLI/Cedar/OpenAPI surfaces only after that private core +6. keep B2b Lance-owned reclamation and a whole-root finite-lifetime budget as + optional future profiles, activated only by a new measured need and their + own RFC/evidence; do not block retain-all on either; +7. implement schema v9/config-v3/state-v2/recovery-v12 privately for the token, + attribution, lifecycle, and correction contracts, and keep every product + surface absent until those cross-version and crash matrices are green; +8. add schema/SDK/HTTP/CLI/Cedar/OpenAPI surfaces only after that private core passes; then activate Phase B2; -8. design and measure any later group-commit policy before claiming a +9. design and measure any later group-commit policy before claiming a performance multiplier; -9. pursue upstream replay-watermark and drain-error fixes plus the exact +10. pursue upstream replay-watermark and drain-error fixes plus the exact receipt/seal API in parallel; remove the RC.1 bridge when the first pair lands, and use the latter to simplify enrollment and broaden topology. We tested the narrower support contract instead of waiting on the upstream calendar. Phase A's v7 foundation and private B1's current v8/config-v2/v11 -worker/fold core are implemented and evidence-green. RFC-026 remains Draft: -private tests can put, receive watcher-only durability acknowledgement, -replay/fold-only, and publish one fold, and the B2-0 design plus two negative -RC.1 reclamation guards are checked in. No schema-v9 token/lifecycle/ -reclamation implementation, production enrollment, acknowledgement, fold, -operator, or public stream path exists. +worker/fold core are implemented, and the widest admitted high-entropy shape +now closes. RFC-026 remains Draft because the public token, attribution, +lifecycle/correction, authorization, and product surfaces are not implemented. +The selected first storage posture is unbounded retain-all: no OmniGraph +MemWAL GC and no file/byte limit. Two negative RC.1 reclamation guards remain +checked in as rationale for that no-deletion rule. No schema-v9 contract, +production enrollment, acknowledgement, fold, operator, or public stream path +exists. diff --git a/crates/omnigraph/src/db/omnigraph.rs b/crates/omnigraph/src/db/omnigraph.rs index 17b24e48..72d8f1d1 100644 --- a/crates/omnigraph/src/db/omnigraph.rs +++ b/crates/omnigraph/src/db/omnigraph.rs @@ -316,11 +316,13 @@ pub struct InitOptions { /// handles from silently selecting different resource contracts. fn private_b1_worker_limits() -> crate::table_store::mem_wal::B1WorkerLimits { crate::table_store::mem_wal::B1WorkerLimits { - // The B1 evidence qualifies one resident writer. Its widest legal - // generation added roughly 126 MiB of whole-process peak RSS (Arrow + - // mandatory PK/runtime/allocator), so admitting four residents here - // would be an unmeasured throughput choice. B2 may raise this only with - // a multi-resident RSS cell. + // The B1 evidence qualifies one resident writer and one exclusive + // fold. The widest legal admission added roughly 126 MiB of + // whole-process peak RSS; its full high-entropy fold added roughly + // 272 MiB over a small-fold baseline and carries a 384-MiB + // remeasurement tripwire. Admitting more residents would therefore be + // an unmeasured throughput choice. A later phase may raise this only + // with a multi-resident RSS cell. max_resident_writers_root: 1, max_resident_writers_per_table: 1, max_reserved_arrow_bytes: 32 * 1024 * 1024, diff --git a/crates/omnigraph/src/db/omnigraph/stream_ingest.rs b/crates/omnigraph/src/db/omnigraph/stream_ingest.rs index 8cef4592..fa5c9a45 100644 --- a/crates/omnigraph/src/db/omnigraph/stream_ingest.rs +++ b/crates/omnigraph/src/db/omnigraph/stream_ingest.rs @@ -9,8 +9,9 @@ use std::sync::Arc; -use arrow_array::{Array, RecordBatch}; +use arrow_array::{Array, RecordBatch, UInt32Array}; use arrow_schema::Schema as ArrowSchema; +use arrow_select::take::take; use futures::TryStreamExt; use lance::dataset::mem_wal::scanner::LsmScanner; use lance::dataset::mem_wal::{DatasetMemWalExt, ShardWriter}; @@ -1032,14 +1033,73 @@ async fn scan_fresh_generation( .await .map_err(|error| OmniError::Lance(error.to_string()))?; let mut batches = Vec::new(); + let mut rows = 0_u64; + let mut logical_bytes = 0_u64; while let Some(batch) = stream .try_next() .await .map_err(|error| OmniError::Lance(error.to_string()))? { - batches.push(batch); + if batch.num_rows() == 0 { + continue; + } + rows = rows + .checked_add( + u64::try_from(batch.num_rows()).map_err(|_| { + OmniError::manifest_internal("stream fold row count exceeds u64") + })?, + ) + .ok_or_else(|| OmniError::manifest_internal("stream fold row count overflow"))?; + if rows > B1_MAX_GENERATION_ROWS { + return Err(OmniError::resource_limit( + format!("stream fold rows for {}", capture.entry.table_key), + B1_MAX_GENERATION_ROWS, + rows, + )); + } + for column in batch.columns() { + let bytes = column + .to_data() + .get_slice_memory_size() + .map_err(|error| OmniError::Lance(error.to_string()))?; + logical_bytes = logical_bytes + .checked_add(u64::try_from(bytes).map_err(|_| { + OmniError::manifest_internal("stream fold logical bytes exceed u64") + })?) + .ok_or_else(|| { + OmniError::manifest_internal("stream fold logical byte count overflow") + })?; + } + if logical_bytes > B1_MAX_GENERATION_ARROW_BYTES { + return Err(OmniError::resource_limit( + format!("stream fold bytes for {}", capture.entry.table_key), + B1_MAX_GENERATION_ARROW_BYTES, + logical_bytes, + )); + } + + // LsmScanner may emit slices whose Utf8 and other variable-width + // arrays retain sparse backing buffers much larger than the selected + // rows. A no-op concat preserves those buffers. Taking every selected + // row rebuilds dense owned arrays, after which dropping `batch` + // releases the scanner representation before the next slice arrives. + let row_count = u32::try_from(batch.num_rows()) + .map_err(|_| OmniError::manifest_internal("stream fold batch row count exceeds u32"))?; + let indices = UInt32Array::from_iter_values(0..row_count); + let columns = batch + .columns() + .iter() + .map(|column| { + take(column.as_ref(), &indices, None) + .map_err(|error| OmniError::Lance(error.to_string())) + }) + .collect::>>()?; + batches.push( + RecordBatch::try_new(batch.schema(), columns) + .map_err(|error| OmniError::Lance(error.to_string()))?, + ); } - if batches.iter().all(|batch| batch.num_rows() == 0) { + if rows == 0 { return Err(OmniError::manifest_internal( "stream fold fresh-only scan returned no live rows", )); diff --git a/crates/omnigraph/tests/memwal_stream_cost.rs b/crates/omnigraph/tests/memwal_stream_cost.rs index a5111745..78d6a3e0 100644 --- a/crates/omnigraph/tests/memwal_stream_cost.rs +++ b/crates/omnigraph/tests/memwal_stream_cost.rs @@ -10,6 +10,7 @@ mod helpers; +use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use std::sync::Arc; use std::time::Instant; @@ -17,10 +18,13 @@ use std::time::Instant; use arrow_array::{ArrayRef, BooleanArray, Int32Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; use fail::FailScenario; -use lance::dataset::mem_wal::schema_with_tombstone; +use futures::TryStreamExt; use lance::dataset::mem_wal::write::StoredBatch; +use lance::dataset::mem_wal::{ShardManifestStore, schema_with_tombstone}; use lance_core::utils::bloomfilter::sbbf::Sbbf; +use lance_index::mem_wal::{FlushedGeneration, ShardId, ShardManifest, ShardStatus}; use lance_io::utils::tracking_store::{IOTracker, IoStats}; +use object_store::path::Path as ObjectPath; use omnigraph::db::{Omnigraph, ReadTarget}; use omnigraph::error::OmniError; use omnigraph::failpoints::{ScopedFailPoint, names}; @@ -45,11 +49,370 @@ query add_history($tick: I32) { const WIDEST_ROWS: usize = 8_192; const WIDEST_PAYLOAD_BYTES: usize = 4_032; const HARD_ARROW_BYTES: usize = 32 * 1024 * 1024; +const FOLD_RSS_DELTA_REMEASURE_BYTES: u64 = 384 * 1024 * 1024; const NO_AUTO_ROLL_BYTES: usize = 1024 * 1024 * 1024; const NO_AUTO_ROLL_ROWS: usize = 8_193; const NO_AUTO_ROLL_BATCHES: usize = 8_193; const RSS_CHILD_ENV: &str = "OMNIGRAPH_MEMWAL_COST_CHILD"; +const SURVEYED_LANCE_REV: &str = "cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum MemWalObjectKind { + Wal, + ShardManifestVersion, + ShardManifestHint, + GenerationData, + GenerationManifest, + GenerationTransaction, + GenerationDeletion, + GenerationPkSidecar, + GenerationBloom, + GenerationUserIndex, + Unknown, +} + +#[derive(Debug, Clone, Default)] +struct CurrentMemWalInventory { + objects: BTreeMap, + generation_roots: BTreeSet, + referenced_generation_roots: BTreeSet, +} + +impl CurrentMemWalInventory { + fn object_count(&self, kind: MemWalObjectKind) -> usize { + self.objects + .values() + .filter(|(candidate, _)| *candidate == kind) + .count() + } + + fn bytes(&self, kind: MemWalObjectKind) -> u64 { + self.objects + .values() + .filter(|(candidate, _)| *candidate == kind) + .map(|(_, bytes)| *bytes) + .sum() + } + + fn immutable_object_bytes(&self) -> u64 { + self.objects + .values() + .filter(|(kind, _)| *kind != MemWalObjectKind::ShardManifestHint) + .map(|(_, bytes)| *bytes) + .sum() + } + + fn unknown_paths(&self) -> Vec<&str> { + self.objects + .iter() + .filter_map(|(path, (kind, _))| { + (*kind == MemWalObjectKind::Unknown).then_some(path.as_str()) + }) + .collect() + } + + fn generation_subtree_objects(&self) -> BTreeMap { + self.objects + .iter() + .filter(|(_, (kind, _))| { + matches!( + kind, + MemWalObjectKind::GenerationData + | MemWalObjectKind::GenerationManifest + | MemWalObjectKind::GenerationTransaction + | MemWalObjectKind::GenerationDeletion + | MemWalObjectKind::GenerationPkSidecar + | MemWalObjectKind::GenerationBloom + | MemWalObjectKind::GenerationUserIndex + ) + }) + .map(|(path, value)| (path.clone(), *value)) + .collect() + } + + fn assert_path_class_size_retained_by(&self, later: &Self) { + for (path, (kind, bytes)) in &self.objects { + if *kind == MemWalObjectKind::ShardManifestHint { + continue; + } + assert_eq!( + later.objects.get(path), + Some(&(*kind, *bytes)), + "retain-all evidence lost or changed the class/size of listed object {path}" + ); + } + } +} + +fn is_generation_root(component: &str) -> bool { + let Some((hash, generation)) = component.split_once("_gen_") else { + return false; + }; + let Ok(generation_number) = generation.parse::() else { + return false; + }; + hash.len() == 8 + && hash + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + && generation_number > 0 + && generation_number.to_string() == generation +} + +fn parse_positive_bit_reversed_filename(filename: &str, extension: &str) -> Option { + let Some(stem) = filename.strip_suffix(&format!(".{extension}")) else { + return None; + }; + if stem.len() != 64 || !stem.bytes().all(|byte| matches!(byte, b'0' | b'1')) { + return None; + } + let reversed = u64::from_str_radix(stem, 2).ok()?; + let value = reversed.reverse_bits(); + (value > 0).then_some(value) +} + +fn parse_canonical_shard_id(value: &str) -> Option { + let shard_id = ShardId::parse_str(value).ok()?; + (shard_id.as_hyphenated().to_string() == value).then_some(shard_id) +} + +fn join_object_path(base: &str, suffix: &str) -> String { + if base.is_empty() { + suffix.to_string() + } else { + format!("{base}/{suffix}") + } +} + +fn classify_mem_wal_object( + components: &[&str], + mem_wal_index: usize, +) -> (MemWalObjectKind, Option) { + let tail = &components[mem_wal_index + 1..]; + if tail.len() < 3 || parse_canonical_shard_id(tail[0]).is_none() { + return (MemWalObjectKind::Unknown, None); + } + + match tail[1] { + "wal" + if tail.len() == 3 + && parse_positive_bit_reversed_filename(tail[2], "arrow").is_some() => + { + (MemWalObjectKind::Wal, None) + } + "manifest" if tail.len() == 3 && tail[2] == "version_hint.json" => { + (MemWalObjectKind::ShardManifestHint, None) + } + "manifest" + if tail.len() == 3 + && parse_positive_bit_reversed_filename(tail[2], "binpb").is_some() => + { + (MemWalObjectKind::ShardManifestVersion, None) + } + generation if is_generation_root(generation) => { + let root = components[..mem_wal_index + 3].join("/"); + let kind = match tail.get(2).copied() { + Some("bloom_filter.bin") if tail.len() == 3 => MemWalObjectKind::GenerationBloom, + Some("data") if tail.len() >= 4 => MemWalObjectKind::GenerationData, + Some("_versions") if tail.len() >= 4 => MemWalObjectKind::GenerationManifest, + Some("_transactions") if tail.len() >= 4 => MemWalObjectKind::GenerationTransaction, + Some("_deletions") if tail.len() >= 4 => MemWalObjectKind::GenerationDeletion, + Some("_pk_index") if tail.len() >= 4 => MemWalObjectKind::GenerationPkSidecar, + Some("_indices") if tail.len() >= 4 => MemWalObjectKind::GenerationUserIndex, + _ => MemWalObjectKind::Unknown, + }; + (kind, Some(root)) + } + _ => (MemWalObjectKind::Unknown, None), + } +} + +fn validated_referenced_generation_roots( + table_base: &str, + directory_shard_id: ShardId, + manifest: &ShardManifest, + listed_generation_roots: &BTreeSet, + listed_manifest_versions: &BTreeSet, +) -> BTreeSet { + assert_eq!( + manifest.shard_id, directory_shard_id, + "Gate R0 must fail closed when decoded shard authority disagrees with its directory" + ); + assert!( + manifest.version > 0, + "Gate R0 must fail closed on a zero shard-manifest version" + ); + let latest_listed_version = listed_manifest_versions + .last() + .copied() + .expect("Gate R0 must fail closed when a shard has no listed manifest version"); + assert_eq!( + latest_listed_version, manifest.version, + "Gate R0 must fail closed when the latest shard-manifest filename disagrees with its body" + ); + let expected_manifest_version_count = usize::try_from(manifest.version) + .expect("Gate R0 must fail closed when shard-manifest authority exceeds usize"); + assert_eq!( + listed_manifest_versions.len(), + expected_manifest_version_count, + "Gate R0 must fail closed on a gapped or noncanonical shard-manifest version chain" + ); + assert!( + manifest.current_generation > 0, + "Gate R0 must fail closed on zero next-generation allocation authority" + ); + + let shard_prefix = join_object_path( + table_base, + &format!("_mem_wal/{}", directory_shard_id.as_hyphenated()), + ); + let mut generations = BTreeSet::new(); + let mut paths = BTreeSet::new(); + let mut roots = BTreeSet::new(); + for flushed in &manifest.flushed_generations { + assert!( + flushed.generation > 0 && generations.insert(flushed.generation), + "Gate R0 must fail closed on a zero or duplicate flushed generation" + ); + assert!( + paths.insert(flushed.path.clone()), + "Gate R0 must fail closed on a duplicate flushed-generation path" + ); + assert!( + is_generation_root(&flushed.path), + "Gate R0 must fail closed on a noncanonical flushed-generation path" + ); + let (_, path_generation) = flushed + .path + .split_once("_gen_") + .expect("validated generation root must contain _gen_"); + assert_eq!( + path_generation.parse::().unwrap(), + flushed.generation, + "Gate R0 must fail closed when generation authority disagrees with its path" + ); + assert!( + manifest.current_generation > flushed.generation, + "Gate R0 must fail closed when next-generation allocation authority has not advanced past a flushed generation" + ); + let root = join_object_path(&shard_prefix, &flushed.path); + assert!( + listed_generation_roots.contains(&root), + "Gate R0 must fail closed when shard authority references a missing listed generation" + ); + assert!( + roots.insert(root), + "Gate R0 must fail closed on duplicate generation authority" + ); + } + roots +} + +/// Strict current-object inventory for the part of retained storage that stock +/// Lance exposes through ordinary LIST. This deliberately does not pretend to +/// see incomplete multipart uploads, superseded provider versions, delete +/// markers, local staged temp files, or billed storage. +async fn current_mem_wal_inventory(uri: &str) -> CurrentMemWalInventory { + let (store, root_path) = lance_io::object_store::ObjectStore::from_uri(uri) + .await + .expect("Gate R0 fixture URI must resolve"); + let listed = store + .inner + .list(Some(&root_path)) + .try_collect::>() + .await + .expect("Gate R0 inventory must fail closed on a listing error"); + + let mut inventory = CurrentMemWalInventory::default(); + let mut shards = BTreeSet::new(); + let mut manifest_versions = BTreeMap::<(String, ShardId), BTreeSet>::new(); + for metadata in listed { + let path = metadata.location.as_ref().to_string(); + let components = path.split('/').collect::>(); + let Some(mem_wal_index) = components + .iter() + .position(|component| *component == "_mem_wal") + else { + continue; + }; + let (kind, generation_root) = classify_mem_wal_object(&components, mem_wal_index); + if let Some(root) = generation_root { + inventory.generation_roots.insert(root); + } + let bytes = u64::try_from(metadata.size).expect("object size must fit u64"); + assert!( + inventory + .objects + .insert(path.clone(), (kind, bytes)) + .is_none(), + "object listing returned duplicate path {path}" + ); + + let tail = &components[mem_wal_index + 1..]; + if let Some(shard) = tail + .first() + .and_then(|value| parse_canonical_shard_id(value)) + { + let table_base = components[..mem_wal_index].join("/"); + shards.insert((table_base.clone(), shard)); + if kind == MemWalObjectKind::ShardManifestVersion { + let version = parse_positive_bit_reversed_filename(tail[2], "binpb") + .expect("classified shard-manifest version must decode"); + assert!( + manifest_versions + .entry((table_base, shard)) + .or_default() + .insert(version), + "object listing returned duplicate decoded shard-manifest version" + ); + } + } + } + + for (table_base, shard_id) in shards { + let table_path = + ObjectPath::parse(&table_base).expect("listed table base must remain a valid path"); + let manifest_store = ShardManifestStore::new(Arc::clone(&store), &table_path, shard_id, 2); + let manifest = manifest_store + .read_latest() + .await + .expect("Gate R0 must fail closed on an unreadable shard manifest") + .expect("Gate R0 must fail closed when a listed shard has no latest manifest"); + let listed_manifest_versions = manifest_versions + .get(&(table_base.clone(), shard_id)) + .expect("Gate R0 must fail closed when a shard has no listed manifest version"); + inventory + .referenced_generation_roots + .extend(validated_referenced_generation_roots( + &table_base, + shard_id, + &manifest, + &inventory.generation_roots, + listed_manifest_versions, + )); + } + + inventory +} + +async fn current_recovery_sidecars(uri: &str) -> Vec { + let (store, root_path) = lance_io::object_store::ObjectStore::from_uri(uri) + .await + .expect("Gate R0 fixture URI must resolve"); + let mut paths = store + .inner + .list(Some(&root_path)) + .try_collect::>() + .await + .expect("Gate R0 recovery inventory must fail closed on a listing error") + .into_iter() + .map(|metadata| metadata.location.as_ref().to_string()) + .filter(|path| path.split('/').any(|component| component == "__recovery")) + .collect::>(); + paths.sort(); + paths +} #[derive(Debug, Clone, Copy, Default)] struct PathIo { @@ -212,6 +575,25 @@ struct RetainedMetadataSample { serialized_manifest_bytes: u64, cold_elapsed_us: u128, table: PathIo, + inventory: CurrentMemWalInventory, +} + +#[derive(Debug)] +struct WidestRetainedGrowthSample { + before: CurrentMemWalInventory, + after_ack: CurrentMemWalInventory, + after_fold: CurrentMemWalInventory, + table_ack: PathIo, + table_fold: PathIo, + manifest_version_before: u64, + manifest_version_after_ack: u64, + manifest_version_after_fold: u64, + table_version_before: u64, + table_version_after_ack: u64, + table_version_after_fold: u64, + visible_rows: usize, + sampled_payloads: Vec<(String, String)>, + recovery_sidecars_after_fold: Vec, } async fn initialize_history(uri: &str, depth: u64, compact_before_enrollment: bool) { @@ -689,7 +1071,7 @@ fn generation_data_file_bytes(root: &Path) -> u64 { #[tokio::test] #[serial] -async fn retained_merged_generation_metadata_cost_is_swept_explicitly() { +async fn gate_r0_retain_all_current_object_growth_is_swept_explicitly() { let _scenario = FailScenario::setup(); let dir = tempfile::tempdir().unwrap(); let uri = dir.path().to_str().unwrap(); @@ -722,6 +1104,7 @@ async fn retained_merged_generation_metadata_cost_is_swept_explicitly() { } let serialized_manifest_bytes = max_shard_manifest_bytes(dir.path()); + let inventory = current_mem_wal_inventory(uri).await; let pending = score_batch( Arc::clone(&schema), &format!("metadata-measured-{checkpoint}"), @@ -758,6 +1141,7 @@ async fn retained_merged_generation_metadata_cost_is_swept_explicitly() { serialized_manifest_bytes, cold_elapsed_us: started.elapsed().as_micros(), table: PathIo::from_stats(&table_tracker.incremental_stats()), + inventory, }); let _ = manifest_tracker.incremental_stats(); @@ -770,22 +1154,669 @@ async fn retained_merged_generation_metadata_cost_is_swept_explicitly() { }) .await; - for sample in &samples { + for (index, sample) in samples.iter().enumerate() { eprintln!( - "B1 retained merged_generations={}: largest_retained_manifest_bytes={} cold_elapsed_us={} shard_manifest_reads={} aggregate_table_read_bytes={}", + "B1 retained merged_generations={}: largest_retained_manifest_bytes={} current_listed_immutable_bytes={} current_listed_objects={} generation_roots={} manifest_version_objects={} cold_elapsed_us={} shard_manifest_reads={} aggregate_table_read_bytes={}", sample.merged_generations, sample.serialized_manifest_bytes, + sample.inventory.immutable_object_bytes(), + sample.inventory.objects.len(), + sample.inventory.generation_roots.len(), + sample + .inventory + .object_count(MemWalObjectKind::ShardManifestVersion), sample.cold_elapsed_us, sample.table.shard_manifest_reads, sample.table.total_read_bytes, ); assert!(sample.table.shard_manifest_reads > 0, "{sample:?}"); + assert_eq!( + sample.inventory.generation_roots.len(), + usize::try_from(sample.merged_generations).unwrap(), + "one no-roll B1 fold must retain one generation root" + ); + assert_eq!( + sample.inventory.generation_roots, sample.inventory.referenced_generation_roots, + "the success-only sweep must not confuse referenced generations with crash orphans" + ); + assert!( + sample.inventory.unknown_paths().is_empty(), + "current-object census missed paths: {:?}", + sample.inventory.unknown_paths() + ); + if index > 0 { + samples[index - 1] + .inventory + .assert_path_class_size_retained_by(&sample.inventory); + } } assert!( samples.last().unwrap().serialized_manifest_bytes > samples.first().unwrap().serialized_manifest_bytes, "B1 intentionally retains merged generation entries; the serialized shard metadata must expose that growth: {samples:#?}" ); + assert!( + samples.last().unwrap().inventory.immutable_object_bytes() + > samples.first().unwrap().inventory.immutable_object_bytes(), + "retain-all physical objects must expose cumulative growth: {samples:#?}" + ); + assert!( + samples + .last() + .unwrap() + .inventory + .object_count(MemWalObjectKind::ShardManifestVersion) + > samples + .first() + .unwrap() + .inventory + .object_count(MemWalObjectKind::ShardManifestVersion), + "versioned shard authority is retained growth too: {samples:#?}" + ); +} + +fn classify_test_path(path: &str) -> MemWalObjectKind { + let components = path.split('/').collect::>(); + let mem_wal_index = components + .iter() + .position(|component| *component == "_mem_wal") + .expect("test path must contain _mem_wal"); + classify_mem_wal_object(&components, mem_wal_index).0 +} + +fn bit_reversed_stem(value: u64) -> String { + format!("{:064b}", value.reverse_bits()) +} + +fn gate_r0_manifest( + shard_id: ShardId, + flushed_generations: Vec, +) -> ShardManifest { + ShardManifest { + shard_id, + version: 1, + shard_spec_id: 0, + shard_field_values: Default::default(), + writer_epoch: 1, + replay_after_wal_entry_position: 0, + wal_entry_position_last_seen: 0, + current_generation: flushed_generations + .iter() + .map(|generation| generation.generation) + .max() + .unwrap_or(0) + .checked_add(1) + .expect("Gate R0 fixture generation must leave successor authority"), + flushed_generations, + status: ShardStatus::Active, + } +} + +#[test] +fn gate_r0_current_inventory_path_classifier_is_canonical_and_fail_closed() { + let shard = "abcdefab-cdef-abcd-efab-cdefabcdefab"; + let bits = bit_reversed_stem(1); + assert_eq!( + classify_test_path(&format!("table/_mem_wal/{shard}/wal/{bits}.arrow")), + MemWalObjectKind::Wal + ); + assert_eq!( + classify_test_path(&format!("table/_mem_wal/{shard}/manifest/{bits}.binpb")), + MemWalObjectKind::ShardManifestVersion + ); + assert_eq!( + classify_test_path(&format!( + "table/_mem_wal/{shard}/abcdef12_gen_1/data/file.lance" + )), + MemWalObjectKind::GenerationData + ); + + let malformed = [ + format!("table/_mem_wal/{shard}/wal/{}.arrow", "0".repeat(63)), + format!("table/_mem_wal/{shard}/wal/{}.arrow", bit_reversed_stem(0)), + format!("table/_mem_wal/{shard}/wal/{bits}.arrow.extra"), + format!("table/_mem_wal/{shard}/manifest/not-a-version.binpb"), + format!( + "table/_mem_wal/{}/wal/{bits}.arrow", + shard.to_ascii_uppercase() + ), + format!("table/_mem_wal/{shard}/ABCDEF12_gen_1/data/file.lance"), + format!("table/_mem_wal/{shard}/abcdef12_gen_0/data/file.lance"), + format!("table/_mem_wal/{shard}/abcdef12_gen_01/data/file.lance"), + ]; + for path in malformed { + assert_eq!( + classify_test_path(&path), + MemWalObjectKind::Unknown, + "malformed path must fail closed: {path}" + ); + } +} + +#[test] +#[should_panic(expected = "decoded shard authority disagrees with its directory")] +fn gate_r0_current_inventory_refuses_manifest_for_another_shard() { + let directory_shard = ShardId::parse_str("abcdefab-cdef-abcd-efab-cdefabcdefab").unwrap(); + let foreign_shard = ShardId::parse_str("01234567-89ab-cdef-0123-456789abcdef").unwrap(); + let manifest = gate_r0_manifest(foreign_shard, Vec::new()); + let _ = validated_referenced_generation_roots( + "table", + directory_shard, + &manifest, + &BTreeSet::new(), + &BTreeSet::from([1]), + ); +} + +#[test] +#[should_panic(expected = "zero or duplicate flushed generation")] +fn gate_r0_current_inventory_refuses_duplicate_generation_authority() { + let shard = ShardId::parse_str("abcdefab-cdef-abcd-efab-cdefabcdefab").unwrap(); + let generation = FlushedGeneration { + generation: 1, + path: "abcdef12_gen_1".to_string(), + }; + let manifest = gate_r0_manifest(shard, vec![generation.clone(), generation]); + let listed = BTreeSet::from([format!( + "table/_mem_wal/{}/abcdef12_gen_1", + shard.as_hyphenated() + )]); + let _ = validated_referenced_generation_roots( + "table", + shard, + &manifest, + &listed, + &BTreeSet::from([1]), + ); +} + +#[test] +#[should_panic(expected = "zero next-generation allocation authority")] +fn gate_r0_current_inventory_refuses_zero_current_generation() { + let shard = ShardId::parse_str("abcdefab-cdef-abcd-efab-cdefabcdefab").unwrap(); + let mut manifest = gate_r0_manifest(shard, Vec::new()); + manifest.current_generation = 0; + let _ = validated_referenced_generation_roots( + "table", + shard, + &manifest, + &BTreeSet::new(), + &BTreeSet::from([1]), + ); +} + +#[test] +#[should_panic(expected = "allocation authority has not advanced past a flushed generation")] +fn gate_r0_current_inventory_refuses_non_advanced_current_generation() { + let shard = ShardId::parse_str("abcdefab-cdef-abcd-efab-cdefabcdefab").unwrap(); + let generation = FlushedGeneration { + generation: 1, + path: "abcdef12_gen_1".to_string(), + }; + let mut manifest = gate_r0_manifest(shard, vec![generation]); + manifest.current_generation = 1; + let listed = BTreeSet::from([format!( + "table/_mem_wal/{}/abcdef12_gen_1", + shard.as_hyphenated() + )]); + let _ = validated_referenced_generation_roots( + "table", + shard, + &manifest, + &listed, + &BTreeSet::from([1]), + ); +} + +#[test] +#[should_panic(expected = "latest shard-manifest filename disagrees with its body")] +fn gate_r0_current_inventory_refuses_manifest_filename_body_version_mismatch() { + let shard = ShardId::parse_str("abcdefab-cdef-abcd-efab-cdefabcdefab").unwrap(); + let manifest = gate_r0_manifest(shard, Vec::new()); + let _ = validated_referenced_generation_roots( + "table", + shard, + &manifest, + &BTreeSet::new(), + &BTreeSet::from([2]), + ); +} + +#[test] +#[should_panic(expected = "gapped or noncanonical shard-manifest version chain")] +fn gate_r0_current_inventory_refuses_gapped_manifest_version_chain() { + let shard = ShardId::parse_str("abcdefab-cdef-abcd-efab-cdefabcdefab").unwrap(); + let mut manifest = gate_r0_manifest(shard, Vec::new()); + manifest.version = 3; + let _ = validated_referenced_generation_roots( + "table", + shard, + &manifest, + &BTreeSet::new(), + &BTreeSet::from([1, 3]), + ); +} + +#[tokio::test] +#[should_panic(expected = "listed shard has no latest manifest")] +async fn gate_r0_current_inventory_refuses_listed_shard_without_manifest() { + let dir = tempfile::tempdir().unwrap(); + let shard = "abcdefab-cdef-abcd-efab-cdefabcdefab"; + let wal_dir = dir.path().join("_mem_wal").join(shard).join("wal"); + std::fs::create_dir_all(&wal_dir).unwrap(); + std::fs::write( + wal_dir.join(format!("{}.arrow", bit_reversed_stem(1))), + b"x", + ) + .unwrap(); + let _ = current_mem_wal_inventory(dir.path().to_str().unwrap()).await; +} + +#[test] +fn gate_r0_source_audit_revision_tripwire_matches_surveyed_lance() { + let expected_source = format!( + "git+https://github.com/lance-format/lance?rev={SURVEYED_LANCE_REV}#{SURVEYED_LANCE_REV}" + ); + let expected_lance_repository_packages = BTreeSet::from([ + "fsst", + "lance", + "lance-arrow", + "lance-arrow-scalar", + "lance-arrow-stats", + "lance-bitpacking", + "lance-core", + "lance-datafusion", + "lance-datagen", + "lance-derive", + "lance-encoding", + "lance-file", + "lance-index", + "lance-index-core", + "lance-io", + "lance-linalg", + "lance-namespace", + "lance-namespace-impls", + "lance-select", + "lance-table", + "lance-tokenizer", + ]); + let mut observed = BTreeMap::<&str, Vec<&str>>::new(); + for package in include_str!("../../../Cargo.lock") + .split("[[package]]") + .skip(1) + { + let value = |key: &str| { + package.lines().find_map(|line| { + line.strip_prefix(&format!("{key} = \"")) + .and_then(|value| value.strip_suffix('"')) + }) + }; + let Some(name) = value("name") else { + continue; + }; + let Some(source) = value("source") else { + continue; + }; + if source.starts_with("git+https://github.com/lance-format/lance?") { + assert_eq!( + source, expected_source, + "Lance-repository package {name} moved or became mixed; Gate R0's source audit must be rerun" + ); + observed.entry(name).or_default().push(source); + } + } + assert_eq!( + observed.keys().copied().collect::>(), + expected_lance_repository_packages, + "the exact audited Lance-repository package family changed; Gate R0's source audit must be rerun" + ); + for package in &expected_lance_repository_packages { + assert_eq!( + observed.get(package).map(Vec::as_slice), + Some([expected_source.as_str()].as_slice()), + "audited Lance-repository package {package} moved, disappeared, or became mixed; Gate R0's source audit must be rerun" + ); + } + let assert_direct_manifest_pin = |manifest: &str, package: &str| { + let prefix = format!("{package} = "); + let lines = manifest + .lines() + .filter(|line| line.starts_with(&prefix)) + .collect::>(); + assert_eq!( + lines.len(), + 1, + "Gate R0 must find exactly one direct source declaration for {package}" + ); + assert!( + lines[0].contains("git = \"https://github.com/lance-format/lance\"") + && lines[0].contains(&format!("rev = \"{SURVEYED_LANCE_REV}\"")), + "direct Lance manifest pin moved without rerunning Gate R0: {}", + lines[0] + ); + }; + let workspace_manifest = include_str!("../../../Cargo.toml"); + for package in [ + "lance", + "lance-core", + "lance-select", + "lance-datafusion", + "lance-file", + "lance-index", + "lance-linalg", + "lance-namespace", + "lance-namespace-impls", + "lance-table", + ] { + assert_direct_manifest_pin(workspace_manifest, package); + } + assert_direct_manifest_pin(include_str!("../Cargo.toml"), "lance-io"); +} + +#[tokio::test] +#[serial] +async fn gate_r0_referenced_cut_retry_reuses_the_same_generation_root_local() { + let _scenario = FailScenario::setup(); + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + init_enrolled(uri, STREAM_SCHEMA).await; + let db = Arc::new(Omnigraph::open(uri).await.unwrap()); + let batch = score_batch(table_schema(&db).await, "r0-retry", 1); + db.failpoint_stream_b1_for_test(TABLE, Some(batch), 0) + .await + .unwrap(); + + let error = { + let _after_drain = + ScopedFailPoint::new(names::STREAM_FOLD_POST_DRAIN_PRE_SIDECAR, "return"); + db.failpoint_stream_b1_for_test(TABLE, None, 0) + .await + .expect_err("stop after Lance published the generation cut") + }; + assert!( + error + .to_string() + .contains(names::STREAM_FOLD_POST_DRAIN_PRE_SIDECAR), + "{error:?}" + ); + let after_failed_fold = current_mem_wal_inventory(uri).await; + assert_eq!(after_failed_fold.generation_roots.len(), 1); + assert_eq!( + after_failed_fold.generation_roots, + after_failed_fold.referenced_generation_roots + ); + assert!( + after_failed_fold.unknown_paths().is_empty(), + "{after_failed_fold:#?}" + ); + + db.failpoint_stream_b1_for_test(TABLE, None, 0) + .await + .unwrap(); + let after_retry = current_mem_wal_inventory(uri).await; + assert_eq!( + after_retry.generation_roots, after_failed_fold.generation_roots, + "a retry after the referenced cut must fold-only, not materialize another random root" + ); + assert!(after_retry.unknown_paths().is_empty(), "{after_retry:#?}"); + assert_eq!( + after_retry.generation_roots, + after_retry.referenced_generation_roots + ); + assert_eq!( + after_retry.generation_subtree_objects(), + after_failed_fold.generation_subtree_objects(), + "fold-only retry must not add, replace, or resize any generation-subtree object" + ); + after_failed_fold.assert_path_class_size_retained_by(&after_retry); + assert_eq!(helpers::count_rows(&db, TABLE).await, 1); +} + +fn deterministic_high_entropy_ascii(row: usize, len: usize) -> String { + const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + let mut state = (row as u64 + 1).wrapping_mul(0x9e37_79b9_7f4a_7c15); + let mut bytes = Vec::with_capacity(len); + for ordinal in 0..len { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state = state.wrapping_add((ordinal as u64).wrapping_mul(0xd6e8_feb8_6659_fd93)); + bytes.push(ALPHABET[(state as usize) & (ALPHABET.len() - 1)]); + } + String::from_utf8(bytes).unwrap() +} + +async fn widest_high_entropy_payload_batch(db: &Omnigraph) -> RecordBatch { + let schema = table_schema(db).await; + let ids = Arc::new(StringArray::from_iter_values( + (0..WIDEST_ROWS).map(|row| format!("rss-{row:05}")), + )) as ArrayRef; + let values = + Arc::new(StringArray::from_iter_values((0..WIDEST_ROWS).map(|row| { + deterministic_high_entropy_ascii(row, WIDEST_PAYLOAD_BYTES) + }))) as ArrayRef; + let batch = RecordBatch::try_new(schema, vec![ids, values]).unwrap(); + let mut stored_columns = batch.columns().to_vec(); + stored_columns.push(Arc::new(BooleanArray::from(vec![false; WIDEST_ROWS])) as ArrayRef); + let stored = RecordBatch::try_new( + schema_with_tombstone(batch.schema().as_ref()), + stored_columns, + ) + .unwrap(); + assert!( + stored.get_array_memory_size() > 31 * 1024 * 1024 + && stored.get_array_memory_size() <= HARD_ARROW_BYTES, + "Gate R0 fixture must remain a concrete near-cap legal batch" + ); + batch +} + +async fn widest_retained_growth_at_uri(uri: &str) -> WidestRetainedGrowthSample { + init_enrolled(uri, PAYLOAD_SCHEMA).await; + let before = current_mem_wal_inventory(uri).await; + let table_tracker = IOTracker::default(); + let manifest_tracker = IOTracker::default(); + with_raw_io_trackers(&table_tracker, &manifest_tracker, async { + let db = Arc::new(Omnigraph::open(uri).await.unwrap()); + let batch = widest_high_entropy_payload_batch(&db).await; + let before_snapshot = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); + let manifest_version_before = before_snapshot.version(); + let table_version_before = before_snapshot.entry(TABLE).unwrap().table_version; + let _ = table_tracker.incremental_stats(); + let _ = manifest_tracker.incremental_stats(); + + db.failpoint_stream_b1_for_test(TABLE, Some(batch), 0) + .await + .unwrap(); + let table_ack = PathIo::from_stats(&table_tracker.incremental_stats()); + let _ = manifest_tracker.incremental_stats(); + let after_ack = current_mem_wal_inventory(uri).await; + let after_ack_snapshot = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); + let manifest_version_after_ack = after_ack_snapshot.version(); + let table_version_after_ack = after_ack_snapshot.entry(TABLE).unwrap().table_version; + + db.failpoint_stream_b1_for_test(TABLE, None, 0) + .await + .expect("every admitted near-cap generation must close"); + let table_fold = PathIo::from_stats(&table_tracker.incremental_stats()); + let after_fold = current_mem_wal_inventory(uri).await; + let after_fold_snapshot = db.snapshot_of(ReadTarget::branch("main")).await.unwrap(); + let manifest_version_after_fold = after_fold_snapshot.version(); + let table_version_after_fold = after_fold_snapshot.entry(TABLE).unwrap().table_version; + + let wanted = BTreeSet::from([ + "rss-00000".to_string(), + format!("rss-{:05}", WIDEST_ROWS / 2), + format!("rss-{:05}", WIDEST_ROWS - 1), + ]); + let visible = helpers::read_table(&db, TABLE).await; + let visible_rows = visible.iter().map(RecordBatch::num_rows).sum(); + let mut sampled_payloads = Vec::new(); + for batch in &visible { + let ids = batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .expect("folded id must remain Utf8"); + let payloads = batch + .column_by_name("payload") + .unwrap() + .as_any() + .downcast_ref::() + .expect("folded payload must remain Utf8"); + for row in 0..batch.num_rows() { + if wanted.contains(ids.value(row)) { + sampled_payloads + .push((ids.value(row).to_string(), payloads.value(row).to_string())); + } + } + } + sampled_payloads.sort_by(|left, right| left.0.cmp(&right.0)); + + WidestRetainedGrowthSample { + before, + after_ack, + after_fold, + table_ack, + table_fold, + manifest_version_before, + manifest_version_after_ack, + manifest_version_after_fold, + table_version_before, + table_version_after_ack, + table_version_after_fold, + visible_rows, + sampled_payloads, + recovery_sidecars_after_fold: current_recovery_sidecars(uri).await, + } + }) + .await +} + +fn assert_widest_retained_growth(sample: &WidestRetainedGrowthSample, backend: &str) { + eprintln!( + "Gate R0 widest {backend}: fold=closed visible_rows={} before_immutable_bytes={} after_ack_immutable_bytes={} after_fold_immutable_bytes={} wal_bytes={} generation_data_bytes={} generation_pk_bytes={} generation_bloom_bytes={} ack_writes={} fold_generation_writes={} manifest_versions={}/{}/{} table_versions={}/{}/{}", + sample.visible_rows, + sample.before.immutable_object_bytes(), + sample.after_ack.immutable_object_bytes(), + sample.after_fold.immutable_object_bytes(), + sample.after_fold.bytes(MemWalObjectKind::Wal), + sample.after_fold.bytes(MemWalObjectKind::GenerationData), + sample + .after_fold + .bytes(MemWalObjectKind::GenerationPkSidecar), + sample.after_fold.bytes(MemWalObjectKind::GenerationBloom), + sample.table_ack.total_writes, + sample.table_fold.generation_writes, + sample.manifest_version_before, + sample.manifest_version_after_ack, + sample.manifest_version_after_fold, + sample.table_version_before, + sample.table_version_after_ack, + sample.table_version_after_fold, + ); + sample + .before + .assert_path_class_size_retained_by(&sample.after_ack); + sample + .after_ack + .assert_path_class_size_retained_by(&sample.after_fold); + assert!(sample.before.unknown_paths().is_empty(), "{sample:#?}"); + assert!(sample.after_ack.unknown_paths().is_empty(), "{sample:#?}"); + assert!(sample.after_fold.unknown_paths().is_empty(), "{sample:#?}"); + assert_eq!(sample.table_ack.generation_writes, 0, "{sample:#?}"); + assert!( + sample.after_ack.bytes(MemWalObjectKind::Wal) + >= u64::try_from(HARD_ARROW_BYTES * 9 / 10).unwrap(), + "high-entropy WAL evidence compressed below its remeasurement tripwire" + ); + assert!(sample.after_ack.generation_roots.is_empty()); + assert_eq!(sample.after_fold.generation_roots.len(), 1); + assert_eq!( + sample.after_fold.generation_roots, + sample.after_fold.referenced_generation_roots + ); + assert!( + sample.after_fold.bytes(MemWalObjectKind::GenerationData) + >= u64::try_from(HARD_ARROW_BYTES * 9 / 10).unwrap(), + "high-entropy generation evidence compressed below its remeasurement tripwire" + ); + assert!( + sample + .after_fold + .bytes(MemWalObjectKind::GenerationPkSidecar) + > 0 + ); + assert!(sample.after_fold.bytes(MemWalObjectKind::GenerationBloom) > 0); + assert!(sample.after_fold.immutable_object_bytes() > sample.after_ack.immutable_object_bytes()); + assert_eq!( + sample.manifest_version_after_ack, sample.manifest_version_before, + "durable acknowledgement must not publish the graph" + ); + assert_eq!( + sample.table_version_after_ack, sample.table_version_before, + "durable acknowledgement must not advance the base table" + ); + assert_eq!( + sample.manifest_version_after_fold, + sample.manifest_version_before + 1, + "one fold must have exactly one graph visibility point" + ); + assert_eq!( + sample.table_version_after_fold, + sample.table_version_before + 1, + "one fold must commit exactly one Lance table effect" + ); + assert_eq!(sample.visible_rows, WIDEST_ROWS); + let expected_samples = [0, WIDEST_ROWS / 2, WIDEST_ROWS - 1] + .into_iter() + .map(|row| { + ( + format!("rss-{row:05}"), + deterministic_high_entropy_ascii(row, WIDEST_PAYLOAD_BYTES), + ) + }) + .collect::>(); + assert_eq!(sample.sampled_payloads, expected_samples); + assert!( + sample.recovery_sidecars_after_fold.is_empty(), + "successful closure must retire its recovery sidecar: {:?}", + sample.recovery_sidecars_after_fold + ); +} + +#[tokio::test] +#[serial] +async fn gate_r0_widest_generation_closes_and_records_retain_all_growth_local() { + let dir = tempfile::tempdir().unwrap(); + let sample = widest_retained_growth_at_uri(dir.path().to_str().unwrap()).await; + assert_widest_retained_growth(&sample, "local"); +} + +#[tokio::test] +#[serial] +async fn gate_r0_widest_generation_closes_and_records_retain_all_growth_on_configured_rustfs() { + let Some(bucket) = std::env::var("OMNIGRAPH_S3_TEST_BUCKET").ok() else { + eprintln!("SKIP: OMNIGRAPH_S3_TEST_BUCKET not set; RustFS Gate R0 closure not measured"); + return; + }; + let prefix = std::env::var("OMNIGRAPH_S3_TEST_PREFIX") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| "omnigraph-itests".to_string()); + let uri = format!( + "s3://{bucket}/{prefix}/memwal-gate-r0/{}-{}", + std::process::id(), + ulid::Ulid::new() + ); + let sample = widest_retained_growth_at_uri(&uri).await; + storage_for_uri(&uri) + .unwrap() + .delete_prefix(&uri) + .await + .expect("configured RustFS Gate R0 cleanup failed"); + assert_widest_retained_growth(&sample, "rustfs"); } fn widest_accounting_batch(payload_bytes: usize) -> RecordBatch { @@ -960,14 +1991,25 @@ fn widest_legal_generation_records_no_roll_estimates_and_peak_rss() { { let baseline_peak_rss = run_rss_child("baseline"); let widest_peak_rss = run_rss_child("widest"); + let baseline_fold_peak_rss = run_rss_child("baseline-fold"); + let widest_fold_peak_rss = run_rss_child("widest-fold"); let signed_process_delta = i128::from(widest_peak_rss) - i128::from(baseline_peak_rss); + let fold_process_delta = widest_fold_peak_rss.saturating_sub(baseline_fold_peak_rss); eprintln!( - "B1 widest legal generation: rows={WIDEST_ROWS} one_batch_payload_bytes_per_row={WIDEST_PAYLOAD_BYTES} one_batch_post_tombstone_arrow_reservation={arrow_reservation} one_batch_batch_store_estimate={batch_store_estimate} rc_pk_bloom_estimate={bloom_estimate} one_batch_rc_roll_trigger_estimate={rc_trigger_estimate} fragmented_batches={WIDEST_ROWS} fragmented_payload_bytes_per_row={fragmented_payload_bytes} fragmented_post_tombstone_arrow_reservation={fragmented_arrow_reservation} fragmented_batch_store_estimate={fragmented_batch_store_estimate} fragmented_rc_roll_trigger_estimate={fragmented_rc_trigger_estimate} no_roll_bytes={NO_AUTO_ROLL_BYTES} rss_shape=one_batch baseline_peak_rss={baseline_peak_rss} widest_peak_rss={widest_peak_rss} signed_whole_process_delta={signed_process_delta}" + "B1 widest legal generation: rows={WIDEST_ROWS} one_batch_payload_bytes_per_row={WIDEST_PAYLOAD_BYTES} one_batch_post_tombstone_arrow_reservation={arrow_reservation} one_batch_batch_store_estimate={batch_store_estimate} rc_pk_bloom_estimate={bloom_estimate} one_batch_rc_roll_trigger_estimate={rc_trigger_estimate} fragmented_batches={WIDEST_ROWS} fragmented_payload_bytes_per_row={fragmented_payload_bytes} fragmented_post_tombstone_arrow_reservation={fragmented_arrow_reservation} fragmented_batch_store_estimate={fragmented_batch_store_estimate} fragmented_rc_roll_trigger_estimate={fragmented_rc_trigger_estimate} no_roll_bytes={NO_AUTO_ROLL_BYTES} rss_shape=one_batch baseline_peak_rss={baseline_peak_rss} widest_peak_rss={widest_peak_rss} signed_whole_process_delta={signed_process_delta} baseline_fold_peak_rss={baseline_fold_peak_rss} widest_fold_peak_rss={widest_fold_peak_rss} fold_process_delta={fold_process_delta} fold_delta_remeasure_bytes={FOLD_RSS_DELTA_REMEASURE_BYTES}" ); assert!( widest_peak_rss > baseline_peak_rss, "the isolated widest generation should have a visible whole-process RSS cost" ); + assert!( + widest_fold_peak_rss > baseline_fold_peak_rss, + "the isolated widest fold should have a visible whole-process RSS cost" + ); + assert!( + fold_process_delta <= FOLD_RSS_DELTA_REMEASURE_BYTES, + "the dense fold path crossed its measured RSS-delta envelope; remeasure before changing the admission or compaction shape" + ); } #[cfg(not(unix))] @@ -995,11 +2037,17 @@ fn widest_legal_generation_cost_child() { let uri = dir.path().to_str().unwrap(); init_enrolled(uri, PAYLOAD_SCHEMA).await; let db = Arc::new(Omnigraph::open(uri).await.unwrap()); - let batch = match mode.as_str() { - "baseline" => payload_batch(&db, 1, 1).await, - "widest" => payload_batch(&db, WIDEST_ROWS, WIDEST_PAYLOAD_BYTES).await, + let (batch, fold) = match mode.as_str() { + "baseline" => (payload_batch(&db, 1, 1).await, false), + "widest" => ( + payload_batch(&db, WIDEST_ROWS, WIDEST_PAYLOAD_BYTES).await, + false, + ), + "baseline-fold" => (payload_batch(&db, 1, 1).await, true), + "widest-fold" => (widest_high_entropy_payload_batch(&db).await, true), other => panic!("unknown RSS child mode '{other}'"), }; + let expected_rows = batch.num_rows(); let table_tracker = IOTracker::default(); let manifest_tracker = IOTracker::default(); with_raw_io_trackers(&table_tracker, &manifest_tracker, async { @@ -1010,6 +2058,18 @@ fn widest_legal_generation_cost_child() { let table = PathIo::from_stats(&table_tracker.incremental_stats()); assert!(table.wal_writes > 0, "{table:?}"); assert_eq!(table.generation_writes, 0, "{table:?}"); + if fold { + db.failpoint_stream_b1_for_test(TABLE, None, 0) + .await + .expect("RSS fold child must close its admitted generation"); + let fold_table = PathIo::from_stats(&table_tracker.incremental_stats()); + assert!(fold_table.generation_writes > 0, "{fold_table:?}"); + assert_eq!( + helpers::count_rows(&db, TABLE).await, + expected_rows, + "RSS fold child must publish every admitted row" + ); + } }) .await; }); diff --git a/docs/dev/canon.md b/docs/dev/canon.md index b138eaf2..587bb2f6 100644 --- a/docs/dev/canon.md +++ b/docs/dev/canon.md @@ -3,7 +3,7 @@ **Audience:** maintainers, contributors, and coding agents — internal **Type:** narrative reference ("the book"), read top-to-bottom **Status:** living document -**Surveyed:** OmniGraph 0.8.1 development (`main`), with the RFC-026 Phase A foundation and private Phase B1 core present and Phase B2-0 specified; Lance 9.0.0-rc.1 (git rev `cec0b7df`); internal manifest schema v8 +**Surveyed:** OmniGraph 0.8.1 development (`main`), with RFC-026 Phase A and the private Phase B1 core present, near-cap fold closure green, and unbounded retain-all selected; Lance 9.0.0-rc.1 (git rev `cec0b7df`); internal manifest schema v8 --- @@ -513,11 +513,14 @@ witness-update adapter existed. Native branch create/delete was the narrow exception: it did not advance a table HEAD, so it could proceed at `SEALED`; it still refused `OPEN` or `DRAINING`. Schema v8 preserves those exclusions and adds only private B1's exact one-generation fold as a guarded HEAD/witness -transition. RFC-026 B2-0 now specifies the next contract: compare-and-chain +transition. RFC-026's common B2 inventory specifies the next contract: +compare-and-chain write/predecessor tokens, trusted hidden row attribution, a manifest-selected current-token participant, durable `OPEN -> DRAINING -> SEALED -> OPEN`, -bounded `REPLACE`/`WITHDRAW` correction, and Lance-owned reclamation with an -enforced storage admission watermark. None is implemented. Schema +bounded `REPLACE`/`WITHDRAW` correction, and unbounded retain-all with no +OmniGraph MemWAL deletion. Managed reclamation is a deferred Lance-owned +optimization, not an activation gate. None of the common contracts is +implemented. Schema v9/config-v3/state-v2/recovery-v12 and every public row surface stay inactive until their crash, storage, and cross-version evidence passes. Phase D owns future automatic operation-scoped @@ -735,7 +738,7 @@ Phase-A foundation and adds config-v2 private one-generation admission plus recovery-v11 fold. V6→v7 and v7→v8 each require export/init/load rebuilds; both genuine old/new-binary refusal/rebuild strands are green. -B2-0 assigns the next strict strand—schema v9, stream-config v3, state protocol +The common B2 inventory assigns the next strict strand—schema v9, stream-config v3, state protocol v2, and recovery-v12—but does not activate it. V9 is where trusted stream-row metadata and the manifest-selected current-token dataset would enter the format. It must refuse/rebuild against v8 in both directions; no serde default @@ -935,9 +938,10 @@ on the proposer.) parallel log of data. RFC-026 Phase A already consumes Lance's MemWAL initializer for recoverable empty enrollment; private Phase B1 now consumes its bounded data/ack/replay/fold path rather than building one. Public B2 - remains inactive. B2-0 requires the missing reclamation operation to live in - Lance and forbids OmniGraph from deleting raw `_mem_wal` paths — **(draft - RFC-026)**. + remains inactive. The selected profile forbids OmniGraph from deleting raw + `_mem_wal` paths and accepts monotonic storage plus loud provider exhaustion; + a future managed-reclamation profile would require the missing operation to + live in Lance — **(draft RFC-026)**. - **Mixed constructive/destructive single mutations (D₂).** Keeps in-query read-your-writes unambiguous and each table at one commit per query; the alternative buys an in-query delete-view machine in the hot path. @@ -993,46 +997,42 @@ preserves that historical foundation. **Implemented private core:** RFC-026 Phase B1 in internal schema v8/config-v2 with a schema-v11 `StreamFold` envelope—one root-scoped, no-roll generation; -watcher-only durable acknowledgement; conservative replay; exact +watcher success plus the same writer's post-durability epoch check before a +clean acknowledgement; conservative replay; exact seal/retirement/fold; and atomic table-pointer plus lifecycle-witness publish. -The local cost matrix, configured-RustFS warm-ack cell, genuine v7↔v8 -refusal/rebuild, and 24/24 graph-level suite are green. It is reachable only +The cost, genuine v7↔v8 refusal/rebuild, and graph-level suite remain green. +Gate R0's legal high-entropy near-cap failure is repaired: fold scanning now +charges logical slices and densifies selected rows, so the batch folds and +publishes without lowering the 8,192-row/32-MiB admission cap. The measured +one-exclusive-fold RSS delta is 284,934,144 bytes (~272 MiB); the 384-MiB CI +threshold is a remeasurement tripwire, not a runtime allocator limit. B1 is reachable only through a feature-gated private engine seam and folds only already-normalized physical rows/vectors without external embedding or unspecified derivation. RFC-026 remains Draft: there is no production enrollment, row put/ack/fold, drain/resume, or fresh-read surface. -**Specified, inactive contract:** RFC-026 Phase B2-0 defines explicit +**Retain-all decision and the inactive public contract:** The first profile is +unbounded retain-all on stock Lance. OmniGraph performs no MemWAL GC and claims +no retained-byte/file-count bound; provider exhaustion is an accepted loud +operational risk. RC.1's missing durable cross-open materialization-attempt +receipt and complete physical-output envelope remain documented limits on any +future bounded-storage claim, not blockers for this profile. No schema or +product state was activated by the decision. + +The common B2 inventory defines explicit enrollment, compare-and-chain tokens, trusted visible-winner/current-withdrawal attribution, persistent lifecycle/correction with monotonic revisions and -bounded terminal management receipts, and an enforced retained-storage -admission watermark backed by Lance-owned reclamation. Two checked-in RC.1 +bounded terminal management receipts. The retain-all profile adds no storage +admission watermark. Managed reclamation remains deferred Lance-owned work. +Two checked-in RC.1 guards prove generic cleanup does not reclaim `_mem_wal` and deleting the successor's empty WAL fence sentinel can let a stale writer report WAL success. Those are no-go evidence, not a collector. -The next dependency is Lance-owned durable inspect/plan/execute plus -attempt/receipt recovery, bounded history checkpointing, an enforced physical -growth reservation, bounded materialization attempts/control headroom, and the -post-success epoch patch. Its claims are sentinel-first and manifest-named; -ordinary claims preserve the replay cursor, while only proved whole-cut reclaim -advances it. Enrollment creates the genesis bootstrap before a new MemWAL -details kind that stock RC.1 rejects. A per-binding reserve-first ledger owns -the physical budget; versioned/soft-delete/Object-Lock stores are refused unless -all retained bytes are exactly accountable and eligible versions removable. -OmniGraph will pin the patch at an exact reviewed fork commit without waiting -for release. - -That ledger bounds `_mem_wal`, not base/token or shared manifest history. B2-0 -therefore specifies a separate manifest-authoritative, graph-global -`GraphHistoryBudget`. Every manifest writer participates in its reserve-first -publication and source-bounded physical-growth envelope; pending recovery -charges remain durable. Initial B2 globally serializes this gate from final -reserve/sidecar arm through physical effects, manifest CAS, and charge -finalization. Dynamic per-stream closure reserves cannot be spent -by ordinary work or by another stream. Reaching the shared floor returns -`GraphRebuildRequired` before effect while correction, quiesce, and `SEALED` -export/rebuild remain funded. This keeps B2 a finite-lifetime root profile; -indefinite in-place maintenance remains later work. +If managed reclamation is later justified, its dependency remains Lance-owned +durable inspect/plan/execute with receipt recovery and post-success fencing. +Likewise, a future hard whole-root lifetime promise may introduce a +`GraphHistoryBudget`, but it is not part of retain-all or the immediate public +activation plan. **Roadmap / research:** durable table heads / heads format **(RFC-024, research-blocked after Gate A rejected the first @@ -1040,10 +1040,10 @@ physical access shape)**; checkpoint-pinned retention **(RFC-025, research-blocked after Gate 0 rejected the current compacted registry-access shape)**; public MemWAL row admission and later lifecycle/read phases **(RFC-026, draft; -private one-generation Phase B1 implemented; B2-0 contracts specified; public -strict Phase B2 only after the Lance patch plus token/attribution, -reclamation/enforced-watermark, correction, lifecycle, cross-version, and product-parity -evidence)**; +private one-generation Phase B1 implemented and closure-green; unbounded +retain-all selected on stock RC.1; managed reclamation is optional later work; +public strict activation still requires the common token/attribution, +correction, lifecycle, cross-version, and product-parity evidence)**; lineage-based merge deltas **(RFC-027, research-blocked)**; background reconciler; planner statistics/cost model; policy pushdown; ingest-time embeddings; per-query @@ -1061,7 +1061,7 @@ resource budgets. | **R4: Manifest authority access grows with commit count.** Current-state resolution folds history; a selective index does not by itself bound the complete physical read. | Medium | `optimize` compacts internal tables (keeps periodically-optimized shipped paths flat where separately cost-gated). RFC-024 Gate A rejected durable heads because representative RustFS latest-manifest reads/bytes grow despite flat exact-BTREE row/range work. RFC-025 Gate 0 independently rejected checkpoint-registry activation: at local 10→1,000 on RC.1, uncompacted reconciled work and the eight-fragment tail stay flat, but compacted list/cleanup scan bytes grow 17,012→38,000 cold and 12,336→15,064 warm; exact-show bytes and operation counts also grow. Both RFCs are research-blocked; v8 retains the journal fold and internal-table *cleanup* remains deferred behind the resurrection watermark. | | **R5: Schema identity corruption or alias/identity drift.** Internal schema v5 introduced stable IDs/incarnation as durable authority; v6, v7, and v8 preserve them. | Medium | Open/init validate the SchemaIR domain and exact bidirectional IR↔manifest identity/path/alias contract; every active recovery envelope carries the identity pair; zero, duplicate, missing, or mismatched identity fails closed. | | **R6: Merge cost at divergence** — full-width classification and history-growing manifest folds. | Medium | Coherent coordinator scans plus retained probe handles reduced the pre-slice measured depth-5/depth-80 baseline from 59/651 manifest reads to 40/410 and cap the common fast-forward route at three internal opens and three scans, but the uncompacted-history slope remains. `merge_cost.rs` keeps both facts visible; O(delta) merge is blocked on a real deletion-delta source **(RFC-027)**; fragment adoption is **(draft RFC-0001)**. | -| **R7: No public streaming row path** — production writes are still capped by the `graph_head` CAS rate; high-frequency small writes remain wasteful outside the private evidence seam. | Medium | MemWAL is the strategic substrate. Phase A makes its opaque initializer + separate shard effect recoverable for one main-only/unsharded/single-live-writer-process empty enrollment, durably records lifecycle authority, and excludes competing local effects. Private B1 implements and evidence-qualifies one root-scoped, no-rollover generation capped at 8,192 rows/32 MiB, watcher-only acknowledgement, conservative replay, and one recovery-v11 fold; its local/RustFS, cross-version, and 24/24 graph-level gates are green. B2-0 now specifies attribution, compare-and-chain sequencing, current-token authority, correction, persistent lifecycle, and reclamation/enforced watermark. Stock RC.1 generic cleanup and raw successor-fence-sentinel deletion are proven no-gos, so the first implementation dependency is the reviewed Lance durable inspect/plan/execute, reclaim recovery/history checkpoint, enforced growth reservation, and post-success-fence patch, pinned without waiting for release. Public B2 remains closed until that and the private v9 crash/storage/cross-version matrices pass. A public exact enrollment receipt plus reversible admission seal gates only broader overlapping-process topology. | +| **R7: No public streaming row path** — production writes are still capped by the `graph_head` CAS rate; high-frequency small writes remain wasteful outside the private evidence seam. | Medium | MemWAL is the strategic substrate. Phase A makes its opaque initializer + separate shard effect recoverable for one main-only/unsharded/single-live-writer-process empty enrollment, durably records lifecycle authority, and excludes competing local effects. Private B1 implements one root-scoped, no-rollover generation with watcher-plus-post-fence acknowledgement, conservative replay, and one recovery-v11 fold. Its legal high-entropy near-cap shape now closes through logical-slice accounting plus dense `take`; the measured one-exclusive-fold RSS delta is 284,934,144 bytes (~272 MiB), with a 384-MiB CI remeasurement tripwire. The selected first profile retains every MemWAL object without an OmniGraph byte/file bound and accepts loud provider exhaustion. Managed reclamation and a whole-root history budget are optional later work. Public activation remains closed on the common token/attribution/lifecycle/correction, private-format, cross-version, and product-parity contracts—not on a storage receipt. A public exact enrollment receipt plus reversible admission seal gates broader overlapping-process topology. | | **R8: Some operations lack enforced memory/time budgets.** | Medium | Known gap, narrowed and accepted for RFC-023. Its direct-substrate instrument rejected the first whole-delta fenced adopt (~447 MB peak at 100K × 256 versus ~74 MB Append), and the first corrected production 10K series failed at 30.0× / 108,625,920 bytes overhead; both negative results remain evidence. Mutation/Load now refuses a keyed table above 8,192 rows / 32 MiB before arm, while BranchMerge uses a recovery-enrolled chain with the same per-chunk bounds and a 1,024-transaction ceiling. The inductive certificate route removes the general diff, temporary delta, target preflight, and target join without weakening that chain. Final five-pair production medians passed at 31/8 ms (3.875×) for 10K and 136/35 ms (~3.886×) for 100K; maximum signed paired RSS overheads were 24,297,472 and 32,604,160 bytes. Inclusive row/transaction ceilings, byte refusal (including materialized blobs), operation-wide validation retention, exact source/target incarnation revalidation, second-generation certificate composition, and both between-chunk recovery directions are pinned; other operations still need explicit bounds. | | **R9: Local-FS conditional-write emulation** (`write_text_if_match` check-then-act gap). | Low | All current callers sit behind the cluster lock protocol; S3 uses true conditional puts; close before admitting any lock-free caller. | | **R10: Doc/spec drift as the system grows** — this document included. | Low | Maintenance contract (same-PR doc updates, `check-agents-md.sh` link CI, "don't lie" stale markers); this canon defers to area docs by construction. | @@ -1092,8 +1092,9 @@ Live design questions, each owned by an RFC or a known gap — not a wishlist: 4. **Which capability owns the next rebuild boundary after v8?** RFC-028 activated stable identity in v5; RFC-023 assigned exact-`id` fencing to v6; RFC-026 Phase A assigned lifecycle/enrollment authority to v7, and private - Phase B1 assigned the data-bearing config-v2/recovery-v11 core to v8. B2-0 - assigns token/lifecycle/correction to schema v9/config-v3/state-v2/ + Phase B1 assigned the data-bearing config-v2/recovery-v11 core to v8. The + common B2 inventory assigns token/lifecycle/correction to schema + v9/config-v3/state-v2/ recovery-v12, but that strand is not active. RFC-024's heads, RFC-025's retention, and later RFC-026 phases remain independently reviewable. Any later format activation requires its own @@ -1127,7 +1128,7 @@ The plan of record is the RFC-022…028 family (all under | [0023 — Key-conflict fencing](../rfcs/0023-key-conflict-fencing.md) | Substrate-native keyed-write fencing via Lance's unenforced-PK filter; fleet/format activation barrier | **Implemented** (2026-07-15) | | [0024 — Durable table heads](../rfcs/0024-durable-table-heads.md) | Materialized head-row research; the first exact-BTREE candidate bounded scan work but failed the full latest-manifest/object-byte cost gate | **Research blocked** | | [0025 — Checkpoint-pinned retention](../rfcs/0025-checkpoint-retention.md) | Named checkpoints as authoritative retention roots, materialized as Lance tags; current in-manifest registry lookup rejected by Gate 0 | **Research-blocked** | -| [0026 — MemWAL streaming ingest](../rfcs/0026-memwal-streaming-ingest.md) | Durability-first streaming writes: one-generation private watcher acknowledgement, then graph-atomic fold; B2-0 specifies compare-and-chain tokens, attribution, lifecycle/correction, and Lance-owned reclamation/enforced admission watermark | **Draft; Phase A/B1 implemented and evidence-green; B2-0 specified; public B2 inactive** | +| [0026 — MemWAL streaming ingest](../rfcs/0026-memwal-streaming-ingest.md) | Durability-first streaming writes: one-generation private watcher-plus-post-fence acknowledgement, then graph-atomic fold; legal near-cap closure is green; the selected first profile retains every MemWAL object with no OmniGraph byte/file limit; common token/attribution/lifecycle/correction contracts remain inactive | **Draft; Phase A/B1 private core implemented; unbounded retain-all selected; public inactive** | | [0027 — Lineage merge deltas](../rfcs/0027-lineage-merge-deltas.md) | O(delta) merge classification from row-version lineage | Research-blocked | Deliberately split, not one mega-format: identity, key fencing, head rows, diff --git a/docs/dev/ci.md b/docs/dev/ci.md index 93fa90b6..67de4605 100644 --- a/docs/dev/ci.md +++ b/docs/dev/ci.md @@ -2,13 +2,13 @@ `.github/workflows/`: -- **ci.yml**: text-only changes skip; otherwise `cargo test --workspace --locked` on ubuntu-latest with protobuf compiler. OpenAPI-drift check that auto-commits the regenerated `openapi.json` for same-repository PRs. Also runs the AGENTS.md cross-link integrity check (`scripts/check-agents-md.sh`). +- **ci.yml**: text-only changes skip; otherwise `cargo test --workspace --locked` on ubuntu-latest with protobuf compiler, the engine/cluster failpoint suites, and RFC-026 Gate R0's local retention/closure cells. The Gate-R0 step requires a nonzero pass count, the measured local evidence line, every load-bearing classifier/authority/source/retry/sweep/widest cell by exact name and polarity, and the full-fold RSS remeasurement cell with its 384-MiB tripwire. A rename, partial filter, inverted test, or empty match therefore fails. The tripwire detects implementation-shape drift; it is not a runtime allocator limit. OpenAPI-drift check that auto-commits the regenerated `openapi.json` for same-repository PRs. Also runs the AGENTS.md cross-link integrity check (`scripts/check-agents-md.sh`). - **`Test Workspace` does not run on pull requests.** The job is gated `if: github.event_name != 'pull_request'`, so the full workspace + failpoints suite runs only on push to `main` (post-merge), on `v*` tags, and on manual `workflow_dispatch`. This was a deliberate PR-latency trade-off — it was the slowest gate (~15min warm, up to the 75min cold ceiling). `RustFS S3 Integration` `needs: test`, so it is push-/dispatch-only for the same reason. The fast PR gates remain: `Classify Changes`, `Check AGENTS.md Links`, and `Test omnigraph-server --features aws`. `Test Workspace` is correspondingly **not** in the required-check list (`.github/branch-protection.json`); see [branch-protection.md](branch-protection.md). - **Consequences to internalize:** (1) a regression that the suite would catch now lands on `main` and turns the post-merge run red, rather than being blocked pre-merge — `main` can briefly break, so run `cargo test --workspace --locked` locally before merging anything non-trivial, or trigger this workflow on your branch via the Actions "Run workflow" button. (2) `openapi.json` is no longer auto-regenerated on PRs (that step is inside the `test` job); for server/API changes, regenerate it locally with `OMNIGRAPH_UPDATE_OPENAPI=1 cargo test -p omnigraph-server --test openapi` and commit it, or the strict drift check fails the post-merge `main` run. - **Applying this policy:** removing `Test Workspace` from the JSON is inert until an admin runs `./scripts/apply-branch-protection.sh`. **Run it immediately after this change merges** — until then GitHub still requires a `Test Workspace` context that no longer reports on PRs, which leaves every open PR permanently pending (the job-never-reports trap). - **AWS feature build job**: `cargo build/test -p omnigraph-server --features aws` on ubuntu-latest. - **Windows binary build job**: `cargo build --release --locked -p omnigraph-cli -p omnigraph-server` on windows-latest with smoke checks for `omnigraph.exe version`, `omnigraph-server.exe --help`, and PowerShell installer syntax. -- **RustFS S3 integration**: spins up RustFS in Docker and runs the bucket-gated S3 suites against it. **Sharded across parallel runners** (`strategy.matrix.shard`: `engine` = `s3_storage` plus the exact `lance_surface_guards::public_physical_ref_token_rejects_s3_same_version_aba` guard, `server` = server `s3`, `cluster` = `s3_cluster`, `cli` = `local_cli_s3_end_to_end_init_load_read_flow`, `failpoints` = `failpoints s3_`), one suite per shard with `fail-fast: false` and a per-shard `rust-cache` key. The physical-ref step checks that exactly one test matched so a renamed guard cannot pass vacuously. This job carries **correctness** suites only; RFC-013's `write_cost_s3` and RFC-024's `durable_head_lookup_cost` S3 matrix remain on-demand **cost** instruments (cost/perf contracts belong in a dedicated harness, not the correctness path). The tests run in seconds; the wall-clock is the per-shard `cargo test` **compile** of the engine tree, so on a cold cache (any `Cargo.lock` change) six serial steps summed past the old 75-min timeout — sharding makes wall-clock the slowest single shard (~40m cold, ~25m warm). `needs: test`, so like `Test Workspace` it is push-/dispatch-only. Not a required check. +- **RustFS S3 integration**: spins up RustFS in Docker and runs the bucket-gated S3 suites against it. **Sharded across parallel runners** (`strategy.matrix.shard`: `engine` = `s3_storage` plus the exact `lance_surface_guards::public_physical_ref_token_rejects_s3_same_version_aba` guard, `server` = server `s3`, `cluster` = `s3_cluster`, `cli` = `local_cli_s3_end_to_end_init_load_read_flow`, `failpoints` = `failpoints s3_` plus RFC-026 Gate R0's exact configured-RustFS widest-generation retain-all closure cell), one suite per shard with `fail-fast: false` and a per-shard `rust-cache` key. The physical-ref and Gate-R0 steps check that their exact filters matched so a renamed test cannot pass vacuously; Gate R0 additionally requires the measured `rustfs` evidence line, so its explicit unconfigured skip fails CI. This job carries **correctness and load-bearing decision** suites only; RFC-013's `write_cost_s3` and RFC-024's `durable_head_lookup_cost` S3 matrices remain on-demand **cost** instruments. Gate R0's configured-object-store cell preserves closure and monotonic-retention evidence, not a finite storage claim. The tests run in seconds; the wall-clock is the per-shard `cargo test` **compile** of the engine tree, so on a cold cache (any `Cargo.lock` change) six serial steps summed past the old 75-min timeout — sharding makes wall-clock the slowest single shard (~40m cold, ~25m warm). `needs: test`, so like `Test Workspace` it is push-/dispatch-only. Not a required check. - **release-edge.yml**: on every push to main, retags `edge`, builds Linux x86_64 / Linux arm64 / macOS arm64 archives and Windows x86_64 zip + sha256, publishes a rolling prerelease, then smoke-tests the Windows PowerShell installer against `edge`. - **release.yml**: on `v*` tags, builds the Linux x86_64 / Linux arm64 / macOS arm64 archives and Windows x86_64 zip release matrix, updates the Homebrew tap (`scripts/update-homebrew-formula.sh`) by pushing the regenerated formula to `ModernRelay/homebrew-tap`, and smoke-tests the Windows PowerShell installer against the tag. - **package.yml**: manual ECR image build; emits two image tags per commit (``, `-aws`) via CodeBuild. diff --git a/docs/dev/index.md b/docs/dev/index.md index 04a08be9..2b357db3 100644 --- a/docs/dev/index.md +++ b/docs/dev/index.md @@ -90,7 +90,7 @@ as durable disposition history after closure, so RFC backlinks stay valid. | Area | Read | |---|---| -| RFC-022–028 split architecture review — RFC-022/023/028 implemented; RFC-024/025/027 research-blocked; RFC-026 Phase A and private B1 implemented/evidence-green, B2-0 specified, and public B2 inactive; dependency corrections and acceptance evidence remain tracked | [rfc-022-027-architecture-review.md](rfc-022-027-architecture-review.md) | +| RFC-022–028 split architecture review — RFC-022/023/028 implemented; RFC-024/025/027 research-blocked; RFC-026 Phase A/B1 private core implemented with widest-shape closure green, unbounded retain-all selected, and common public/B2b contracts inactive; dependency corrections and acceptance evidence remain tracked | [rfc-022-027-architecture-review.md](rfc-022-027-architecture-review.md) | ## Active Implementation Plans diff --git a/docs/dev/invariants.md b/docs/dev/invariants.md index 908094ec..98067c5e 100644 --- a/docs/dev/invariants.md +++ b/docs/dev/invariants.md @@ -158,7 +158,12 @@ converge the physical state. Nothing is dropped, auto-merged, or degraded silently. Permitted retries are defined by the operation's contract and bounded; failures are never swallowed. Metrics and capabilities must be exposed before planning or - operator behavior depends on them. + operator behavior depends on them. These bounds apply to row and memory + admission, deadlines, retries, and ambiguous outcomes; they do not require + an OmniGraph storage quota. RFC-026's selected unbounded retain-all profile + has no byte, object, file, or history ceiling. Exhaustion at the storage + provider is surfaced and may halt progress without weakening durability, + recovery, or manifest-only visibility. 14. **Evidence matches the boundary and reversibility.** Tests exercise the layer whose contract changed; end-to-end coverage does not replace missing @@ -187,7 +192,7 @@ converge the physical state. | Constructive mutations | In-memory `MutationStaging`, one end-of-query table commit per touched table, then one manifest publish | [writes.md](writes.md), [execution.md](execution.md) | | Keyed writes | Every current v8 node/edge table declares exact non-null physical `id` as Lance's unenforced PK from creation, using the v6-introduced, v7/v8-preserved exact-`id` fence. General production strict insert and upsert use the sealed exact-`id`, forced-v2 MergeInsert adapter; strict insert exact-probes its pinned parent before minting `omnigraph.insert_absence=v1`, while an all-new upsert may mint it only when completed effect statistics prove one attempt inserted every source row with zero updates, deletes, or skipped duplicates. Certificate admission is optional: BranchMerge uses the shortcut only when every transaction in the complete contiguous source interval carries v1 and its persisted operation is the full pure-insert `Update` shape (exact parent, no removed or updated fragments, nonempty new fragments with `physical_rows`, no field or generation rewrites, `RewriteRows`, exact-`id` filter, full nested schema preorder, and matching physical-row total). It then rechecks both source and target native ref incarnations and passes owned batches through an opaque capability whose one production mint site is structurally guarded. The proven publisher stages immutable fragments with `InsertBuilder`, replaces the uncommitted Append operation with that filter-bearing `Update`, and performs zero target preflights, target merge joins, or committed Appends. Missing, cleaned, unknown, or malformed proof uses the general ordered diff. Mutation/Load remains one keyed transaction per table and rejects more than 8,192 rows or 32 MiB before arm; BranchMerge keeps its bounded v4 chain and exact-recovery limits. Raw Lance graph writers are unsupported, and the certificate is an internal non-cryptographic capability rather than an authenticity mechanism. The final production cost gate passed at 10K (3.875× median; 24,297,472-byte max paired RSS overhead) and 100K (~3.886×; 32,604,160 bytes) | [RFC-023](../rfcs/0023-key-conflict-fencing.md), [writes.md](writes.md), [execution.md](execution.md) | | Deletes | Staged like inserts/updates (`stage_delete` via Lance 7.0 `DeleteBuilder::execute_uncommitted`, MR-A) — no inline HEAD advance; mixed insert/update/delete in one query rejected by D2 as a deliberate boundary (constructive XOR destructive per query; compose via separate mutations or a branch) | [query-language.md](../user/queries/index.md), [writes.md](writes.md) | -| Streaming ingest | RFC-026 adopts Lance MemWAL as the strategic substrate and remains Draft; public row streaming is inactive. Historical Phase A schema v7 implemented exact bounded enrollment recovery, identity-keyed lifecycle authority, process-local admission/exclusion, current-HEAD witnesses, compatible-open validation, partial-format refusal, and strict v6↔v7 rebuild for the main-only, one-unsharded-shard, one-live-writer-process profile. Current schema v8/config-v2/recovery-v11 adds the private B1 core behind one feature-gated, doc-hidden engine seam: one already-normalized physical batch enters one hard-capped 8,192-row/32-MiB generation; successful Lance watcher completion proves durability, and a clean `DurableBatchAck` additionally requires the same `ShardWriter::check_fenced()` to return `Ok(())` immediately afterward; automatic rollover is prevented and the writer retires before a successor-generation put; every post-invocation fence, read, task, or deadline ambiguity is typed `AckUnknown` plus worker retirement; replayed or flushed-unmerged residue is fold-only through the guarded BatchStore watermark bridge; and one exact strict fold publishes the achieved table pointer, lifecycle witness, and lineage only at the `__manifest` CAS. Physical vector columns are supplied before acknowledgement and pass through fold unchanged; B1 neither calls an external embedding provider nor invents unspecified fold-derived fields. B1 is evidence-green at this private correctness boundary: genuine v7↔v8 refusal/rebuild and the graph-level B1 behavior/crash/race suite, including post-watcher epoch loss, pass. Post-containment local acknowledgement work remains history-flat at 9 table reads / 219 bytes; configured RustFS retains only the 2026-07-19 pre-containment baseline and requires rerun before a current object-store cost claim. There is no production first-use caller, schema/SDK/HTTP/CLI/OpenAPI surface, operator drain/resume, correction, GC, or fresh read. The adapter recheck does not retract durable WAL bytes, protect raw Lance callers, make fence-sentinel deletion safe, or enable reclamation. B2-0 now specifies—but does not implement—explicit enrollment, durable contributor attribution, compare-and-chain retry safety, a graph-global current-token participant, monotonic lifecycle revisions with bounded terminal management receipts, strict correction, Lance-owned bounded reclamation and post-success fencing, and a graph-global `GraphHistoryBudget` checked by every manifest publisher. Public activation remains gated on implementing those contracts and passing their crash, storage, cross-version, authorization, and product-parity evidence | [RFC-026](../rfcs/0026-memwal-streaming-ingest.md), [writes.md](writes.md) | +| Streaming ingest | RFC-026 adopts Lance MemWAL as the strategic substrate and remains Draft; public row streaming is inactive. Historical Phase A schema v7 implemented exact bounded enrollment recovery, identity-keyed lifecycle authority, process-local admission/exclusion, current-HEAD witnesses, compatible-open validation, partial-format refusal, and strict v6↔v7 rebuild for the main-only, one-unsharded-shard, one-live-writer-process profile. Current schema v8/config-v2/recovery-v11 adds the private B1 core behind one feature-gated, doc-hidden engine seam: one normalized physical batch enters one hard-capped 8,192-row/32-MiB generation; watcher success plus the same writer's post-durability `check_fenced()` is required for a clean acknowledgement; replayed or flushed-unmerged residue is fold-only; and one exact strict fold publishes table pointer, lifecycle witness, and lineage only at the `__manifest` CAS. Fold scanning now charges each logical slice against the generation cap and copies every scanner emission into dense owned arrays before retaining it. The deterministic 8,192-row high-entropy near-cap cell therefore closes and publishes; its isolated fold RSS delta measured 284,934,144 bytes (about 272 MiB), below the 384-MiB remeasurement tripwire. The selected unbounded retain-all profile imposes no OmniGraph retained-byte, object-count, file-count, or history quota and performs no raw `_mem_wal` deletion. Provider exhaustion is loud and may block admission, fold, or recovery; row/memory/deadline/retry/ambiguity bounds remain. RC.1's absent cross-open materialization-attempt receipt and complete physical-output envelope are no longer activation blockers for this profile, but remain inputs to the future B2b managed-reclamation route. Compare-and-chain tokens, attribution, lifecycle receipts, correction, authorization, and product parity remain unimplemented. There is no production first-use caller, schema/SDK/HTTP/CLI/OpenAPI surface, operator drain/resume, correction, GC, fresh read, or schema v9 | [RFC-026](../rfcs/0026-memwal-streaming-ingest.md), [writes.md](writes.md) | | Branch create/delete | `__manifest` `BranchContents` is the single logical authority. Lance create is physically two-phase, so OmniGraph prevalidates names, enforces path-prefix-disjoint live graph names, reclaims an absent-ref clone-only tree, and uses a bounded completion classifier; delete removes authority before tree cleanup, so an absent ref is success and derived tree reclaim may converge later. Neither control emits graph lineage. Under schema/source-target/all-table gates, each control uses one operation-local post-gate manifest/namespace capture rather than refreshing the handle-local coordinator around table-gate acquisition; successful ref movement explicitly invalidates derived read caches. Per-table forks are derived state, reclaimed best-effort with `cleanup` as backstop. A target-scoped unresolved sidecar may be made unreachable by deletion and is then audit-discarded by recovery; graph-global SchemaApply still blocks | [branches-commits.md](../user/branching/index.md), [maintenance.md](../user/operations/maintenance.md), [writes.md](writes.md) | | Cleanup retention | Explicit cleanup derives exact `keep` cutoffs from Lance's available version list, caps each main-table GC cutoff at the oldest exact main version inherited by any live lazy graph branch, and refuses uncovered main HEAD drift. Lance protects native per-table branch refs itself. The graph-wide live-reference preflight fails closed before the first table GC, after which individual table failures remain fault-isolated | [maintenance.md](../user/operations/maintenance.md), [writes.md](writes.md) | | Optimize visibility | One operation-local accepted catalog and fresh main snapshot are planned under schema → main → all-table gates. Every productive table shares one identity-bearing v9 recovery envelope; bounded-parallel physical effects become visible through at most one monotonic manifest/lineage CAS. Complete crash residuals roll forward together and partial residuals compensate before visibility. Optimize's payload remains a bounded maintenance adapter rather than an exact caller-minted Lance transaction proof, so it is supported within the single-writer-process recovery boundary. Exact provenance is deferred until Lance exposes a stable public caller-controlled maintenance transaction API and OmniGraph has distributed recovery fencing | [maintenance.md](../user/operations/maintenance.md), [writes.md](writes.md), [RFC-022](../rfcs/0022-unified-write-path.md) | @@ -223,7 +228,7 @@ incomplete would let a legacy writer prepare from unresolved physical state. Do not hide these behind invariant wording. Either move them forward or keep them explicit. -- **Private B1 core is evidence-green; public row streaming is not:** RC.1's public +- **Private B1 closure is repaired; public row streaming is not active:** RC.1's public initializer commits the singleton MemWAL index internally and shard claim is a separate durable effect. RFC-026 no longer treats an upstream release as the only calendar path: a production-neutral harness proved exact @@ -248,7 +253,7 @@ them explicit. error, replay leaves its BatchStore flush watermark unset, and a late drain waiter can miss a completed failure. Current schema v8/config-v2/recovery-v11 implements the - bounded private response: one root-scoped worker and one hard-bounded, + bounded private response: one root-scoped worker and one nominally bounded, no-rollover generation per writer; admission held before epoch claim; watcher-backed durability followed by a same-writer post-durability `check_fenced()` before clean acknowledgement; every post-invocation fence or @@ -263,31 +268,41 @@ them explicit. already-normalized physical rows and vectors; it performs no external embedding call or unspecified fold-derived-field materialization. It is not a product surface and performs no GC or correction. - B2 is the later public strict activation. B2-0 now specifies, but does not - implement, compare-and-chain tokens, trusted attribution, a graph-global - manifest-selected current-token participant, persistent lifecycle/correction - with monotonic revisions and bounded terminal management receipts, and Lance- - owned reclamation. The reclamation contract uses sentinel-first - manifest-named claims, a fail-closed bootstrap/checkpoint format, one per- - binding reserve-first physical ledger, exact multipart/versioned-object - accounting or refusal, and hard admission/control bounds; OmniGraph never - deletes raw `_mem_wal` paths. The MemWAL bound does not cover base/token/shared - manifest history. B2-0 therefore adds one manifest-authoritative, graph-global - `GraphHistoryBudget`: every manifest publisher reserves its exact publication - and source-bounded physical-growth envelope before effect and holds the gate - through effect, manifest CAS, and charge finalization, while dynamic per- - stream closure reserves remain independently spendable through correction, - quiesce, and `SEALED` rebuild. Public B2 is finite-lifetime per root until a - later accepted retention strand. Schema v9/config-v3/state-v2/recovery-v12 and - every product surface remain inactive until those contracts and their crash, - storage, cross-version, authorization, and parity evidence are green. - B1's genuine v7↔v8 cross-version/rebuild and complete graph-level B1 suite, - including post-watcher epoch loss, pass. The post-containment local cost run - remains history-flat; the configured-RustFS post-containment rerun is pending. - The non-flat uncompacted-manifest term remains explicit and no product - latency or group-commit multiplier is inferred. The exact upstream receipt and - cross-process seal remain required to broaden topology, not to run B1 inside - the existing boundary. See + The closure defect found by Gate R0 is repaired at the fold boundary: the + scanner charges logical slice bytes against the same 32-MiB generation limit, + takes every emitted row into dense owned Arrow arrays, and drops the sparse + parent representation before retaining the batch. The deterministic + 8,192-row high-entropy near-cap generation now folds and publishes exactly + once. Its isolated fold RSS delta was 284,934,144 bytes (about 272 MiB), below + the 384-MiB remeasurement tripwire; crossing that tripwire requires + remeasurement rather than a silent admission change. + + The selected next profile is unbounded retain-all. A flush may write a fresh + randomized generation subtree before the shard-manifest CAS, and a cold + replay may therefore leave additional materialization roots. OmniGraph + deliberately imposes no retained-byte, object-count, file-count, or history + quota, adds no cross-open attempt ledger for quota enforcement, and never + deletes raw `_mem_wal` paths. The strict current-object census remains useful + as retention and path-shape evidence, not as a claim about multipart uploads, + provider versions, billed bytes, or a finite storage envelope. Provider + exhaustion is a loud storage failure and may halt admission, folding, or + recovery until capacity is restored; it never permits an acknowledgement to + disappear or a partial effect to become graph-visible. Row and Arrow-memory + admission, deadlines, retries, and ambiguity handling remain bounded. + + RC.1's missing durable materialization-attempt receipt and reserve-first + complete-output envelope are consequently not blockers for unbounded + retain-all. They remain relevant to the future B2b Lance-owned + inspect/plan/execute, checkpoint, inventory/accounting, and reclamation + design. Compare-and-chain tokens, trusted attribution, graph-global + current-token authority, lifecycle receipts, strict correction, + authorization, and product parity remain required before a public profile; + a `GraphHistoryBudget` or retained-storage watermark belongs only to a future + bounded/managed profile, not this one. Schema + v9/config-v3/state-v2/recovery-v12 and every product surface remain inactive. + The exact upstream receipt and cross-process seal remain required to broaden + topology, not to run the existing private seam inside its current process + boundary. See [RFC-026](../rfcs/0026-memwal-streaming-ingest.md). - **Storage abstraction:** `TableStorage` is present, sealed, and canonical for staged writes. MR-854 made `db.storage()` staged-only; the exact EnsureIndices diff --git a/docs/dev/lance.md b/docs/dev/lance.md index ec6b8e10..8d42ec1d 100644 --- a/docs/dev/lance.md +++ b/docs/dev/lance.md @@ -171,7 +171,7 @@ for schemas that avoid the newly reserved names documented below. RC.1 requires Rust 1.91 or newer; OmniGraph continues to track stable Rust and this audit ran on Rust 1.95. -#### Post-audit implementation notes: 2026-07-18–19 +#### Post-audit implementation notes: 2026-07-18–20 RFC-026 Gate E0 subsequently passed and Phase A activated OmniGraph internal schema v7. V7 adds only the bounded streaming foundation: exact recovery-v10 @@ -227,7 +227,9 @@ manifest. And RC.1 writes the randomized generation dataset and sidecars before that manifest CAS, so a crash may leave a complete or partial unreferenced `{hash}_gen_N/` subtree. B1 treats only that recognized subtree shape as retained derived orphan output—never adopted, scanned, or deleted—and leaves -reclamation to B2's proved GC contract. Seal/drain/abort are background-owned +it in place under the selected unbounded retain-all profile. A later B2b +managed profile would need a separately proved Lance-owned GC contract. +Seal/drain/abort are background-owned with deadline-bounded caller waits: an error or stalled handler keeps the original task and abort completion retained, the registry retired, and admission closed. The caller never cancels `shutdown_all`, retries abort, or @@ -248,21 +250,23 @@ drain's captured floor. Replay preserves possible residue but cannot resolve which caller attempt produced it. Configuration identity binds only correctness/topology/no-rollover fields; explicit runtime policy and injected Session/store capabilities remain separate. Private B1 activates graph schema -v8 and stream-config v2 rather than adopting v7/config-v1 in place. Phase B2 is -the later public strict activation. B2-0 now specifies explicit enrollment, -durable trusted contributor attribution, compare-and-chain write/predecessor -tokens, a manifest-selected current-token participant, bounded correction, and -persistent revisioned quiesce/resume/abort-drain/rebuild with bounded terminal -management receipts. B1's genuine v7↔v8 -cross-version/rebuild, -complete 24-cell graph-level B1 suite, qualified local cost/PK-index/RSS, and -configured RustFS cost gates now pass; the measured uncompacted-manifest term remains -explicitly non-flat. That makes the private B1 boundary green, not public. The -missing combined enrollment receipt and cross-process admission seal still gate -broader topology, not B1 inside the existing single-live-writer-process -boundary. - -The 2026-07-19 B2-0 reclamation audit adds a separate stock-RC.1 no-go. Generic +v8 and stream-config v2 rather than adopting v7/config-v1 in place. The selected +next profile is B2a **unbounded retain-all**: OmniGraph deletes no raw +`_mem_wal` path, imposes no retained-byte/object/file/history quota, and accepts +loud provider exhaustion. Explicit enrollment, trusted attribution, +compare-and-chain tokens, manifest-selected current-token authority, bounded +correction, revisioned lifecycle/management receipts, authorization, and +product parity remain common contracts. A graph-history budget is not one of +them for this unbounded profile. Gate R0's legal high-entropy near-cap shape +originally retained sparse scanner backing buffers and failed the fold charge; +logical-slice accounting plus dense owned copies now make that exact shape +acknowledge, materialize, fold, and publish. A reference isolated run measured +a 284,934,144-byte fold RSS delta, below the 384-MiB CI remeasurement tripwire. +The missing combined enrollment receipt and cross-process admission seal still +gate broader topology, not the existing private single-live-writer-process +seam or unbounded retention. + +The 2026-07-19 B2b reclamation audit adds a separate stock-RC.1 no-go. Generic `cleanup_old_versions` can reclaim ordinary dataset versions but does not walk or delete `_mem_wal`; the runtime guard `cleanup_old_versions_does_not_reclaim_mem_wal_objects` pins that ownership @@ -273,7 +277,9 @@ the stale writer can still complete the put and receive watcher success even though an explicit check returns `PeerClaimedEpoch`. These guards forbid raw MemWAL deletion in OmniGraph; they do not implement reclamation. -RFC-026 §4.5 therefore requires a Lance-owned opaque inspect/plan/execute +If bounded managed reclamation is later scheduled, RFC-026 §4.5.2 requires B2b +to use a Lance-owned opaque +inspect/plan/execute primitive with exact base/shard/history and whole-cut/cursor witnesses, durable attempt/receipt lost-result recovery, manifest-version plus epoch advancement before deletion, conservative orphan/unknown classification, bounded @@ -297,23 +303,80 @@ double-reserve across shards. Lance must source and enforce the maximum physical-growth reservation, durable materialization-attempt limit, bounded claim/reclaim/checkpoint history, and emergency control headroom before admission; local/RustFS measurement validates those bounds but does not create -them. OmniGraph will author that patch, open it upstream, and pin its exact -reviewed fork commit without waiting for a release. Public B2 remains inactive -until the positive local/RustFS crash and bound-validation matrix passes. WAL- -prefix reclamation is limited to a quiescent whole cut because RC.1 does not -persist a per-generation WAL range. - -That Lance-owned ledger bounds only one physical `_mem_wal` binding. It does not -bound the graph's base/token datasets or shared `__manifest` history. B2-0 -therefore separately specifies a manifest-authoritative graph-global -`GraphHistoryBudget`, checked by every manifest publisher. Each operation -reserves the exact publication and source-bounded physical-growth envelope -before effect, holds the global gate through effect/CAS/finalization, persists -pending recovery charges, and settles them exactly once; -dynamic per-stream closure reserves cannot be borrowed by ordinary work or by -another stream. This OmniGraph authority is specified but inactive together -with schema v9/config-v3/state-v2/recovery-v12 and requires its own bootstrap, -crash, refusal, every-writer, and local/RustFS physical-bound evidence. +them. An upstream proposal remains useful for a future bounded profile, but it +is not on the unbounded retain-all activation path. WAL-prefix reclamation is +limited to a quiescent whole cut because RC.1 does not persist a per-generation +WAL range. + +That optional Lance-owned ledger would bound only one physical `_mem_wal` +binding; it would not bound the graph's base/token datasets or shared +`__manifest` history. A future product that promises a finite whole-root bound +would therefore need a separate RFC for a manifest-authoritative graph-global +`GraphHistoryBudget`, including bootstrap, crash, refusal, every-writer, and +local/RustFS physical-bound evidence. The selected retain-all profile makes no +such promise and carries no such authority. + +#### Gate R0 source audit: 2026-07-20 bounded-retention result and 2026-07-21 disposition + +Gate R0 returned **no-go** for a B2a contract that promised finite retained +storage on stock RC.1. The result is tied to exact revision +`cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d`; the checked-in decision test fails +when the lockfile pin moves so the audit cannot silently outlive its source. +On 2026-07-21 the RFC selected unbounded retain-all instead. The same source +facts remain true, but they no longer block a profile with no storage ceiling, +no raw-path deletion, and loud provider exhaustion. + +The materialization ordering is decisive. `MemTableFlusher::flush` checks the +epoch, chooses a fresh eight-hex random generation directory, writes the Lance +generation dataset, optional deletion state, Bloom filter, and mandatory PK +sidecar, and only then calls the shard-manifest CAS +(`mem_wal/memtable/flush.rs:223–280`; path construction is +`mem_wal/util.rs:181–209`). Each `?` may leave a partial or complete +unreferenced subtree. One background message invokes the flusher once and does +not transparently retry the failed message (`mem_wal/write.rs:2658–2716`), but +that is not a lifetime bound: reopen claims another epoch, reconstructs the +MemTable at unchanged `manifest.current_generation`, replays WAL, and the next +flush chooses another random path (`write.rs:1365–1403`, `1474–1517`). Durable +`ShardManifest` state has no attempt ID, counter, reservation, inventory, or +receipt (`lance-table/src/system_index/mem_wal.rs:177–205`). + +One clean unindexed B1 attempt includes generation data, a transaction, +generation manifest, optional deletion vector, Bloom object, and the PK-BTree +`page_data.lance` plus `page_lookup.lance`; Blob schemas may add packed or +dedicated sidecars. Shard-manifest versions/hints, WAL entries, and one +successor fence sentinel per reopen sit outside that randomized subtree. +`FlushResult` returns generation/path/rows/covered WAL position, not a physical +inventory or byte receipt. The data writer sets `max_rows_per_file = +usize::MAX`; its byte target is soft and checked after a group is written. +Uploads above 5 MiB may use multipart, whose abort-on-drop is best effort and +cannot run after process death; local writes likewise use adjacent randomized +temporary files. Provider and multipart retry settings are not represented in +MemWAL authority. + +The production-neutral current-object census confirms success-path structure +without overclaiming provider state. Locally, one/four/eight successful folds +retain approximately 37.4 / 150.6 / 302.3 thousand currently listed immutable +bytes and exactly the same number of referenced generation roots; every earlier +listed path retains the same class and size. Generated metadata makes the exact +byte totals vary slightly, so monotonic currently listed growth is the stable +assertion. +A retry after the referenced cut reuses that exact root. Before the closure +repair, the deterministic high-entropy near-cap cell recorded 33,174,630 listed +immutable bytes after acknowledgement and about 65.1 million after generation +materialization, while sparse scanner arrays retained roughly 252.8 million +bytes of backing allocation and tripped the 33,554,432-byte logical charge. +Fold now charges the scanner's logical slices and densifies each emission before +retaining it; the same 8,192-row shape folds and publishes. Ordinary LIST still +cannot see incomplete multipart uploads, superseded provider versions, delete +markers, local staging residue, or billed bytes. These measurements validate +facts; they do not supply a finite storage envelope, and unbounded retain-all +does not claim one. + +The selected profile does not add a test-only materialization-attempt ledger or +a pre-attempt storage reservation: those would exist only to enforce a bound the +profile deliberately does not promise. They become relevant again if a later +RFC proposes bounded retention. No schema v9 or public surface is authorized by +the source audit or closure repair alone. The no-roll profile uses fixed portable capacities (`8,193` rows/batches and 1-GiB byte/unflushed thresholds), not architecture-dependent `usize::MAX` diff --git a/docs/dev/rfc-022-027-architecture-review.md b/docs/dev/rfc-022-027-architecture-review.md index ec36cf41..4af96cb7 100644 --- a/docs/dev/rfc-022-027-architecture-review.md +++ b/docs/dev/rfc-022-027-architecture-review.md @@ -1,9 +1,11 @@ # Architecture review: RFC-022 through RFC-028 **Status:** RFC-022, RFC-023, and RFC-028 implemented; RFC-024, RFC-025, and -RFC-027 research-blocked; RFC-026 Phase A and private B1 implemented/evidence- -green, Phase B2-0 specified, and public B2 inactive +RFC-027 research-blocked; RFC-026 Phase A implemented, private B1's legal +near-cap closure repaired and green, unbounded retain-all selected for B2a, +B2b/common contracts inactive, and no public stream surface exists **Date:** 2026-07-11 +**Last updated:** 2026-07-21 **Audience:** RFC authors, engine/storage maintainers, and release reviewers **Reviewed against:** OmniGraph 0.8.1; Lance 9.0.0-beta.15 at `f24e42c11a742581365e1cbe17c906ea2dac1bc6`; full Lance transaction, @@ -28,16 +30,33 @@ activated internal schema v7 with exact empty-enrollment recovery, identity-keyed lifecycle authority, process-local admission/exclusion, partial-format refusal, and strict v6↔v7 rebuild. Private Phase B1 subsequently activated schema v8/config-v2, one root-singleflight no-roll generation, -watcher-only acknowledgement, conservative replay/fold-only reopen, and a -recovery-v11 exact graph-atomic fold. Genuine v7↔v8 rebuild/refusal, all 24 -graph-level race/crash cells, and the qualified local plus configured-RustFS -cost/RSS evidence pass. No production or public caller reaches the row seam; -B2 remains inactive. B2-0 now specifies explicit enrollment, compare-and-chain -tokens, trusted row attribution, manifest-selected current-token state, -protocol-v2 lifecycle revisions plus bounded management receipts, bounded -correction, Lance-owned reclamation with an enforced storage admission -watermark, and a graph-global `GraphHistoryBudget` checked by every manifest -publisher. Two RC.1 surface guards prove generic cleanup +watcher success plus a same-writer post-durability epoch check, conservative +replay/fold-only reopen, and a recovery-v11 exact graph-atomic fold. Genuine +v7↔v8 rebuild/refusal, the prior 24 graph-level race/crash cells, and qualified +cost/RSS evidence remain green. Gate R0 then found a legal high-entropy +near-cap generation that acknowledged and materialized but could not fold +within the original closure implementation. B1 now charges the scanner's +logical slices against the same 32-MiB generation cap and copies every scanner +emission into dense owned arrays before retaining it. The 8,192-row near-cap +cell folds and publishes successfully; the reference subprocess run measured a +284,934,144-byte (about 272 MiB) isolated fold RSS delta, below the 384-MiB +remeasurement tripwire. No production or public caller reaches the row seam. + +Gate R0's 2026-07-20 **historical** no-go applies to a finite or otherwise +bounded retain-all claim on stock RC.1. Random generation roots have no +MemWAL-persisted/enforced durable cross-open materialization-attempt +cap/receipt; RC.1 exposes no admission-grade reserve-first complete +physical-output envelope; and current LIST cannot prove all provider-retained +or billed bytes. The selected B2a profile deliberately makes none of those +finite-storage claims: retain every `_mem_wal` object, perform no raw WAL GC, +impose no retained-byte, object-count, file-count, or history quota, and surface +provider exhaustion loudly. The missing materialization-attempt receipt and +complete envelope therefore do not block B2a. No test-only attempt ledger or +`GraphHistoryBudget` is on its immediate path. B2b remains optional future +Lance-owned managed reclamation. Common explicit enrollment, +compare-and-chain tokens, trusted attribution, revisioned lifecycle and +management receipts, correction, authorization, and product parity remain +specified, required, and inactive. Two RC.1 surface guards prove generic cleanup non-ownership and the stale-writer hazard created by deleting the successor's empty WAL fence sentinel; neither is an implementation of reclamation. The local RFC-024/025 decision instruments were rerun: both remain @@ -56,9 +75,11 @@ and acceptance gates. the full MemWAL table and system-index specifications, including the durability/fencing behavior carried forward from beta.17, are reflected in the RFC's survey and acceptance guards. Gate E0 passed at the explicit bounded -profile and Phase A consumed it as described above; RFC-026 remains draft and -public row streaming is inactive. The B2-0 contract closure and stock-RC.1 -reclamation no-go are recorded in RFC-026 §4.1–§4.6 and BLOCKER-14 below. +process/topology profile and Phase A consumed it as described above; RFC-026 +remains draft and public row streaming is inactive. Gate R0's historical +finite-retention no-go, the selected unbounded B2a disposition, the optional +B2b route, and the common contract inventory are recorded in RFC-026 §0.2, +§4.1–§4.6, and BLOCKER-14 below. **RFC-024/025/027 substrate claims revalidated against:** the current RC.1 revision and the full matching table/index/branch/tag/cleanup, transaction, and row-lineage specification sections; their survey headers now record RC.1. @@ -461,12 +482,18 @@ integration. [§9](../rfcs/0026-memwal-streaming-ingest.md#9-fresh-read-cuts), and [§11–13](../rfcs/0026-memwal-streaming-ingest.md#11-format-activation-and-rebuild) -**Status:** Closed through private Phase B1 implementation/evidence and Phase -B2-0 format-contract specification on 2026-07-19; public B2 activation remains -open. RC.1 lacks the ideal caller-owned enrollment receipt and cross-process -shard-admission seal, but upstream release timing is not the bounded-profile -calendar. Gate E0's exact-classification and ABA cells passed using -known-version probes and strict object-store classification. +**Status:** Format separation is specified. Gate R0's legal near-cap closure +failure was repaired on 2026-07-21 with logical-slice accounting and dense +scanner-output copies; the closure and RSS cells are green. The selected B2a +profile is unbounded retain-all with no raw `_mem_wal` GC and no retained-byte, +object-count, file-count, or history quota. Gate R0's historical no-go applies +only to claiming a finite storage bound on stock RC.1. Public activation remains +closed on the common contracts below, not on the missing materialization-attempt +receipt or physical-output envelope. RC.1 also lacks the ideal caller-owned +enrollment receipt and cross-process shard-admission seal, but those remain +broader-topology gates rather than dependencies of the implemented private +single-live-writer-process profile. Gate E0's exact-classification and ABA +cells passed using known-version probes and strict object-store classification. Phase A then activated a main-only, unsharded, single-live-writer-process empty enrollment foundation in internal schema v7. Private B1 activated the bounded data-bearing schema-v8/config-v2 core. RFC-026 remains draft and public row @@ -496,16 +523,15 @@ empty-enrollment contract: implemented v8 is the first data-bearing stream format, uses stream-config v2 and recovery schema v11, refuses v7/config-v1, and carries genuine v7↔v8 rebuild/refusal evidence. -B2 is a third strict strand, not an in-place reinterpretation of v8. B2-0 -assigns internal schema v9, stream-config v3, stream-state protocol v2, and -recovery-v12 to trusted hidden row metadata, the manifest-selected current-token -participant, persistent revisioned lifecycle/correction, bounded terminal -management receipts, and their multi-participant recovery. V9 also owns the -manifest-authoritative graph-global `GraphHistoryBudget`; every manifest writer -must reserve and settle its publication plus source-bounded physical-growth -envelope while preserving dynamic per-stream closure reserves. Those versions -are specified but inactive; genuine v8↔v9 refusal/rebuild and budget -initialization/refusal remain implementation gates. +B2 is a third strict strand, not an in-place reinterpretation of v8. The common +B2 inventory assigns internal schema v9, stream-config v3, stream-state +protocol v2, and recovery-v12 to trusted hidden row metadata, the +manifest-selected current-token participant, persistent revisioned +lifecycle/correction, bounded terminal management receipts, and their +multi-participant recovery. Those versions are specified but inactive; genuine +v8↔v9 refusal/rebuild remains an implementation gate. The selected unbounded +B2a profile adds no storage watermark or `GraphHistoryBudget`; those mechanisms +would belong only to a separately justified future bounded/managed profile. RFC-026 §12 records those tests and §13 places the completed Phase A before the public stream path. RFC-026 §3/§8 also make a rematerialized table a separate physical @@ -536,25 +562,31 @@ failed handler removed the watcher. Both are explicit B1 compatibility gates, not inferred success. The implemented private design is a root-scoped singleflight worker and one -hard-bounded generation per writer: internal schema v8/config-v2, no automatic +nominally bounded generation per writer: internal schema v8/config-v2, no automatic rollover, at most 8,192 rows/32 MiB for the complete generation, explicit admission from before epoch claim, conservative fold-only replay/unmerged-state routing, the pinned public BatchStore watermark bridge before replay reseal, drain proof from empty frozen refs plus the authoritative generation/cursor, immediate writer retirement, one recovery-v11 keyed fold, and higher-epoch -reopen only after publication. `AckUnknown` preserves possible -residue through replay but remains permanently caller-ambiguous. B2 later owns -public strict activation and cannot start merely because B1 is green. B2-0 now -specifies durable authenticated-contributor attribution, compare-and-chain -tokens, a manifest-selected current-token participant, bounded -`REPLACE`/`WITHDRAW` correction, persistent revisioned -status/quiesce/resume/abort-drain/rebuild with bounded management receipts, and -the reclamation/enforced-watermark -contract. Their implementation and crash/cross-version/storage evidence remain -gates. A `SEALED` B2 stream permits export/rebuild; in-place maintenance still -waits for Phase D. - -The bounded candidate instead makes the support restriction load-bearing. It +reopen only after publication. `AckUnknown` preserves possible residue through +replay but remains permanently caller-ambiguous. Fold now charges each scanner +logical slice against the generation cap and takes each emission into dense +owned arrays before retaining it; the legal 8,192-row high-entropy near-cap +cell folds and publishes successfully. B2 later owns public strict activation. +The common B2 inventory specifies explicit enrollment, durable trusted +contributor attribution, compare-and-chain tokens, a manifest-selected +current-token participant, bounded `REPLACE`/`WITHDRAW` correction, persistent +revisioned status/quiesce/resume/abort-drain/rebuild with bounded management +receipts, authorization, and SDK/CLI/HTTP/OpenAPI product parity. B2a selects +unbounded retain-all and loud provider exhaustion with no raw WAL deletion or +storage/history quota. B2b managed reclamation is an optional future +Lance-owned route, not an activation dependency for B2a. The common +implementation plus its crash, cross-version, authorization, and product +evidence remain gates. A `SEALED` B2 stream permits export/rebuild; in-place +maintenance still waits for Phase D. + +The private single-live-writer-process candidate makes that support restriction +load-bearing. It grants an `OPEN` stream exclusive authority to advance that base-table HEAD; every other writer/maintenance/control path touching the table must refuse pre-effect or drain. Stable enrollment identity is kept separate from the @@ -587,11 +619,13 @@ unreadable exact HEAD errors. The configured RustFS cell passes non-vacuously with the same shape plus the declared positive and listing-dependent negative matrix. Surface guards pin the doc-hidden successor, flush/drain, merged-generation, and S3 ABA shapes; CI rejects skipped E0/ABA cells. This was -green evidence for the bounded profile; Phase A subsequently consumed it, but +green evidence for the bounded process/topology profile; Phase A subsequently +consumed it, but it did not authorize a row acknowledgement. -Replica scope must match RFC-023's recovery support boundary. The bounded -profile does not permit overlapping replicas: a crash successor proceeds only +Replica scope must match RFC-023's recovery support boundary. This +process/topology profile does not permit overlapping replicas: a crash +successor proceeds only after external exclusivity, recovery, and a higher shard epoch. Do not advertise general replica failover merely because MemWAL has a shard epoch. @@ -916,15 +950,19 @@ Do not weaken the physical-I/O gate, infer success from flat row/range counters, or add a second authority dataset whose rows cannot share the main-manifest CAS. -### BLOCKER-14 — stock RC.1 cannot safely reclaim MemWAL state +### BLOCKER-14 — stock RC.1 cannot safely reclaim MemWAL state (B2b) -**Affected:** [RFC-026 §4.5](../rfcs/0026-memwal-streaming-ingest.md#45-reclamation-decision-own-the-missing-lance-primitive), +**Affected:** [RFC-026 §4.5.2](../rfcs/0026-memwal-streaming-ingest.md#452-b2b-managed-reclamation-own-the-missing-lance-primitive), [§6](../rfcs/0026-memwal-streaming-ingest.md#6-fold-protocol), and -[§12.4](../rfcs/0026-memwal-streaming-ingest.md#124-phase-b2--public-strict-activation-gates) +[§12.6](../rfcs/0026-memwal-streaming-ingest.md#126-common-public-and-b2b-managed-reclamation-activation-gates) -**Status:** Design-dispositioned on 2026-07-19; implementation-blocking for -public B2 activation. This blocker does not reopen the MemWAL substrate choice -or require waiting for an upstream release. +**Status:** Design-dispositioned on 2026-07-19 as the optional B2b +managed-reclamation route; implementation-blocking only if that profile is +pursued. Gate R0's 2026-07-20 result remains a historical no-go for a finite or +otherwise bounded retain-all claim on stock RC.1. It does not reject the +selected unbounded B2a profile, whose legal near-cap B1 closure was repaired on +2026-07-21. Neither result reopens the MemWAL substrate choice or activates a +public surface. Two production-neutral guards pin the current RC.1 boundary: @@ -942,8 +980,32 @@ collector; they do not prove a safe cleanup implementation. WAL positions also have no per-generation durable range in RC.1, so prefix deletion is limited to a quiescent whole cut rather than inferred from mutable cursor hints. -**Required disposition:** author the capability inside Lance as an opaque -inspect/plan/execute operation with durable attempt/receipt recovery. Its plan +Gate R0 deliberately tested whether **finite-lifetime, bounded** retain-all +could avoid this reclamation dependency. Its revision-pinned source audit and +evidence harness returned **no-go for that bounded claim**. A flush chooses a +fresh random generation subtree and can leave partial output before the +shard-manifest CAS. Reopen can retry materialization at another root, while +durable shard state has no attempt ID/counter/reservation/receipt persisted and +enforced across cold opens. RC.1 exposes no admission-grade reserve-first +estimator or complete receipt for the +data/deletion/bloom/PK/blob/transaction/manifest/local-staging/multipart +output. The retained one/four/eight-fold current-object census and +referenced-cut retry prove useful invariants but cannot establish provider +versions, incomplete multipart, or billed bytes. + +The selected B2a profile accepts that limitation explicitly. It retains all raw +`_mem_wal` state, performs no OmniGraph WAL deletion, sets no retained-byte, +object-count, file-count, or history quota, and treats provider exhaustion as a +loud failure that may stop admission, fold, or recovery progress without +weakening acknowledgement durability or manifest-only visibility. Because it +makes no finite-storage claim, B2a does not need a test-only materialization +attempt ledger, a complete reserve-first physical-output envelope, or a +`GraphHistoryBudget`. Any later production state still binds to the existing +manifest/recovery authority. + +**Optional future B2b disposition:** if managed reclamation is pursued, author +the capability inside Lance as an opaque inspect/plan/execute operation with +durable attempt/receipt recovery. Its plan binds exact shard-manifest history, whole-cut cursor predicates, base merge/index state, writer epoch, graph-approved generation, and object/byte budgets. Execution revalidates those witnesses, advances both shard-manifest @@ -961,11 +1023,12 @@ materialization attempts and control headroom, and an epoch recheck after every successful WAL atomic PUT. One bootstrap-selected reserve-first ledger owns that budget per physical binding and settles only from exact inventory; versioned/soft-delete/Object-Lock stores are refused unless every retained byte -is accounted and eligible versions are permanently removed. OmniGraph opens the -upstream PR but pins the exact reviewed fork commit immediately, so release -timing is not the calendar. +is accounted and eligible versions are permanently removed. This is +Lance-owned future work, not an OmniGraph raw-path collector and not an +immediate B2a dependency. -Public B2 remains closed until the positive local/RustFS stale-plan, +That managed-reclamation profile remains closed until the positive +local/RustFS stale-plan, prune-CAS, partial-delete, lost-result/receipt, authoritative-checkpoint-plus-successor-chain orphan, unknown-state, genesis/first-publication, claim/reclaim receipt-expiry, @@ -974,26 +1037,12 @@ The source-derived reservation must be enforced before admission; a separate physical-expansion instrument validates it but cannot create a hard bound. Runtime never releases budget merely because a fold completed. -The per-binding ledger bounds `_mem_wal` only. B2-0 separately specifies one -manifest-authoritative, graph-global `GraphHistoryBudget` checked by every -manifest publisher, including ordinary table-disjoint writers. Each operation -reserves its exact logical publication and a source-bounded physical-growth -envelope before effect; durable recovery charges settle exactly once. Initial -B2 holds the gate continuously through physical effects, manifest CAS, and -charge finalization rather than composing speculative pending revisions. Dynamic -per-stream closure reserves remain independently available for block/correction, -quiesce, and `SEALED` rebuild even when ordinary work reaches the shared floor. -The universal order is sorted relevant stream admission → graph history → -schema → main → stream token → sorted tables; late global discovery -releases all of those gates and restarts from the root barrier. - -This closes only `_mem_wal` retention. B2 still refuses in-place maintenance on -an enrolled table, and base/token/shared-manifest histories continue to grow. -The graph-global budget therefore makes B2 a finite-lifetime root profile: -ordinary work returns `GraphRebuildRequired` at the aggregate closure floor, -while each enrolled stream keeps its reserved fold/correction/quiesce path to -`SEALED` export/rebuild. Indefinite in-place service remains a later -maintenance/retention strand. +The earlier bounded-profile sketch paired that per-binding ledger with a +manifest-authoritative graph-global `GraphHistoryBudget`, because reclaiming +`_mem_wal` alone would not bound base, token, and shared-manifest histories. +That sketch remains design history for a possible B2b amendment. It is not part +of the selected unbounded B2a profile, is not a common public-stream contract, +and is not on the immediate implementation path. ### TIGHTENING-01 — symmetric Lance conflicts are not obviously an activation gate @@ -1169,19 +1218,22 @@ The review does not require all RFCs to land together. A safe order is: evidence-backed operational contract passes the full physical-I/O boundary, then require physical protection of checkpoints/live graph branches and strict-rebuild implementation evidence; -6. retain RFC-026 Gate E0's green exact-version evidence, Phase A's implemented - bounded enrollment foundation, and private B1's green schema-v8/config-v2 - one-generation put/watcher/replay/recovery-v11 strict fold evidence. Retain - B2-0's specified explicit enrollment, compare-and-chain token, attribution, - state-v2 lifecycle revisions/management receipts, bounded correction, - reclamation/enforced-watermark, and graph-global manifest-history budget - contracts. First - author and pin the reviewed Lance durable reclamation, bounded-history, - enforced-growth-reservation, and post-success-fence patch and pass its - local/RustFS evidence; then implement schema-v9/config-v3/recovery-v12 plus - reserve-first graph-history accounting privately. Expose B2 only after every - crash, enforced-bound, cross-version, authorization, and product-parity gate - passes. Retain the public exact enrollment receipt +6. retain RFC-026 Gate E0's green exact-version evidence and Phase A's + implemented bounded enrollment foundation. Preserve private B1's + schema-v8/config-v2 put/watcher/post-fence/replay/recovery-v11 evidence and + its repaired legal near-cap closure: logical-slice accounting plus dense + scanner-output copies now fold and publish the 8,192-row high-entropy cell, + with the isolated fold RSS delta below the remeasurement tripwire. Proceed + with the selected unbounded B2a retain-all profile: never delete raw + `_mem_wal` state, impose no retained-byte/object/file/history quota, and + surface provider exhaustion loudly. Do not add a materialization-attempt + ledger, reserve-first complete physical-output envelope, storage watermark, + or `GraphHistoryBudget` to that immediate path. Implement and prove the + common explicit-enrollment, compare-and-chain-token, trusted-attribution, + revisioned-lifecycle/management-receipt, correction, authorization, and + SDK/CLI/HTTP/OpenAPI parity contracts before exposing B2. Keep B2b + Lance-owned managed reclamation as an optional future route with its own + evidence and format amendment. Retain the public exact enrollment receipt and cross-process admission seal as the gate for broader topology; 7. keep RFC-027 in research until deletion-delta discovery passes its stated correctness and flat-cost gates. diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 52d0182c..c2b63c15 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -25,9 +25,9 @@ The engine's `tests/` is the principal coverage surface; most graph-shaped behav | `writes.rs` | Direct-publish writes: cancellation, RFC-022 non-strict full-attempt reprepare from fresh branch authority, strict stale-write conflicts, multi-statement atomicity, MR-794 staged-write rewire (D₂ rejection, insert+update coalesce, multi-append coalesce, partial-failure recovery, load RI/cardinality recovery); RFC-023 pins the inclusive 8,192-row keyed input ceiling, the same exact/+1 boundary on streamed mutation-update matches, no-effect state for both refusals, and oversized stored-Blob rejection before payload read. Crate-internal pending-scan cells pin inclusive/+1 32 MiB accounting plus pending-key shadow-before-charge. The lance#7444 row-id-overlap regression (`filtered_read_after_merge_update_and_delete_keeps_row_ids_consistent` — merge-load → same-key merge-load → delete → keyed point lookup, green only under the vendored lance-table patch — plus its append-only control) | | `src/table_store/staged_tests.rs` | Crate-internal staged primitives. RFC-023 pins one exact target preflight for general StrictInsert, durable v1 mint/commit/reopen/history persistence, exact-`id` filter emission, typed `KeyConflict`, and missing/wrong PK refusal. `all_new_upsert_certifies_insert_absence_and_persists_it_in_history` proves an all-new completed Upsert receives the optional certificate, a mixed/update Upsert does not, unrelated transaction properties survive, and UUID rebinding does not erase it. Proven-insert cells show the opaque path performs zero strict preflights; stages with `InsertBuilder` but commits the full pure-insert `Update` shape (exact parent and `id` filter, `RewriteRows`, no updates/removals, full nested schema preorder, physical rows); persists/re-admits its own output for proof composition; leaves new fragments outside old index coverage; and fails same-key races loudly in proven/proven and proven/general orders. The in-source `exec/merge.rs` certificate unit table rejects missing/unknown properties, wrong parent/filter/full-preorder/mode/offsets, rewrite/removal shapes, missing `physical_rows`, and Append. Source-interval cells pin exact selection, lazy retained-parent splitting, coalescing, and pinned Lance's approximate raw-emission boundary while every normalized/writer chunk remains hard-capped. Generic `stage_append`/`stage_merge_insert` remain primitive tests only. The file also owns index staging and `commit_staged{,_exact}` | | `forbidden_apis.rs` | Defense-in-depth syntax-tree/source guard over the whole engine. The primary boundary is Rust visibility: raw storage/coordinator/handle-cache modules are crate-private; public `Snapshot::open` returns `SnapshotTable`; and `SnapshotScanner` executes reads without exposing Lance's raw scanner or physical plan. The guard pins those visibility/return-type boundaries, classifies public async inherent `Omnigraph` methods plus loader conveniences, classifies every crate-visible async method on `GraphCoordinator` / `ManifestCoordinator`, and exact-counts registered method/UFCS durable-call shapes including recovery. RFC-023 rejects production graph call sites of generic `stage_append{,_stream}` and `proven_insert_capability_has_one_production_mint_site` pins `ProvenInsertChunk::from_verified_history` to the complete-history classifier in `exec/merge.rs`, preventing the no-preflight capability from becoming a reusable bypass. At the RFC-026 Phase-A checkpoint the guard registered only the exact v10 enrollment gateway and feature-gated test seam, counted its sidecar/index/shard durability primitives, and kept every row-put/ack/fold surface absent; the Phase-B1 owner below now exact-counts only the approved crate-private put/fold durable-call sites while retaining the absence of public schema, SDK, HTTP, CLI, and OpenAPI side doors. It also counts selected raw `SnapshotHandle` / Dataset shapes, rejects renamed-owner/macro/include/path-lookalike forms, skips structurally test-only code, and pins retired escape hatches absent. This is intentionally not a Rust macro-expander or general alias analysis; `// forbidden-api-allow: ` exempts reviewed inline-Lance lines only | -| `lance_surface_guards.rs` | Pins the Lance API surfaces omnigraph depends on (named runtime + compile-only guards; see [lance.md](lance.md)) — the first smoke check on any Lance version bump. `cached_and_zero_cache_sessions_share_store_registry_not_metadata_cache` proves a cached data Session and zero-cache control Session reuse one live `ObjectStoreRegistry` client while their metadata caches remain isolated. `_compile_uncommitted_full_table_vector_index_shape` pins the public `IndexMetadata` shape suitable for `Operation::CreateIndex`; `compact_files_succeeds_on_blob_columns` pins blob-v2 compaction; Guard 9 pins clone-only branch reclaim semantics. RFC-023's `unenforced_pk_filter_shape_is_route_dependent` explicitly forces v2 versus indexed routes and pins the `Some(populated)` / `Some(empty)` / `None` key-filter shapes; `unenforced_pk_conflict_matrix_is_directional` pins the directional filtered/unfiltered and filtered/Append matrix. RFC-024's compile guard pins the public `BranchIdentifier` + current table version + current `Transaction.uuid` + `ManifestLocation.e_tag` current-HEAD witness; the local/shared-`Session` guard proves unchanged-reopen stability, ordinary-commit movement, and same-version ABA, while RustFS covers object-store ABA. RFC-025 adds exact main/named-branch tag-target, sparse cleanup pin/unpin, and branch-tree-deletion guards. RFC-026 pins doc-hidden `has_successor_version`, initializer/readback/shard-writer/durability/fencing, flush/drain, replay watermark, scanner, and merged-generation shapes; runtime Gate E0 classification belongs to `memwal_enrollment_gate.rs`, while v7 Phase-A and v8 B1 publication/recovery belong to the manifest/failpoint suites. B2-0 adds `cleanup_old_versions_does_not_reclaim_mem_wal_objects` and `mem_wal_deleted_fence_slot_allows_stale_writer_success_on_pinned_lance`: the first proves generic cleanup leaves the present MemWAL fixture unchanged and the second proves deleting the successor's empty fence sentinel is unsafe. The pinned source audit, not those two tests alone, establishes that stock RC.1 exposes no owned MemWAL reclamation API. The RC.1 compiler guard pins the five surveyed public Lance virtual system-column constants to early `.pg` rejection. These guards prove substrate shapes/tokens and negative ownership boundaries; they do not by themselves prove heads/checkpoint activation, the v8 publisher, or a safe reclamation implementation | +| `lance_surface_guards.rs` | Pins the Lance API surfaces omnigraph depends on (named runtime + compile-only guards; see [lance.md](lance.md)) — the first smoke check on any Lance version bump. `cached_and_zero_cache_sessions_share_store_registry_not_metadata_cache` proves a cached data Session and zero-cache control Session reuse one live `ObjectStoreRegistry` client while their metadata caches remain isolated. `_compile_uncommitted_full_table_vector_index_shape` pins the public `IndexMetadata` shape suitable for `Operation::CreateIndex`; `compact_files_succeeds_on_blob_columns` pins blob-v2 compaction; Guard 9 pins clone-only branch reclaim semantics. RFC-023's `unenforced_pk_filter_shape_is_route_dependent` explicitly forces v2 versus indexed routes and pins the `Some(populated)` / `Some(empty)` / `None` key-filter shapes; `unenforced_pk_conflict_matrix_is_directional` pins the directional filtered/unfiltered and filtered/Append matrix. RFC-024's compile guard pins the public `BranchIdentifier` + current table version + current `Transaction.uuid` + `ManifestLocation.e_tag` current-HEAD witness; the local/shared-`Session` guard proves unchanged-reopen stability, ordinary-commit movement, and same-version ABA, while RustFS covers object-store ABA. RFC-025 adds exact main/named-branch tag-target, sparse cleanup pin/unpin, and branch-tree-deletion guards. RFC-026 pins doc-hidden `has_successor_version`, initializer/readback/shard-writer/durability/fencing, flush/drain, replay watermark, scanner, and merged-generation shapes; runtime Gate E0 classification belongs to `memwal_enrollment_gate.rs`, while v7 Phase-A and v8 B1 publication/recovery belong to the manifest/failpoint suites. B2b adds `cleanup_old_versions_does_not_reclaim_mem_wal_objects` and `mem_wal_deleted_fence_slot_allows_stale_writer_success_on_pinned_lance`: the first proves generic cleanup leaves the present MemWAL fixture unchanged and the second proves deleting the successor's empty fence sentinel is unsafe. The pinned source audit, not those two tests alone, establishes that stock RC.1 exposes no owned MemWAL reclamation API. The RC.1 compiler guard pins the five surveyed public Lance virtual system-column constants to early `.pg` rejection. These guards prove substrate shapes/tokens and negative ownership boundaries; they do not by themselves prove heads/checkpoint activation, the v8 publisher, or a safe reclamation implementation | | `memwal_enrollment_gate.rs` | RFC-026's green production-neutral Gate E0 harness, isolated from the production manifest and graph writer. Fourteen substantive local cells plus one explicit unconfigured-S3 skip cover exact no-effect / `N + 1` index / pre-minted empty-shard classification, buried-effect refusal, marker survival, strict inventory/error handling, and the broad fail-closed matrix. The rejected first instrument used `checkout_latest` plus `IOTracker`, which missed local `read_dir`. The accepted exact-version classifier pins doc-hidden `has_successor_version`; its `AttemptTracker` records failed/`NotFound` attempts before forwarding and proves the identical complete six-attempt shape at baseline versions 8/80: four successful manifest HEADs, one `NotFound` manifest HEAD, one successful manifest GET, zero lists. A Unix execute-only `_versions` tripwire proves exact probing works when latest enumeration fails and an unreadable exact HEAD errors. The configured RustFS exact cell passes non-vacuously with the same zero-list shape and owns the positive lost-result/index/empty-shard/reopen sequence plus foreign shard, malformed/loose root, durable WAL, persisted cursor, and corrupt-manifest negatives. S3 ABA remains in `lance_surface_guards.rs`; CI rejects skipped E0/ABA cells. This file never mutates production manifest/schema state or deletes ambiguous artifacts; Phase A consumes its classifier through the private adapter | -| `memwal_stream_cost.rs` | Feature-gated RFC-026 B1 decision instrument. It separately measures warm already-claimed durability acknowledgement at compacted graph-history endpoints, cold claim/reopen/replay against retained WAL depth, one selected generation's fold scan, retained already-merged shard metadata, and the shared uncompacted graph-manifest fold/publisher term. It also pins both one-batch and maximally fragmented legal-generation no-auto-roll estimates and records a paired one-batch whole-process peak-RSS delta. The exact local and configured-RustFS evidence is recorded in “RFC-026 Phase B1 coverage ownership” below; debug timings are not a product latency claim. | +| `memwal_stream_cost.rs` | Feature-gated RFC-026 B1 and Gate-R0 decision instrument. It separately measures warm already-claimed durability acknowledgement, cold replay, selected-generation fold scanning, retained merged metadata, the uncompacted graph-manifest term, legal no-roll estimates, and paired peak RSS. Gate R0 adds a revision-pinned source-audit tripwire, strict current-object classification/reference census, listed path/class/size retain-all comparisons at one/four/eight folds, referenced-cut retry reuse, and deterministic high-entropy near-cap local/configured-RustFS cells. The near-cap cell now positively proves that an admitted 8,192-row/near-32-MiB generation acknowledges without graph visibility, then folds and publishes exactly once after logical-slice charging plus dense per-scanner-batch take. The isolated fold RSS delta measured 284,934,144 bytes (about 272 MiB), below a 384-MiB remeasurement tripwire; that tripwire is not a runtime allocator limit. Current LIST evidence explicitly excludes multipart/provider versions/local staging/billed bytes and is retention evidence, not a quota. A green test proves private closure and retain-all behavior; it does not activate a public API. | | `durable_head_lookup_cost.rs` | RFC-024 Gate A decision instrument, isolated from the production manifest schema/publisher. At fixed catalog width 10 it runs the full absent/reconciled/one-uncovered/eight-uncovered/reconciled-after-tail matrix over compacted and uncompacted histories, with cold-open and warm-repeat measurements on local FS and bucket-gated S3/RustFS. Default depths are 20/80; the ignored decision-scale cell runs 10/100/1,000. Correct exact heads, flat indexed `rows_scanned`/range work, an index-absent growing negative control, and observable bounded tails all pass; after the eight-fragment tail, `optimize_indices` returns coverage to zero uncovered and representative `rows_scanned`/range work from 27→10 / 17→10. The test deliberately pins the no-go: uncompacted RustFS cold object reads/bytes and compacted byte terms grow, while RC.1 also crosses a bounded one-operation boundary by 1,000 commits, so RFC-024 remains research-blocked. `rows_scanned` is an RC.1 debug proxy, not a universal decoded-row counter. Object-store wrapper bytes and Lance execution-summary bytes are separate fixture-owned metrics and are not additive | | `checkpoint_retention_cost.rs` | RFC-025 Gate 0 decision instrument, isolated from the production manifest schema. It models three live checkpoints at catalog width 10 and measures complete list, exact show, and cleanup-root authority reads across absent/reconciled/eight-uncovered index states, compacted/uncompacted layouts, and cold/warm access. It also owns the reference V1 name-normalization matrix. Default local depths 20/80 pass the checked-in **no-go-preservation** assertions; the RC.1 ignored 10/100/1,000 run shows reconciled uncompacted work and the bounded tail flat, but rejects the current format shape after compaction: list/cleanup scan bytes grow 17,012→38,000 cold and 12,336→15,064 warm; show grows 29,348→53,064 and 24,672→30,128; scan operations add one at 1,000. The S3/RustFS cell is bucket-gated and was not run for this decision. The result keeps RFC-025 research-blocked; current v8 adds no checkpoint state | | `warm_read_cost.rs` | Cost-budget tests for the warm read/control path (query-latency work), measured at the object-store boundary with Lance `IOTracker` (the LanceDB IO-counted pattern): a warm same-branch read does 0 manifest opens, 1 version probe, validates the schema once (Fix 1 / finding A / Fix 2 at commit-history depth); a cold other-branch resolution derives snapshot state and lineage from one coherent manifest open/scan; native branch create and create-from each use one post-gate open/scan, while delete uses one target capture plus one native-ref opener and only one row scan; stale same-branch reads perform exactly 2 probes and refresh manifest-only; recreated non-main branches with the same Lance version refresh by incarnation; recreated branch-owned table handles are distinguished by table e_tag or refresh-time cache clearing; recreated traversal topology is protected by per-edge-table e_tag in the graph-index cache key or refresh-time cache clearing; a warm *repeat* read does 0 table opens via the held-handle cache and a write re-opens only the changed table at its new version/e_tag (Fix 3/6A). Also the CSR topology-build cost guards: `fresh_branch_traversal_reuses_main_graph_index` (A1 — a lazy-fork branch reuses main's cached CSR index, 0 rebuilds via `graph_build_count`) and `single_edge_query_builds_only_referenced_edge` (A2 — a one-edge query builds only that edge via `graph_edges_built`); both force CSR via the scoped `with_traversal_mode` seam, so they need no `#[serial]`. See "Cost-budget tests" below. | @@ -62,7 +62,7 @@ The engine's `tests/` is the principal coverage surface; most graph-shaped behav | `recovery.rs` | Open-time recovery sweep — identity-bearing schema-v9 envelopes for Mutation/Load exact transaction identity, BranchMerge exact chains/ref-only effects + complete delta, SchemaApply exact overwrite/create ownership + schema promotion, EnsureIndices exact mixed-index transactions/authority/lineage/delta/first-touch identity, and Optimize's bounded maintenance payload, plus RFC-026's dedicated schema-v10 roll-forward-only StreamEnrollment envelope and schema-v11 StreamFold envelope binding the exact config-v2 profile, shard cut, merged generation, Lance transaction, and fixed lineage. Explicit refusal replaces alias inference for identity-less input; restartable compensation, fixed logical/rollback IDs, branch-token comparison, fresh under-gate reread/reparse, all-or-nothing roll-forward/rollback/refusal, recovery audit, and read-only guards remain pinned. RFC-023 additionally asserts that restoring a feature-branch sidecar leaves the selected feature ref with exact-`id` PK metadata | | `composite_flow.rs` | Compositional/narrative end-to-end stories — multi-step flows that compose mechanics covered by other test files. Catches integration regressions where individual operations all pass their unit tests but their composition breaks (sequential merges, post-merge main writes, time-travel through merge DAG, reopen consistency over multi-merge histories, post-optimize and post-cleanup strict writes). | -RFC-026 reclamation qualification: the two B2-0 runtime guards do not prove +RFC-026 reclamation qualification: the two B2b runtime guards do not prove that every possible safe RC.1 API is absent. The source audit establishes that surface fact; the tests prove the narrower generic-cleanup non-ownership and deleted-successor-sentinel fencing hazard. @@ -268,29 +268,90 @@ beside an unmerged generation, wide/derived embedding expansion beyond the post-fold byte limit, active-state authoritative cursor validation, and the deliberate absence of a B1 correction generation. +### RFC-026 Gate R0 coverage ownership (retain-all closure) + +Gate R0 extends the existing `memwal_stream_cost.rs` owner rather than creating +a parallel streaming silo. Its revision-pinned source audit records that stock +RC.1 has neither an admission-grade reserve-first complete physical-output +envelope/receipt nor a MemWAL-persisted/enforced durable cross-open cap/receipt +for randomized generation-materialization attempts. Those facts prevent a +finite storage-bound claim, but they are not blockers for the selected +unbounded retain-all profile. A lockfile pin change still fails the tripwire +until the source audit is refreshed. + +The strict current-object census recursively lists the graph prefix, classifies +WAL, shard-manifest version/hint, generation data/manifest/transaction/deletion, +PK, Bloom, and maintained-index objects, and compares generation roots with the +latest decoded shard manifest. WAL positions and manifest versions must be +canonical positive 1-based bit-reversed names; generation roots must use +canonical positive decimal generations. Decoded shard identity must match its +directory, generation/path authority must agree and be unique, and every +referenced root must be currently listed. Unknown, malformed, missing, or +duplicate authority fails closed. Except for the mutable best-effort shard +version hint, every earlier path must retain the same class and size. This +proves only current listed objects; incomplete multipart uploads, superseded +provider versions, delete markers, local staging files, and billed storage +remain outside the observation surface. + +The success sweep records 1/4/8 referenced roots and approximately 37.4 / 150.6 +/ 302.3 thousand local currently listed immutable bytes. The small exact total +varies with generated metadata, so root/reference/path-class-size +retention/monotonicity—not one byte total or content identity—is the assertion. +The referenced-cut retry cell fails after drain but before the fold sidecar, +then proves retry reuses the identical root and publishes the row once. The +legal high-entropy near-cap cell avoids the old compression-friendly repeated +payload and now proves the repaired closure path: acknowledgement retains the +WAL while manifest/table versions remain unchanged; fold charges logical +slices against the 32-MiB cap, takes each scanner emission into dense owned +arrays, materializes one referenced generation, publishes exactly one manifest +and table version, verifies all 8,192 rows plus sampled payloads, and retires the +recovery sidecar. The separate subprocess instrument measured a 284,934,144-byte +(about 272 MiB) isolated fold RSS delta under one exclusive fold. Its 384-MiB +threshold is a CI remeasurement tripwire for this implementation shape, not a +runtime memory reservation or allocator cap. + +Run the production-neutral Gate-R0 cells with: + +```bash +cargo test -p omnigraph-engine --features failpoints --test memwal_stream_cost gate_r0_ -- --nocapture +``` + +The configured-RustFS near-cap twin skips explicitly when +`OMNIGRAPH_S3_TEST_BUCKET` is absent. CI requires its measured evidence line and +positive test polarity, so an explicit skip is a failure there. The fragmented +8,192-single-row shape remains an accounting/RSS fixture rather than a claimed +configured-object-store physical run. A green Gate-R0 run means the private +8,192-row/32-MiB closure and retain-all observations reproduced; it does not +activate a format or public API. + Run the private integration owners with: ```bash cargo test -p omnigraph-engine --features failpoints --test memwal_stream cargo test -p omnigraph-engine --features failpoints --test memwal_stream_cost -cargo test -p omnigraph-engine --features failpoints --test failpoints stream_fold +cargo test -p omnigraph-engine --features failpoints --test memwal_stream_cost widest_legal_generation_records_no_roll_estimates_and_peak_rss -- --exact --nocapture ``` -B1 does not add parser/server/CLI tests because it has no public surface. B2-0 -now specifies the missing contracts without activating them. B2 implementation +B1 does not add parser/server/CLI tests because it has no public surface. The +common product contract inventory specifies the missing contracts without +activating them. Any public implementation must extend the existing compiler, server/OpenAPI, CLI parity, Cedar, shutdown, -audit, retention/GC, and genuine rebuild suites together with explicit +audit, no-raw-GC, provider-failure, and genuine rebuild suites together with explicit enrollment, persistent revisioned status/quiesce/resume/abort-drain, bounded -terminal management receipts, strict correction, the enforced retained-storage -admission watermark, and graph-global manifest-history admission controls; -those tests are a B2 gate, not incidental B1 scope. The lifecycle +terminal management receipts, compare-and-chain tokens, attribution, and strict +correction. The selected retain-all profile has no byte/object/file/history +quota; its tests must instead prove that provider exhaustion is loud and cannot +drop an acknowledgement or bypass recovery/manifest visibility. Storage +watermarks and graph-history admission controls belong only to a future B2b +bounded/managed profile. Those tests are a product gate, not incidental B1 +scope. The lifecycle matrix includes `quiesce -> create named branch -> resume`: bounded resume must recheck branch topology under the closed gates and remain `SEALED`, while a compatible main-only resume advances the epoch and opens. -### RFC-026 Phase B2-0 coverage ownership (specified, inactive) +### RFC-026 Phase B2b coverage ownership (specified, inactive) -B2-0 adds evidence at the boundary where the design depends on Lance, while +B2b adds evidence at the boundary where the design depends on Lance, while leaving all production stream surfaces absent. The two checked-in RC.1 guards live in `lance_surface_guards.rs` because they characterize substrate behavior: @@ -342,8 +403,11 @@ refused unless every retained version/delete marker/locked byte is exactly accounted and eligible versions are permanently removed. A separate local/RustFS matrix validates that bound across schemas, fragmentation, crashes, and retries; measurement alone does not establish it. Until all of that is -green, schema v9/config-v3/state-v2/recovery-v12 and every public B2 route -remain inactive. A separate finite-lifetime matrix initializes and validates the +green, no B2b bounded/managed-reclamation route is active. This future matrix +does not gate the selected unbounded retain-all profile; that profile remains +private because the common token, attribution, lifecycle, correction, +authorization, and product-parity contracts above are still inactive. A +separate bounded-profile matrix initializes and validates the manifest-authoritative graph-global `GraphHistoryBudget`, then charges every manifest-writer class and its pending recovery sidecars through reserve, effect, lost publication acknowledgement, exact settlement, and effect-free release. @@ -389,7 +453,13 @@ sidecar arm or graph movement. - **Names are compile-checked.** Every failpoint name is a `pub const` in `omnigraph::failpoints::names` (engine) / `omnigraph_cluster::failpoints::names` (cluster). Call sites and tests reference the constant, never a bare literal — a typo is a compile error, not a silently-never-firing point. Add a new failpoint by adding its const first. - Call sites are inserted at sensitive transaction boundaries (branch create, graph publish commit, the recovery sweep's classify→roll-forward-publish window, cluster apply's payload→state-write window, etc.). - **Serialize and rendezvous, never sleep.** The `fail` registry is process-global, so every failpoint test carries `#[serial]` (`serial_test`). For concurrent tests, use `helpers::failpoint::Rendezvous` (`tests/helpers/failpoint.rs`): `park_first(name)` parks the first thread to hit the point until `release()`, and `wait_until_reached().await` blocks on that condition (it doubles as a fired-assertion). Do not coordinate threads with fixed `sleep`s. -- Activated tests: `crates/omnigraph/tests/failpoints.rs` and `crates/omnigraph-cluster/tests/failpoints.rs` (integration binaries, never in-source — the fail registry is process-global). Run with `cargo test -p omnigraph-engine --features failpoints --test failpoints` / `cargo test -p omnigraph-cluster --features failpoints --test failpoints`. +- Activated tests: `crates/omnigraph/tests/failpoints.rs`, + `crates/omnigraph/tests/memwal_stream_cost.rs`, and + `crates/omnigraph-cluster/tests/failpoints.rs` (integration binaries, never + in-source — the fail registry is process-global). Run the main suites with + `cargo test -p omnigraph-engine --features failpoints --test failpoints` / + `cargo test -p omnigraph-cluster --features failpoints --test failpoints`; + Gate R0's exact command is documented above. ## RustFS / S3 integration @@ -401,6 +471,7 @@ CI runs these S3-backed **correctness** tests against a containerized RustFS ser - `cargo test -p omnigraph-cluster --test s3_cluster` (full control-plane lifecycle on the bucket) - `cargo test -p omnigraph-cli --test system_local local_cli_s3_end_to_end_init_load_read_flow` - `cargo test -p omnigraph-engine --features failpoints --test failpoints s3_` (recovery-sidecar lifecycle on a real bucket) +- `cargo test -p omnigraph-engine --features failpoints --test memwal_stream_cost gate_r0_widest_generation_closes_and_records_retain_all_growth_on_configured_rustfs -- --exact --nocapture` (RFC-026's configured-object-store 8,192-row/near-32-MiB closure twin; the workflow requires its measured `rustfs` line and positive-test polarity, so an explicit skip is a failure) Locally, set `OMNIGRAPH_S3_TEST_BUCKET` (and the usual `AWS_*` vars including `AWS_ENDPOINT_URL_S3` for non-AWS) before running. Without those, S3 tests skip gracefully. @@ -639,6 +710,7 @@ Correctness bugs fail loudly in tests; cost-scaling bugs pass every test and deg - **Keep decision instruments honest when the answer is no.** RFC-024's `durable_head_lookup_cost.rs` attaches tracking before the cold dataset load through `open_tracked_lance_dataset`, then reports object-store wrapper I/O separately from Lance execution-summary I/O. Its reconciled BTREE row/range curve is flat, but its required RustFS cold-open and compacted-byte curves grow; those red design facts are asserted as the current result rather than erased because some counters pass. Run the default local 20/80 matrix with `cargo test -p omnigraph-engine --test durable_head_lookup_cost local_durable_head_lookup_matrix_is_correct_and_observable -- --exact --nocapture`; run the ignored 10/100/1,000 local matrix with `cargo test -p omnigraph-engine --test durable_head_lookup_cost local_durable_head_lookup_matrix_at_one_thousand_commits -- --ignored --exact --nocapture`. The bucket-gated S3 command is in the RustFS section above and remains on demand. - **Apply the same rule to RFC-025.** `checkpoint_retention_cost.rs` keeps live checkpoint count and catalog width fixed while unrelated journal history grows, and counts complete list/show/cleanup-root authority reads. The uncompacted reconciled counters and bounded tail are flat; compacted scan bytes and the 1,000-commit operation boundary are not, so the assertions preserve a no-go. Run the default local matrix with `cargo test -p omnigraph-engine --test checkpoint_retention_cost local_checkpoint_retention_matrix_is_exact_and_records_the_current_no_go -- --exact --nocapture`; run the ignored decision scale with `cargo test -p omnigraph-engine --test checkpoint_retention_cost local_checkpoint_retention_matrix_at_one_thousand_commits -- --ignored --exact --nocapture`. A green test means the known result was reproduced, not that RFC-025 passed Gate 0. - **Keep RFC-026 Gate E0 reproducible.** The first `checkout_latest`/`IOTracker` instrument was false-green because local `read_dir` escaped tracking; it is not acceptance evidence. The green harness uses the public but guide-hidden `Dataset::has_successor_version` from freshly ABA-verified exact `N`, probes only `N + 1`, then uses exact `N + 1` to reject buried `N + 2`. `AttemptTracker` records before forwarding, including failed/`NotFound` HEADs, and versions 8/80 must retain the identical four-success-HEAD + one-NotFound-HEAD + one-success-GET shape with zero lists. The Unix execute-only `_versions` tripwire must keep exact probing green while latest enumeration fails, and an unreadable exact HEAD must error. Run the 14-substantive-cell local file with `cargo test -p omnigraph-engine --test memwal_enrollment_gate -- --nocapture`; its fifteenth bucket-gated cell logs an explicit skip when unconfigured. Run the exact configured RustFS command above for its positive plus listing-dependent negative matrix. Green E0 authorized only Phase A; Phase A has now activated v7 foundation state, but E0 never authorizes row admission, acknowledgement, or fold. +- **Preserve RFC-026 Gate R0's closure and retain-all evidence.** Its current-object census is useful only inside its stated observation boundary; never label LIST totals as provider-retained or billed bytes, and never turn them into a quota claim. The source audit remains pinned to the exact Lance lockfile revision. The high-entropy cell must continue to admit a legal 8,192-row/near-32-MiB generation, keep acknowledgement graph-invisible, close through logical-slice charge plus dense per-scanner-batch take, publish exactly once, and preserve every previously listed immutable path/class/size. Use `cargo test -p omnigraph-engine --features failpoints --test memwal_stream_cost gate_r0_ -- --nocapture`. Also run `widest_legal_generation_records_no_roll_estimates_and_peak_rss` exactly: its observed fold delta was 284,934,144 bytes (about 272 MiB), and 384 MiB is a remeasurement tripwire for the single-exclusive-fold implementation shape, not a runtime hard allocator limit. A green run proves the private evidence only; public product contracts remain inactive. - **Count on the handle that does the reads, not just the one a measured op opens.** Lance's IO-counted tests attach the `IOTracker` to the (warm, cached) dataset and read `incremental_stats()` per request — the tracker MUST be on the handle performing the reads, or warm-handle reads escape. A per-op tracker installed at measure time cannot see reads on a long-lived handle opened earlier (the warm coordinator's `__manifest` handle, reused across writes), so such reads were silently undercounted. Wrap a depth-swept body in `cost_harness` so the manifest tracker is installed before the graph opens and `manifest_reads` is **ground truth** (handle-age-irrelevant). The `version_probes` counter is the freshness-probe *call* count; ground truth additionally reveals that a write's probe does ~3 object-store RPCs (a read's probe is a 0-IO cache hit). `manifest_reads_capture_warm_probe` is the guard that this stays true. - This is the testing companion to invariant 15 in [docs/dev/invariants.md](invariants.md) (hot-path cost is bounded by work, not history). diff --git a/docs/dev/versioning.md b/docs/dev/versioning.md index b9c86826..4da9a874 100644 --- a/docs/dev/versioning.md +++ b/docs/dev/versioning.md @@ -47,9 +47,10 @@ filter-bearing adapter. Internal schema **v8 is the currently served format** and maps to OmniGraph **0.12.x**. It preserves the v5 identity, v6 key-fencing, and v7 stream- lifecycle contracts, then activates RFC-026 Phase B1's private data-bearing -MemWAL core: exact stream-config v2, one bounded generation, watcher-only -durability acknowledgement, conservative replay, explicit seal/drain, and a -dedicated schema-v11 `StreamFold` recovery envelope. The fold publishes its +MemWAL core: exact stream-config v2, one admission-bounded no-roll generation, +watcher success plus the same writer's post-durability epoch check before clean +acknowledgement, conservative replay, explicit seal/drain, and a dedicated +schema-v11 `StreamFold` recovery envelope. The fold publishes its base-table pointer, lifecycle HEAD witness, and fixed lineage in one manifest CAS; no MemWAL state is graph-visible merely because it is durable. @@ -57,6 +58,13 @@ This remains an internal storage/recovery capability, not a public streaming feature. V8 has no `@stream`, production SDK/HTTP/CLI surface, persistent quiesce/resume controls, correction lane, or fresh-read surface. Its private integration seam exists only under the feature-gated crash/evidence suite. +Gate R0 originally found that one legal high-entropy near-cap generation could +acknowledge and materialize but then exceed the fold's 32-MiB closure charge. +The private implementation now charges scanner output by logical slice and +copies each emission into dense owned arrays; that exact 8,192-row shape folds +and publishes successfully. This repair does not create a new format: no +schema v9 or product surface is activated by the selected unbounded retain-all +profile or the optional later B2b managed-reclamation profile. A v7/0.11.x graph is not reinterpreted or migrated in place: config-v1 never becomes data-bearing config-v2 state. Export it with the v7 binary, initialize a diff --git a/docs/dev/writes.md b/docs/dev/writes.md index c82e483a..6a626f8b 100644 --- a/docs/dev/writes.md +++ b/docs/dev/writes.md @@ -618,9 +618,14 @@ Phase B1 adds the private row path: claimed-writer retirement. Any ownership-transfer ambiguity fails closed. The strict fold scans only the selected fresh generation with -`LsmScanner::without_base_table`, validates the already-normalized rows against -one captured graph snapshot, and stages an exact-`id` upsert plus exactly one -`MergedGeneration` marker in the same Lance `Update`. It then arms a +`LsmScanner::without_base_table`. It charges each scanner batch by the logical +memory of its selected slices against the same 8,192-row/32-MiB generation +limit, then uses an Arrow take of every selected row to build dense owned arrays +before retaining that batch. This avoids carrying a small logical slice through +the fold with the scanner's much larger sparse parent buffers. The fold +validates those already-normalized rows against one captured graph snapshot and +stages an exact-`id` upsert plus exactly one `MergedGeneration` marker in the +same Lance `Update`. It then arms a recovery-v11 `StreamFold` sidecar containing the physical cut, pre-minted transaction identity, fixed lineage, and complete pointer/lifecycle outcome. Under admission → schema → main → table gates, the final barrier re-lists all @@ -637,7 +642,35 @@ query-visible only after fold. The support boundary remains main-only, unsharded, one resident stream worker, and one live writer process; the admission lease is not a distributed fence. -RFC-026 Phase B2-0 now specifies—but does not implement—the next contract. +Gate R0's closure defect is now repaired. The deterministic legal high-entropy +near-cap cell acknowledges without moving either manifest or base-table +version, materializes one generation, then folds 8,192 rows and publishes one +table version through one `__manifest` visibility CAS. The dense scanner-batch +copy keeps the accepted shape inside its logical 32-MiB limit. A separate +subprocess measurement recorded a 284,934,144-byte isolated fold RSS delta +(about 272 MiB), below the 384-MiB remeasurement tripwire. That tripwire guards +the measured implementation shape; changing the admission or fold strategy +requires remeasurement. + +The selected next profile is **unbounded retain-all**. It accepts that +stock-RC.1 materialization uses randomized generation roots and that a crash or +cold replay can leave more physical objects. OmniGraph adds no retained-byte, +object-count, file-count, or history quota, no quota-enforcement attempt ledger, +and no raw `_mem_wal` deletion. Current-object LIST evidence remains a useful +retention/path-shape check but does not claim coverage of incomplete multipart +uploads, provider versions/delete markers, local staging, or billed bytes. +Provider exhaustion is surfaced as a storage failure and can halt admission, +fold, or recovery progress; it never becomes silent loss, an early +acknowledgement, or an alternate publication path. The 8,192-row/32-MiB logical +Arrow admission envelope, deadlines, retry counts, and ambiguity outcomes +remain bounded. +RC.1's missing cross-open materialization receipt and complete physical-output +envelope are therefore not blockers for this profile. They remain relevant to +the future B2b managed-reclamation design. This decision activates no format or +product state. + +RFC-026's common B2 inventory specifies—but does not implement—the next product +contract. `@stream` declares eligibility but explicit, request-idempotent enrollment creates the stream incarnation and physical binding. A caller-stable `write_id` and exact predecessor then form an @@ -648,13 +681,13 @@ sequencing authority and publishes atomically with the base pointer under the shared stream-token gate/recovery envelope. Because that participant is graph-global, every token-moving sidecar blocks every manifest/main writer, including table-disjoint writers. A final-barrier late discovery releases all -admission, graph-history, schema, branch, token, and table gates and restarts from the root +admission, schema, branch, token, and table gates and restarts from the root barrier; it never recovers while retaining a leaf gate. Durable `OPEN -> DRAINING -> SEALED -> OPEN` state carries a monotonic lifecycle revision. Every external mutating management call compares that revision and records a bounded, complete terminal receipt; roll-forward-only resume/abort and -bounded `REPLACE`/`WITHDRAW` correction use recovery-v12. Public activation -also requires Lance-owned durable reclamation with whole-cut proof, +bounded `REPLACE`/`WITHDRAW` correction use recovery-v12. The B2b +managed-reclamation profile requires Lance-owned durable reclamation with whole-cut proof, attempt/receipt recovery, bounded history checkpointing, strong PUT/DELETE inventory plus multipart accounting/abort or durable accounting, a post-success epoch check, and a source-derived enforced @@ -674,23 +707,30 @@ deletable. Measurements validate those bounds; they do not create them. Generic Lance cleanup does not reclaim `_mem_wal`, and OmniGraph must never delete its raw paths. Internal schema v9/config-v3/state-v2/recovery-v12 and every public policy/wire surface -remain inactive until their crash, storage, cross-version, authorization, and -parity evidence passes. The physical watermark is per binding and does not -bound base/token or shared manifest history. B2-0 therefore specifies a separate +remain inactive until the common crash, no-delete/provider-failure, +cross-version, authorization, and parity evidence passes. The selected +unbounded retain-all profile has no physical watermark and no graph-history +quota. If a future B2b bounded/managed profile is pursued, its physical +watermark is per binding and does not bound base/token or shared manifest +history; that profile therefore also needs a separate manifest-authoritative, graph-global `GraphHistoryBudget`. Every manifest -publisher—stream and ordinary—takes the graph-history gate, reserves the exact +publisher—stream and ordinary—would take the graph-history gate, reserve the exact logical publication plus a source-bounded physical-growth envelope before -effect, and settles pending recovery charges exactly once. Initial B2 holds that +effect, and settle pending recovery charges exactly once. Initial B2b would hold that gate continuously from final reserve/sidecar arm through physical effects, manifest CAS, and charge finalization, so only one expected budget revision is -live. Dynamic per-stream +live. Its dynamic per-stream closure reserves cannot be consumed by ordinary commits or by another stream; at the shared floor ordinary work returns `GraphRebuildRequired`, while the named stream retains correction, quiesce, and `SEALED` export/rebuild headroom. -The universal B2 gate order is sorted relevant stream admission → graph history -→ schema → main branch → stream token → sorted graph tables; late global +That future bounded profile's gate order is sorted relevant stream admission → +graph history → schema → main branch → stream token → sorted graph +tables. The selected unbounded profile omits the graph-history gate: sorted +relevant stream admission → schema → main branch → stream token → sorted +graph tables. Late global authority discovery releases the complete suffix and restarts at the root -barrier. A writer with no relevant stream still begins at graph history. +barrier. A writer with no relevant stream does not acquire a nonexistent +storage/history budget. Indefinite in-place maintenance remains a later accepted strand. ### Open-time recovery sweep diff --git a/docs/dev/writing-path-state-of-affairs.md b/docs/dev/writing-path-state-of-affairs.md index cfba6a88..03cd1fec 100644 --- a/docs/dev/writing-path-state-of-affairs.md +++ b/docs/dev/writing-path-state-of-affairs.md @@ -1,7 +1,7 @@ # Write Path: State of Affairs **Type:** living architecture and execution summary -**Status:** current as of 2026-07-20 +**Status:** current as of 2026-07-21 **Surveyed:** OmniGraph 0.8.1 development, internal manifest schema v8, Lance 9.0.0-rc.1 at `cec0b7df` **Scope:** the direct-publish graph write path, its RFC-022–028 family, @@ -9,7 +9,9 @@ adjacent control and maintenance operations, known blockers, and the next decision points **Change-set boundary:** this page describes current main through RFC-026's -Phase A foundation, private Phase B1 core, and Phase B2-0 design closure. +Phase A foundation, private Phase B1 core, the Gate R0 findings, the subsequent +all-shape closure repair, and the decision to ship an unbounded retain-all +profile first. RFC-024, RFC-025, and RFC-027 remain research-blocked at their recorded evidence gates. Gate E0 first proved the bounded public Lance state classifier; Phase A activated internal schema v7 with recoverable empty enrollment, durable @@ -18,20 +20,28 @@ strict format refusal/rebuild. Internal schema v8 now preserves that foundation and adds stream-config v2, a root-scoped one-generation worker, watcher-backed durability followed by a same-writer post-durability epoch check before clean acknowledgement, exact replay/seal/retirement, and -recovery-v11 `StreamFold`. The private B1 evidence is green at that bounded -correctness boundary: the graph-level suite now includes post-watcher epoch -loss as `AckUnknown` plus retirement. The 2026-07-20 post-containment local -warm-ack probe remains history-flat at 9 table reads / 219 bytes; configured -RustFS retains only its 2026-07-19 pre-containment baseline and must be rerun -before a current object-store ack-cost claim. B2-0 specifies explicit enrollment, -compare-and-chain tokens, trusted row attribution, manifest-selected current- -token state, persistent lifecycle revisions with bounded management receipts, -strict correction, Lance-owned reclamation with an enforced storage admission -watermark, and a graph-global `GraphHistoryBudget` checked by every manifest -publisher. -Together with the RC.1 source audit, two checked-in guards show why stock RC.1 -does not yet provide safe reclamation: the tests themselves prove generic -cleanup non-ownership and the deleted-successor-sentinel fencing hazard. +recovery-v11 `StreamFold`. Gate R0 found that a legal high-entropy near-cap +generation could be acknowledged and materialized but could not fold because +shared-buffer capacity was charged instead of logical slice size. The fold now +charges logical slices and rebuilds dense arrays with `take`; that exact shape +acknowledges, materializes, folds, and publishes. The measured RSS delta for +one exclusive full fold is 284,934,144 bytes (about 272 MiB). CI's 384-MiB +threshold is a remeasurement tripwire, not a runtime allocator or hard memory +limit. The 2026-07-20 post-containment +local warm-ack probe remains history-flat at 9 table reads / 219 bytes; +configured RustFS retains only its 2026-07-19 pre-containment baseline and must +be rerun before a current object-store ack-cost claim. + +Gate R0 historically rejected a *bounded, finite-lifetime* retain-all claim on +stock RC.1 because materialization has no durable attempt receipt or complete +physical-output envelope. We have deliberately dropped that claim. The first +profile is unbounded retain-all: OmniGraph never deletes MemWAL objects, sets +no retained-byte or file-count limit, and treats provider exhaustion as a loud +operational failure. The missing receipt/envelope therefore does not block +activation. Managed reclamation remains optional Lance-owned work for a later +profile. The common explicit enrollment, compare-and-chain token, trusted +attribution, and lifecycle-receipt/correction contracts remain specified and +inactive; a `GraphHistoryBudget` is not part of the immediate plan. The RFC remains Draft and the implementation remains reachable only through a feature-gated private engine seam: there is still no schema v9/config-v3/ recovery-v12 activation, `@stream` syntax, production/public enrollment, @@ -74,8 +84,8 @@ The central chassis is implemented: [RFC-022](../rfcs/0022-unified-write-path.md [RFC-028](../rfcs/0028-stable-schema-identity.md) are complete at the documented support boundary. The remaining RFCs are not a backlog to implement in numeric order. RFC-024, RFC-025, and RFC-027 remain research-blocked. RFC-026 remains -strategic and Draft. Its production-neutral Gate E0, Phase A foundation, and -private Phase B1 core are complete at their bounded evidence gates. Internal +strategic and Draft. Its production-neutral Gate E0 and Phase A foundation +passed their bounded gates. Internal schema v7 introduced exact main-only empty MemWAL enrollment and local lifecycle exclusion; schema v8/config-v2 adds one private no-rollover generation behind a root-scoped worker, watcher-backed durability followed by @@ -86,14 +96,19 @@ already-normalized vector values—and neither calls an external embedding provider nor invents unspecified derived fields. Native branch controls alone may proceed at `SEALED`, because they do not move table HEAD. This remains a private format/correctness core, not a streaming product: no production caller -can enroll, put, acknowledge, or fold a row. Phase B2-0 has now closed the -explicit-enrollment, attribution, reclamation/orphan cleanup, correction, -same-key retry sequencing, persistent lifecycle/management-receipt, and graph- -global manifest-history designs. Phase B2 is the later implementation and -public activation; its first dependency is the reviewed Lance reclamation and -Lance-owned post-success-fence patch, not product endpoints. B1's adapter -recheck contains a stale epoch from becoming a clean OmniGraph acknowledgement; -it is not the substrate retention/fencing primitive B2 requires. +can enroll, put, acknowledge, or fold a row. Gate R0 exposed two distinct +facts. First, the original accounting could reject one legal admitted near-cap +shape after durable acknowledgement; logical-slice accounting plus dense +rebuilding now closes that shape. Second, stock RC.1 cannot prove a lifetime +bound for materialization attempts or their complete physical growth. The +selected first profile makes no such claim: it retains every MemWAL object +without an OmniGraph file/byte limit and accepts loud provider exhaustion. +Neither a test-only attempt ledger nor managed reclamation is on the immediate +activation path. The next work is the common private token, attribution, +lifecycle, and correction machinery, followed by product contracts only after +their evidence is green. B1's +adapter recheck contains a stale epoch from becoming a clean OmniGraph +acknowledgement; it is not a substrate retention/fencing primitive. The largest remaining correctness boundary is topology: destructive recovery is serialized across every handle in one process, but not against a live writer @@ -189,7 +204,7 @@ These operations do not create a side door around the protocol: | Validation | Value, enum, uniqueness, edge-RI, and cardinality use one catalog-derived, delta-scoped evaluator across mutation, load, and merge. Committed non-key uniqueness probes are batched by constraint group and bounded chunks. | | Recovery | Same-process handles share ordered queues for the same canonical local root or identical normalized opaque remote/custom URI. Stream-admission domains are acquired outside schema → branch → table gates. Mutation/load, SchemaApply, BranchMerge, EnsureIndices, StreamFold, and `refresh` heal roll-forward-only; Optimize, Repair, and Cleanup refuse pending recovery, while branch controls use specialized barriers. Read-write open performs the quiesced full sweep, including exact v10 enrollment and v11 fold completion; read-only open never repairs and refuses unresolved stream recovery. A resolved intent is audited internally with the original actor when present; no-effect cleanup need not create graph lineage. | | RFC-026 Phase A (v7 foundation) | Internal schema v7 introduced identity-keyed lifecycle rows and exact empty main/unsharded enrollment. At that Phase-A-only boundary, any lifecycle row—including `SEALED`—rejected base-table, schema, maintenance, repair-adoption, and recovery effects under a process-local admission lease because no drain/fold witness-update adapter existed. Native branch create/delete alone could proceed at `SEALED` because it did not move table HEAD; `OPEN`/`DRAINING` refused it. Enrollment and open-time validation rejected named-branch overlap and any uncovered lifecycle/MemWAL mismatch. Schema v8 preserves these foundation guarantees. | -| RFC-026 Phase B1 (private core) | Internal schema v8/config-v2 adds one root-scoped, cross-handle serialized worker and a hard-bounded 8,192-row/32-MiB no-roll generation. Watcher success proves durability; a clean `DurableBatchAck` additionally requires the same `ShardWriter::check_fenced()` to succeed immediately afterward. Fence loss, epoch-read failure, owner-task failure, or deadline ambiguity is post-invocation `AckUnknown` plus worker retirement. Reopen/replay is conservative, exact drain proof precedes quiesced abort, and recovery-v11 folds one already-normalized generation then atomically refreshes the table pointer and `OPEN` lifecycle witness. The genuine v7↔v8 run and graph-level suite, including the post-watcher epoch-loss cell, are green. Post-containment local ack work remains history-flat at 9 reads / 219 bytes; the configured-RustFS rerun is pending. The RFC remains Draft and the seam remains private. | +| RFC-026 Phase B1 (private core) | Internal schema v8/config-v2 adds one root-scoped, cross-handle serialized worker and a hard-bounded 8,192-row/32-MiB no-roll generation. Watcher success proves durability; a clean `DurableBatchAck` additionally requires the same `ShardWriter::check_fenced()` to succeed immediately afterward. Fence loss, epoch-read failure, owner-task failure, or deadline ambiguity is post-invocation `AckUnknown` plus worker retirement. Reopen/replay is conservative, exact drain proof precedes quiesced abort, and recovery-v11 folds one already-normalized generation then atomically refreshes the table pointer and `OPEN` lifecycle witness. Gate R0's deterministic legal high-entropy near-cap cell exposed sparse scanner arrays retaining oversized backing buffers. Fold now charges logical slices and copies each scanner emission into dense owned arrays; the exact 8,192-row shape acknowledges, materializes, folds, and publishes. The isolated reference run measured a 284,934,144-byte fold RSS delta, below the 384-MiB CI remeasurement tripwire. B1 remains private. | | Lance access | One process-wide `ObjectStoreRegistry` reuses clients. Each `Omnigraph` handle owns its cached data-table `Session`; one process-wide zero-cache control `Session` opens mutable tips. Only the object-store registry is shared between the data and control sessions. This is “cache the past, never the present,” not one global cached session. | | Maintenance | EnsureIndices stages exact missing-index transactions. Optimize coordinates graph-wide compaction/index work under one bounded recovery envelope. Periodic optimize compacts `__manifest`, but unmaintained history-dependent paths are not globally flat. | @@ -213,7 +228,7 @@ retry rules, recovery classification, and the full owned limits. | [023 — Key-conflict fencing](../rfcs/0023-key-conflict-fencing.md) | **Implemented** | Internal schema v6 introduced exact-`id` PK metadata, closed keyed routing, typed conflicts, bounded replay, rebuild/refusal, and accepted performance evidence; v8 preserves that contract. | | [024 — Durable table heads](../rfcs/0024-durable-table-heads.md) | **Research-blocked** | The first in-manifest BTREE candidate has a specified logical contract and flat indexed row/range work, but fails the complete physical-I/O gate. No head rows or heads format are active. | | [025 — Checkpoint retention](../rfcs/0025-checkpoint-retention.md) | **Research-blocked** | Lance tag/pin semantics pass, but the proposed in-manifest registry access shape is not history-flat after compaction. No checkpoint rows, `ogcp_` production tags, API, or cleanup integration are active. | -| [026 — MemWAL streaming ingest](../rfcs/0026-memwal-streaming-ingest.md) | **Draft; Phase A/B1 implemented and evidence-green; B2-0 specified; public inactive** | Schema v8 preserves v7/recovery-v10 enrollment and adds config-v2 one-generation admission plus recovery-v11 strict fold. Private B1 now admits a clean acknowledgement only after watcher success and a same-writer post-durability epoch check; its graph-level suite covers epoch loss in that window. B2-0 specifies explicit enrollment, v9 token/attribution, state-v2 lifecycle revisions/management receipts and correction, Lance-owned reclamation, and a graph-global reserve-first manifest-history budget; its two stock-RC.1 guards are checked in. The adapter containment neither activates schema v9 nor supplies raw MemWAL GC, a Lance-owned fence, or a production enrollment, `@stream`, put/ack/fold, drain/resume, or fresh-read surface. | +| [026 — MemWAL streaming ingest](../rfcs/0026-memwal-streaming-ingest.md) | **Draft; Phase A/B1 private core implemented; widest-shape closure green; unbounded retain-all selected; public inactive** | Schema v8 preserves v7/recovery-v10 enrollment and adds config-v2 one-generation admission plus recovery-v11 strict fold. A clean acknowledgement still requires watcher success and a same-writer post-durability epoch check; logical-slice charging plus dense copies close the legal near-cap fold shape. Gate R0's historical no-go applies to a finite storage/lifetime promise: stock RC.1 cannot prove an attempt cap or complete physical-growth envelope. The selected first profile makes no such promise—it never deletes MemWAL objects and sets no retained-byte, object, file, or history quota. Common enrollment/token/attribution/lifecycle/correction, authorization, and product-parity contracts remain specified and inactive. `GraphHistoryBudget` and B2b managed reclamation belong to an optional future bounded profile. No schema v9 or product surface is active. | | [027 — Lineage merge deltas](../rfcs/0027-lineage-merge-deltas.md) | **Research-blocked** | The desired O(delta) classifier and fallback contract are specified. Selective live-row and deletion-delta discovery are not yet bounded, so `OrderedTableCursor` remains the correctness path. | | [028 — Stable schema identity](../rfcs/0028-stable-schema-identity.md) | **Implemented** | Rename-stable IDs, table incarnation, identity-derived paths, schema/recovery integration, and strict rebuild activation were introduced in v5 and remain active in v8. | @@ -224,8 +239,9 @@ retry rules, recovery classification, and the full owned limits. | Distributed recovery fence | Process-local queues cannot stop a live foreign process; Lance restore may orphan its commits and native refs lack conditional compare-delete. Supported destructive recovery remains one writer process per graph. | A separately designed and adversarially tested distributed fence before multi-process writers, background compensation, or cross-process exact maintenance recovery. | | History-flat authority | RFC-024/025 show flat BTREE rows/ranges/pages can coexist with history-growing manifest discovery or compacted bytes. No heads/checkpoint format is active; mutable tip caches and a second authority remain rejected. | A new Lance-native access shape—or revised measured operational contract—passes the original cold/warm, compacted/uncompacted, local/object-store gate. | | Internal history GC | Safe live-writer cleanup needs a durable resurrection/retention boundary; otherwise a stalled writer can recreate a collected version. | An evidence-backed cleanup watermark/fence before automated `__manifest` version GC. | -| MemWAL delivery | RC.1 initializes the system index and claims shards as separate effects without a caller-minted combined receipt or cross-process seal. Phase A recovers that gap exactly for main-only, one-shard, one-live-writer-process empty enrollment. At the row boundary, the durability watermark is writer-wide while batch positions reset after MemTable rollover, `put_no_wait` may mutate before returning `Err`, replay leaves its fresh BatchStore WAL watermark unset, and `wait_for_flush_drain` can lose a completed failure before a late waiter snapshots it. Neither batch positions nor WAL statistics are durable receipts. | Private B1 closes exactly that bounded row case. B2-0 closes the remaining logical design with compare-and-chain tokens, trusted hidden row metadata, a manifest-selected current-token participant, protocol-v2 lifecycle, and bounded correction. Implement and prove those contracts privately before adding a public caller. A public receipt/seal or accepted distributed fence remains the exit for overlapping processes and failover. | -| MemWAL reclamation | Stock RC.1 exposes evidence-level raw listing and manifest reads, not a complete classified inventory or safe MemWAL delete/GC primitive. Generic `cleanup_old_versions` leaves `_mem_wal` unchanged. Worse, deleting the successor's empty WAL fence sentinel can let a stale writer complete a WAL PUT and report watcher success because RC.1 has no post-success epoch check. Private B1 now prevents that stale epoch from becoming a clean OmniGraph acknowledgement by checking the same writer after watcher success and returning `AckUnknown` plus retirement on fence/read ambiguity. Raw path deletion in OmniGraph remains forbidden: the adapter check does not retract durable bytes or protect raw Lance callers. | Author Lance-owned durable inspect/plan/execute plus attempt/receipt recovery, Lance-owned post-success fence check, and bounded history checkpoint; open it upstream and pin the exact reviewed fork commit without waiting for a release. Then prove whole-cut/cursor eligibility, strong PUT/DELETE inventory plus multipart accounting/abort or durable accounting, stale plans, prune CAS, partial delete/lost result, orphan/unknown retention, and a source-derived enforced physical-growth reservation on local/RustFS before public admission. Separately prove graph-global manifest-history bootstrap/accounting across every writer and independent per-stream closure reserves. | +| MemWAL delivery and closure | RC.1 initializes the system index and claims shards as separate effects without a caller-minted combined receipt or cross-process seal. Phase A recovers that gap exactly for main-only, one-shard, one-live-writer-process empty enrollment. At the row boundary, the durability watermark is writer-wide while batch positions reset after MemTable rollover, `put_no_wait` may mutate before returning `Err`, replay leaves its fresh BatchStore WAL watermark unset, and `wait_for_flush_drain` can lose a completed failure before a late waiter snapshots it. Neither batch positions nor WAL statistics are durable receipts. Gate R0 also found a private closure bug caused by sparse scanner arrays retaining oversized backing buffers; logical-slice charging and dense copies now close the exact legal near-cap shape. | Implement and prove the common compare-and-chain token, trusted attribution, revisioned lifecycle receipts, bounded correction, authorization, and product-parity contracts privately before adding a public caller. A public receipt/seal or accepted distributed fence remains the exit for overlapping processes and failover, not for the current single-live-writer-process profile. | +| MemWAL retained growth | A clean retained generation has measurable current objects, and the Gate R0 sweep proves that referenced currently listed immutable paths retain their class and size at one/four/eight folds. RC.1 still provides no durable cross-open attempt cap, complete physical-output receipt, or provider-billed-byte inventory. That prevents OmniGraph from promising a finite retained-storage bound. | The selected first profile is deliberately unbounded: never delete raw `_mem_wal` objects, impose no retained-byte/object/file/history quota, and fail loudly if the provider refuses further writes. No attempt ledger or physical-growth reservation is required for a contract that promises no storage bound. Add those only if a later profile claims bounded retention. | +| MemWAL reclamation (optional B2b) | Stock RC.1 exposes evidence-level raw listing and manifest reads, not a complete classified inventory or safe MemWAL delete/GC primitive. Generic `cleanup_old_versions` leaves `_mem_wal` unchanged. Worse, deleting the successor's empty WAL fence sentinel can let a stale writer complete a WAL PUT and report watcher success because RC.1 has no post-success epoch check. Private B1 contains that result for its own acknowledgement but cannot retract durable bytes or protect raw Lance callers. Raw path deletion in OmniGraph remains forbidden. | Keep B2b as optional Lance-owned durable inspect/plan/execute, attempt/receipt recovery, post-success fencing, bounded history checkpoint, strong inventory/accounting, and enforced-watermark work. It is not on the immediate retain-all activation path. | | O(delta) merge | A version-column predicate is still O(rows) without a selective source, and deleted rows have no live version columns. Full ID differencing remains correct. | Bounded live-row and deletion/change discovery, exact shadow agreement, and a table-size-flat one-row-delete gate. | | Optimize provenance | Compaction/reindex has no stable caller-minted transaction covering the complete effect. Optimize therefore uses bounded, not exact, provenance. | Both an upstream maintenance transaction API and distributed recovery fencing. | | Remaining bounds/operations | Some long-running operations lack complete memory/time budgets; rollback waits for quiesced read-write open; recovery audit has no public query. | Incremental, independently owned hardening without widening format or topology. | @@ -241,13 +257,14 @@ must publish atomically with the table pointer. Other writer/control/ maintenance paths touching the table refuse pre-effect or drain first. Gate E0 proved the classifier and witness model with complete direct-probe and object-store evidence; Phase A established the reversible support restriction. -Private B1 now consumes it for one bounded generation, watcher-backed +Private B1 now consumes it for one nominally bounded generation, watcher-backed durability plus the same-writer post-durability epoch check, replay, and strict folding without pretending the process-local lease is a distributed fence, inventing a durable WAL offset, or reusing a watcher across rollover. Fence -loss or uncertainty after durability is `AckUnknown`, never a clean ack. Its -green evidence authorizes only this private bounded core, not raw MemWAL -reclamation, the public B2 product, or a broader topology. +loss or uncertainty after durability is `AckUnknown`, never a clean ack. Gate +R0 exposed a sparse-buffer closure gap; the dense-copy repair closes the exact +legal near-cap shape without changing admission. That result authorizes neither +raw MemWAL reclamation, a public product, nor a broader topology. The full known-gap ledger, including adjacent local-CAS and unsupported multi-version-topology details, remains in @@ -262,7 +279,7 @@ write-path redesign. |---|---| | Lance surfaces consumed by RFC-022/023—transactions, branches, key filters, staged indexes, compaction, and shared sessions—remain compatible; separately surveyed tag/cleanup behavior is also unchanged | Keep the current architecture. PR #364 passed 22 surface guards and 129 runnable failpoint tests; the Gate-0 follow-up adds the 23rd guard plus checkpoint cost evidence. No format redesign is needed. | | Derived MemWAL datasets inherit the base store parameters and `Session`; `put_no_wait` returns an optional watcher whose completion is `Result<()>`, not a durable row coordinate. RC.1's watcher watermark spans the writer while active-MemTable batch positions reset on rollover; `put_no_wait` can also mutate before a later scheduling error. Replay leaves the fresh BatchStore watermark unset, and a late `wait_for_flush_drain` can miss a completed failure. | Shared-session propagation removes one integration concern, but the watcher is safe only inside B1's proved single-generation lifecycle. B1 treats watcher success as necessary durability evidence, then requires the same writer's `check_fenced()` to succeed before clean acknowledgement. Every post-invocation error or ambiguity is `AckUnknown`; rollover remains prevented, the public replay-watermark bridge handles fold-only reseal, generation proof comes from refs plus authoritative manifest state, and the writer retires/reopens before another generation. This does not create an exact combined enrollment receipt or cross-process seal. MemWAL is strategic, not experimental. | -| Generic cleanup ignores `_mem_wal`, and RC.1 does not recheck the writer epoch after a successful WAL PUT | B1 now contains the clean-ack stale-epoch result for its own private caller by rechecking after watcher success and retiring on any fence/read ambiguity. Still do not write an OmniGraph-side collector: that check neither retracts a durable WAL effect nor protects raw Lance callers or reclamation. B2-0 requires Lance-owned durable reclamation, post-success fencing, bounded reclaim-history checkpointing, and an enforceable growth reservation. We will author and pin that reviewed patch ourselves rather than make upstream merge/release timing the calendar; public admission remains closed until its local/RustFS crash and bound-validation evidence pass. | +| Generic cleanup ignores `_mem_wal`, and RC.1 does not recheck the writer epoch after a successful WAL PUT | B1 contains the clean-ack stale-epoch result for its own private caller by rechecking after watcher success and retiring on any fence/read ambiguity. The selected retain-all profile performs no raw-path collection, so generic cleanup's non-ownership is expected. B2b keeps Lance-owned reclamation, post-success fencing, durable receipts, inventory/accounting, and an enforceable growth reservation as optional future work; it does not block unbounded retain-all. | | Experimental, opt-in `DataOverlay` was added | Do not adopt it now. OmniGraph does not enable feature flag 64 or emit the operation; unknown foreign overlay effects remain fail-closed. DataOverlay's experimental status says nothing about MemWAL's status. | | RC.1 expands Lance write rejection from the three row-address names to all five surveyed virtual system-column names | OmniGraph now rejects `_rowid`, `_rowaddr`, `_rowoffset`, `_row_created_at_version`, and `_row_last_updated_at_version` during parsing and accepted-IR validation. A beta.21 development graph using a newly reserved row-version name must be exported with beta.21, renamed, and rebuilt. | | A genuine ordinary-schema beta.21 V2_2 graph forward-opened, queried, and merge-wrote under RC.1 | No general storage-format migration. The reserved-name exception is explicit rather than hidden behind a format bump. | @@ -280,54 +297,45 @@ or slightly lower incremental RSS. That is not a roadmap signal. ### Now -1. **Keep the accepted private B1 boundary narrow and green.** Preserve the - schema-v8/config-v2/recovery-v11 format guards, watcher-success plus - post-durability epoch-check acknowledgement boundary, complete graph-level - B1 behavior/crash/race suite, genuine v7↔v8 refusal/rebuild, and the - post-containment local cost evidence. Rerun the configured-RustFS cell before - replacing its 2026-07-19 pre-containment baseline. Treat those results as - acceptance of one main-only, one-shard, one-live-writer-process - generation—not as permission to expose a product API or claim general - streaming throughput. -2. **Implement the B2-0 substrate dependency first.** Author the Lance-owned - durable opaque inspect/plan/execute reclamation primitive, attempt/receipt - recovery, sentinel-first manifest-named epoch claims, genesis/bootstrap plus - bounded claim/reclaim history checkpoint, and Lance-owned post-success epoch - check; open - the upstream PR and pin the exact reviewed fork commit without waiting for - its release. Add complete local/RustFS whole-cut, lost-result, fail-closed - stock-RC.1 format refusal, strong PUT/DELETE inventory plus multipart - accounting, one per-binding reserve-first durable ledger, versioned/soft- - delete/Object-Lock refusal, bounded materialization attempts/control - headroom, and enforced physical object/byte reservation evidence. Never delete - `_mem_wal` paths from OmniGraph. -3. **Implement the B2 contracts privately in dependency order.** Activate - schema v9/config-v3/state-v2/recovery-v12 only with explicit enrollment, - compare-and-chain token authority, trusted hidden attribution, persistent - revisioned quiesce/resume/abort-drain with bounded management receipts, and - bounded `REPLACE`/`WITHDRAW` correction. Keep the public surface absent while - the crash, rebuild, enforced-watermark, and cross-version matrices converge. - Treat the per-binding watermark as `_mem_wal`-only. Separately initialize one - graph-global `GraphHistoryBudget`, make every manifest writer reserve its - publication and source-bounded physical-growth envelope, and retain dynamic - per-stream closure reserves through `SEALED` rebuild. Base/token/shared- - manifest history remains finite-lifetime until a later maintenance strand. -4. **Add product surfaces last.** Only after the private B2 machinery and - evidence are green should schema intent, SDK, HTTP, CLI, Cedar, OpenAPI, and - shutdown ownership converge on the same core. -5. **Keep RFC-024/025/027 stopped at their research no-gos.** Their blockers are +1. **Keep the repaired B1 closure cell as a permanent regression gate.** The + exact 8,192-row high-entropy shape must continue to acknowledge, materialize, + fold, and publish locally and on configured RustFS. Keep 384 MiB as a CI + remeasurement tripwire for the isolated fold RSS delta, not as a runtime + allocation promise or a retained-storage limit. +2. **Implement the common B2 contracts privately for the selected retain-all + profile.** Add explicit first-use enrollment, compare-and-chain token + authority, trusted hidden contributor attribution, persistent revisioned + quiesce/resume/abort-drain receipts, and bounded `REPLACE`/`WITHDRAW` + correction. Keep every step under recovery and the manifest visibility CAS. + Do not add a storage quota, materialization-attempt ledger, + `GraphHistoryBudget`, or raw `_mem_wal` deletion. +3. **Prove retain-all failure behavior.** Add local and configured-RustFS crash + coverage showing that acknowledged generations remain recoverable, unknown + objects are never adopted or deleted, provider write failures are loud, and + unresolved recovery blocks new progress. Preserve the existing row, + logical-Arrow-memory, deadline, retry, and ambiguity bounds even though + retained storage is unbounded. +4. **Add product surfaces last.** Only after the private common machinery and + evidence are green should schema intent, SDK, HTTP, CLI, Cedar, OpenAPI, + shutdown ownership, and authoritative status converge on the same core. +5. **Keep B2b managed reclamation independent and optional.** If a bounded + profile is scheduled later, author Lance-owned durable inspect/plan/execute, + attempt/receipt recovery, post-success epoch fencing, strong inventory or + durable accounting, and an enforced retained-storage watermark. A + graph-global `GraphHistoryBudget` would require its own RFC and every-writer + evidence. Never delete `_mem_wal` paths from OmniGraph. +6. **Keep RFC-024/025/027 stopped at their research no-gos.** Their blockers are independent of the v8 stream core; do not add RFC-024 heads, RFC-025 graph checkpoint rows/format, or RFC-027 lineage-delta state as incidental B2 - work. Lance's private reclaim-history bootstrap in step 2 is substrate - recovery metadata, not RFC-025 graph checkpoint authority. -6. **Give a distributed recovery fence its own design and evidence gate.** + work. +7. **Give a distributed recovery fence its own design and evidence gate.** Define authority, expiry/renewal, fencing tokens, and crash semantics before implementation; require adversarial multi-process tests on local and object storage. -7. **Continue low-risk v8 hardening.** Add missing resource/time budgets, +8. **Continue low-risk v8 hardening.** Add missing resource/time budgets, preserve cost-at-history-depth gates, and reduce constant factors only where the existing authority model remains intact. -8. **Coordinate upstream without making it the calendar.** The additional +9. **Coordinate upstream without making it the calendar.** The additional Lance asks are replay initializing the per-MemTable WAL watermark, drain completion that cannot lose a finished error, recoverable MemWAL enrollment/admission, conditional native ref operations, exact maintenance @@ -335,17 +343,17 @@ or slightly lower incremental RSS. That is not a roadmap signal. ### Only when an evidence trigger fires -- **Public MemWAL row activation:** Phase A and private B1 are green at their - bounded gates; the RFC remains Draft and public activation remains off. - B2-0 now specifies explicit enrollment, durable contributor attribution, - compare-and-chain sequencing, bounded reclamation plus an enforced retained- - storage admission watermark, strict correction/disposition, persistent - revisioned lifecycle with bounded management receipts, and graph-global - manifest-history admission with per-stream closure reserves. Phase B2 activation still - requires their implementation and evidence, schema/SDK/API/CLI parity, - cancellation ownership, and authoritative status. The exact upstream - enrollment receipt/seal remains the preferred simplification and - broader-topology gate, not a reason to widen B1 while waiting. +- **Public MemWAL row activation:** Phase A passed its bounded gate; private B1 + now closes the exact legal near-cap shape, and unbounded retain-all is the + selected storage posture. The RFC remains Draft and public activation remains + off until explicit enrollment, durable contributor attribution, + compare-and-chain sequencing, strict correction/disposition, persistent + revisioned lifecycle with bounded management receipts, authorization, + schema/SDK/API/CLI parity, cancellation ownership, and authoritative status + pass their gates. Retained storage has no OmniGraph byte/file/history limit; + provider exhaustion fails loudly. The exact upstream enrollment receipt/seal + remains the preferred simplification and broader-topology gate, not a + dependency for the current one-live-writer-process profile. - **Durable heads or checkpoints:** when a new current-authority access shape exists, run the full decision instrument before adding production rows or a format stamp. @@ -369,10 +377,10 @@ Do not: manifest-access cost; - implement RFC-024, RFC-025, or RFC-027 production paths behind a nominal feature flag before their blocking gate closes; -- treat the private schema-v8 B1 core or its green evidence as permission to - expose a put/ack endpoint before the specified B2 attribution, token, - reclamation/enforced-watermark, correction, and persistent lifecycle contracts are - implemented and evidence-green; +- treat the private schema-v8 B1 core or repaired near-cap closure as permission + to expose a put/ack endpoint before explicit enrollment, attribution, + compare-and-chain token, correction, persistent lifecycle, authorization, + and product-parity contracts are implemented and evidence-green; - delete or rewrite `_mem_wal` objects from OmniGraph, or interpret generic Lance version cleanup as MemWAL reclamation; - permit a base-table writer to advance any Phase A lifecycle's HEAD (including @@ -405,7 +413,8 @@ Do not: | MemWAL bounded-enrollment Gate E0 (green decision harness; no row path) | [`memwal_enrollment_gate.rs`](../../crates/omnigraph/tests/memwal_enrollment_gate.rs), [`lance_surface_guards.rs`](../../crates/omnigraph/tests/lance_surface_guards.rs), [RFC-026 §12.1](../rfcs/0026-memwal-streaming-ingest.md) | | MemWAL Phase A lifecycle/exclusion/recovery | [`failpoints.rs`](../../crates/omnigraph/tests/failpoints.rs), [`forbidden_apis.rs`](../../crates/omnigraph/tests/forbidden_apis.rs), manifest/write-queue unit tests, [RFC-026 §12.2](../rfcs/0026-memwal-streaming-ingest.md) | | MemWAL private Phase B1 admission/fold/crash and cost evidence | [`memwal_stream.rs`](../../crates/omnigraph/tests/memwal_stream.rs), [`memwal_stream_cost.rs`](../../crates/omnigraph/tests/memwal_stream_cost.rs), worker/recovery unit tests, [RFC-026 §12.3](../rfcs/0026-memwal-streaming-ingest.md) | -| MemWAL Phase B2-0 reclamation ownership/no-go guards | [`lance_surface_guards.rs`](../../crates/omnigraph/tests/lance_surface_guards.rs), [RFC-026 §4.5](../rfcs/0026-memwal-streaming-ingest.md) | +| MemWAL Gate R0 retention decision, current-object census, attempt reuse, repaired near-cap closure, and fold RSS tripwire | [`memwal_stream_cost.rs`](../../crates/omnigraph/tests/memwal_stream_cost.rs), [RFC-026 §0.2 and §12.4](../rfcs/0026-memwal-streaming-ingest.md) | +| MemWAL B2b reclamation ownership/no-go guards | [`lance_surface_guards.rs`](../../crates/omnigraph/tests/lance_surface_guards.rs), [RFC-026 §4.5.2](../rfcs/0026-memwal-streaming-ingest.md) | | Cross-version refusal/rebuild, including v6↔v7 and v7↔v8 | [`crossversion_upgrade.rs`](../../crates/omnigraph-cli/tests/crossversion_upgrade.rs) | Use [testing.md](testing.md) to find the existing owner before adding coverage. @@ -428,11 +437,18 @@ loss as `AckUnknown` plus retirement, and the genuine v7↔v8 old/new-binary refusal/rebuild remains green. The post-containment local warm-ack result stays flat at 9 reads / 219 bytes; configured RustFS retains only its 2026-07-19 pre-containment baseline and requires rerun before a current object-store -ack-cost claim. That evidence activates only the schema-v8 private core; -the RFC remains Draft and every public B2 surface remains inactive. B2-0's two -additional guards prove why stock RC.1 generic cleanup and raw successor -fence-sentinel deletion cannot be the reclamation implementation; they are a -no-go boundary, not a shipped collector. +ack-cost claim. Gate R0 first exposed, and the repaired regression cell now +closes, the deterministic high-entropy near-cap shape: acknowledgement and +materialization are followed by one successful fold and manifest publication. +The one/four/eight-fold census and source audit still prove that current LIST +cannot establish a lifetime/provider-billed bound and that RC.1 has no durable +attempt cap or reserve-first complete-output envelope. Those are accepted facts +for unbounded retain-all, not blockers: the selected profile promises no +retained-storage ceiling and activates no collector. The RFC remains Draft and +every public surface remains inactive pending the common correctness and product +contracts. B2b's two additional guards prove why stock RC.1 generic cleanup and +raw successor-fence-sentinel deletion cannot implement a later managed profile; +they are a no-go boundary for reclamation, not for retain-all. ## Updating this page diff --git a/docs/rfcs/0026-memwal-streaming-ingest.md b/docs/rfcs/0026-memwal-streaming-ingest.md index c2ab77cd..b9637ead 100644 --- a/docs/rfcs/0026-memwal-streaming-ingest.md +++ b/docs/rfcs/0026-memwal-streaming-ingest.md @@ -10,22 +10,33 @@ owner: OmniGraph maintainers # RFC-026 — MemWAL streaming ingest -**Status:** Draft / Phase A foundation and Phase B1 private core evidence-green; -Phase B2-0 contracts specified below; public streaming inactive +**Status:** Draft / Phase A foundation and the private Phase B1 row/fold core +implemented; the first production profile is unbounded retain-all with no +MemWAL GC; public streaming inactive **Date:** 2026-07-10 **Gate E0 evaluated:** 2026-07-18 **Phase A foundation completed:** 2026-07-18 -**Phase B1 private core green:** 2026-07-19 (internal schema v8, stream-config -v2, recovery-v11, one feature-gated engine seam, and all private B1 acceptance -gates in §12.3 passed) +**Historical Phase B1 subset acceptance:** 2026-07-19 (internal schema v8, +stream-config v2, recovery-v11, one feature-gated engine seam, and the then- +declared §12.3 shape set passed; Gate R0 later superseded the blanket all-shape +claim) **Phase B1 acknowledgement-fence containment:** 2026-07-20 (after watcher success, private B1 requires the same `ShardWriter` to pass `check_fenced()` before a clean acknowledgement; fence loss or an unreadable or unsettled check is `AckUnknown` plus worker retirement) -**Phase B2-0 contract closure:** 2026-07-19 (§4.1–§4.6 specifies the -admission/token, attribution, revision-fenced lifecycle, correction, -reclamation, and graph-global closure-budget contracts; it activates no schema -or product surface) +**Phase B1 near-cap closure repaired:** 2026-07-21 (fresh scanner batches are +densified before retention; the legal high-entropy 32-MiB generation now folds +and publishes exactly once; see §0.2 and §12.3) +**Phase B2 contract inventory:** 2026-07-19 (§4.1–§4.6 specified the common +admission/token, attribution, revision-fenced lifecycle, correction, retention, +and graph-global closure-budget contracts; the 2026-07-21 amendment removes +physical-storage quotas, aggregate receipt caps, and `GraphHistoryBudget` from +the selected retain-all profile; it activates no schema or product surface) +**Gate R0 evaluated:** 2026-07-20 — historical **no-go** for a *bounded* +retain-all profile on stock RC.1; its storage findings are accepted limitations +of the selected unbounded profile and its fold blocker is now closed (§0.2) +**Retention decision:** 2026-07-21 — select unbounded retain-all/no-GC for the +first profile; managed reclamation remains deferred (§4.5) **Author track:** Maintainer design series **Depends on:** [RFC-022](0022-unified-write-path.md)'s unified write and generic recovery-sidecar protocol, plus @@ -112,16 +123,113 @@ refusal/rebuild. It still exposes no production enrollment entry point and cannot append or acknowledge a row. RFC-026 therefore remains draft and public streaming remains inactive until the later gates close. -The implementation remains deliberately split. **Phase B1 is now implemented -privately** for one bounded generation, from admission through crash replay and -one strict RFC-022 fold. It is reachable only through one feature-gated, -doc-hidden engine test seam and is not a product surface. **Phase B2 remains -inactive.** B2-0 now specifies its missing contracts in §4.1–§4.6, while B2 -implementation still owns schema intent, production first use, -SDK/HTTP/CLI/OpenAPI/Cedar parity, minimum durable contributor attribution and -reclamation, and the operator escape/correction path plus an explicit same-key -`AckUnknown` retry contract. B1's implementation and evidence ledger is green -in §12.3; that does not admit a public caller. +The implementation remains deliberately split. **Phase B1 is implemented +privately** for one admission-bounded, no-roll generation, from admission +through crash replay and one strict RFC-022 fold. The 2026-07-21 dense-scan +repair closes the legal near-cap row shape that Gate R0 exposed. The core is +reachable only through one feature-gated, doc-hidden engine test seam and is +not a product surface. The first production profile is now **B2a unbounded +retain-all**: stock Lance owns the WAL, OmniGraph never deletes or reclaims a +MemWAL object, and physical storage is allowed to grow monotonically without a +file, object, or byte quota. **B2b** remains the deferred managed-reclamation +profile using a Lance-owned primitive. The token, trusted-attribution, +recovery, lifecycle, correction, and product-parity contracts in §4.1–§4.4 and +§4.6 still apply. `GraphHistoryBudget`, physical-storage admission, and +aggregate receipt-capacity reservations do not. No schema intent, production +first use, SDK/HTTP/CLI/OpenAPI/Cedar surface, durable contributor attribution, +correction path, or public same-key `AckUnknown` retry contract is implemented. + +### 0.2 Gate R0: historical bounded-retention result and current disposition + +**Gate R0 asked one question:** under the existing main-only, unsharded, +one-live-writer-process boundary and the pinned stock Lance surface, could +OmniGraph place a finite, source-derived upper bound on every retained +`_mem_wal` effect while deleting nothing, and reserve enough of that bound +before acknowledgement for the admitted generation to fold, be corrected if +necessary, quiesce, and reach `SEALED`? + +Gate R0 is production-neutral. It adds no manifest authority row, recovery kind, +format stamp, `@stream` syntax, production enrollment or row caller, +SDK/HTTP/CLI/OpenAPI/Cedar surface, and performs no `_mem_wal` deletion. Its +checked-in decision instrument inventories current listed objects by WAL, +shard-manifest, generation-data, generation-manifest/transaction/deletion, +PK-sidecar, Bloom, and user-index class. Unknown paths fail closed; all listed +immutable paths must retain the same path, class, and listed size between +retain-all checkpoints. This is current-object evidence only: ordinary listing +cannot prove content identity, incomplete +multipart uploads, superseded provider versions, delete markers, local staged +temporary files, or billed storage. + +**Historical Gate R0 result (2026-07-20): no-go for bounded retain-all on stock +Lance RC.1.** Three findings were recorded: + +1. A flush chooses a fresh randomized generation directory, writes the + generation dataset, optional deletion state, Bloom filter, and mandatory PK + sidecar, and only then attempts the shard-manifest CAS. A crash or error can + retain an unreferenced partial/complete subtree. Reopen reconstructs the same + logical generation from WAL and may choose another random directory. Stock + Lance persists or enforces no attempt ID, attempt counter, reservation, or + receipt that bounds materialization across cold opens. Individual process or + provider retries may be configured, but they are not represented in MemWAL + authority and repeated reopen can accumulate additional attempts. +2. RC.1 exposes no admission-grade, reserve-first conservative + physical-output estimator or complete post-attempt storage receipt. The + 32-MiB Arrow admission cap is not a source-derived bound for + data/Blob/transaction/manifest/deletion/PK/Bloom objects, local staging + residue, or multipart/provider residue. Measurements can validate a formula; + they cannot create one. +3. The deterministic high-entropy near-cap cell acknowledged a legal + 33,228,232-byte post-tombstone Arrow generation and retains 33,174,630 + currently listed immutable bytes after acknowledgement. Materialization + raised the listed retained total to about 65.1 million bytes. The old fold + retained sparse `LsmScanner` slices whose variable-width arrays still owned + their much larger backing buffers, so its logical-byte check charged roughly + 252.8 million bytes against the 33,554,432-byte generation limit and refused + closure. + +The success-only sweep remains useful but does not override those blockers: +one/four/eight referenced generation roots retain approximately 37.4 / 150.6 / +302.3 thousand currently listed immutable bytes locally, every earlier listed +path retains its class and size, and a +retry after the referenced cut reuses the exact generation root. Configured +RustFS Gate-R0 growth evidence and the actual 8,192-one-row decision-scale run +were not needed for the 2026-07-20 decision, and no numeric result from either +is claimed here. The configured RustFS cell is now wired as a post-merge/tag +regression signal. + +The 2026-07-21 amendment changes the disposition, not those measurements. The +first two findings remain true, but they are limitations only of a +finite-storage promise. B2a now makes no such promise: it has no MemWAL GC, no +file/object/byte ceiling, no retained-storage admission watermark, no +source-derived physical-output envelope, and no requirement that stock Lance +return a materialization-attempt receipt. Referenced generations, WAL, fence +sentinels, and unreferenced partial or complete materialization subtrees are +retained indefinitely. They are never adopted from path shape or listing, and +ordinary `cleanup` never treats them as reclaimable. Operators must provision +the backing store accordingly. Provider exhaustion remains an explicit +storage failure handled by the existing pre-/post-invocation and recovery +rules; it is never permission to acknowledge, discard, or fabricate success. + +The third finding is fixed. `scan_fresh_generation` now validates the streamed +logical row/byte totals and then takes every selected row into dense owned Arrow +arrays before retaining the batch. Dropping the sparse scanner batch releases +its oversized backing buffers before the next slice. The same deterministic +33,228,232-byte generation now folds all 8,192 rows, advances the base table +once, and becomes graph-visible at exactly one `__manifest` publication. One +isolated widest-fold reference run measured a whole-process peak-RSS delta of +284,934,144 bytes (about 272 MiB / 285 MB) over the small-fold baseline; a +384-MiB delta tripwire requires remeasurement before the admission or compaction +shape is widened. This is measured process memory, not a physical-storage +quota. The 8,192-row and 32-MiB logical generation limits, one-resident-writer +topology, exclusive-fold ownership, typed failures, and recovery barriers all +remain. + +This decision and repair authorize no internal schema +v9/config-v3/state-v2/recovery-v12 state, public stream contract, production +caller, or deletion. They only remove bounded physical retention from the +critical path and make the already-private B1 generation close for the widest +admitted shape. Public activation still requires the correctness and product +gates in §12.6. ## 1. Scope and non-goals @@ -396,14 +504,18 @@ exact incarnation and returns `StreamNotEnrolled` before body admission on ## 4. Future public activation -This section is the Phase-B2 wire contract, not part of the Phase-B1 private -engine slice. Public activation remains blocked until the strict core is green, -durable contributor attribution and bounded reclamation exist, and a -persistent quiesce/rebuild/correction escape prevents an `OPEN` enrollment -from permanently excluding work. B2's escape is quiesce plus export/rebuild -into a different root. In-place SchemaApply and maintenance on an enrolled -table stay refused until Phase D supplies automatic drain, witness update, and -rebind. +This section contains the future-public contract, not part of the Phase-B1 +private engine slice. **B2a unbounded retain-all** is the selected first +profile: it deletes no MemWAL object and performs no physical-storage admission +or accounting. **B2b** is the deferred managed-reclamation profile through the +Lance-owned protocol in §4.5.2. Token safety, trusted attribution, recovery, +revision-fenced lifecycle, correction, and a persistent quiesce/rebuild escape +remain common correctness contracts. B2a retains individually bounded protocol +records indefinitely and paginates their inspection; it has no aggregate +receipt-count/byte cap, `GraphHistoryBudget`, retained-storage quota, or +closure-capacity reservation. In-place SchemaApply and maintenance on an +enrolled table stay refused until Phase D supplies automatic drain, witness +update, and rebind. ### 4.1 Durable row identity and same-key retry safety @@ -647,9 +759,12 @@ protocol v2. `DRAINING` is a durable operation, not an in-memory observation. Every state-v2 row carries a strictly monotonic `lifecycle_revision`; every successful publication of that row increments it exactly once, including a publication that changes only a witness, retention summary, or last-fold -summary. The row also retains the immutable enrollment receipt and a bounded -set of management receipts described below. The complete manifest row -additionally carries: +summary. The row also retains the immutable enrollment receipt and append-only +management and effectful-claim receipt histories described below. Each receipt +has a source-bounded canonical representation, but B2a does not cap the +aggregate count or bytes and never evicts one. The complete manifest row +additionally carries +`current_claim_receipt_id?` and: ```text DrainDescriptor { @@ -682,6 +797,50 @@ StrictBlock { correction_revision, } +ClaimReceipt { + claim_id, + recovery_operation_id, + claim_kind, + profile: RETAIN_ALL | MANAGED_RECLAMATION, + claim_operation_digest, + attempt_count, + attempt_effect_chain: ordered [ClaimAttemptEffect], + attempt_effect_chain_digest, + terminal_attempt_id, + terminal_pre_shard_manifest_version, + achieved_shard_manifest_version, + achieved_writer_epoch, + sentinel_position, + sentinel_digest, + replay_cursor, + terminal_effect_digest, + terminal_classification: + STOCK_MANIFEST_PLUS_SENTINEL | + PATCHED_SENTINEL_PLUS_NAMING_MANIFEST, +} + +ClaimAttemptEffect { + ordinal, + attempt_id, + attempt_plan_digest, + bound_prestate_digest, + storage_envelope_digest?, // B2b only + planned_sentinel_position, + planned_sentinel_digest, + achieved_shard_manifest_version?, + achieved_writer_epoch?, + observed_sentinel_position?, + observed_sentinel_digest?, + attempt_terminal_effect_digest, + classification: + NO_EFFECT | + ABORTED_NO_EFFECT | + STOCK_MANIFEST_ONLY | + STOCK_MANIFEST_PLUS_SENTINEL | + PATCHED_SENTINEL_ONLY | + PATCHED_SENTINEL_PLUS_NAMING_MANIFEST, +} + SealedProof { drain_id, shard_manifest_version, @@ -690,6 +849,8 @@ SealedProof { current_generation, base_merged_generation, base_current_head_witness, + current_claim_receipt_id, + claim_receipt_set_digest, verified_empty_digest, } @@ -720,8 +881,9 @@ operation ID, the expected explicit fold uses `fold_operation_id`; resume/abort-drain uses `resume_id`; correction uses `correction_id`. The idempotency occurrence is `(stream incarnation, operation kind, operation ID)`; the expected revision is part of -its canonical request digest. Before checking the expected revision, the engine checks the bounded persisted -`ManagementReceipt` set: same occurrence and canonical request digest returns +its canonical request digest. Before checking the expected revision, the engine +checks persisted `ManagementReceipt` history: same occurrence and canonical +request digest returns the recorded result, while the same occurrence with another digest is `StreamIdempotencyConflict`. With no receipt, a revision or addressed-authority mismatch is effect-free `StreamLifecycleChanged`; it never retargets the call. @@ -738,28 +900,48 @@ exact declared-schema identity/configuration witness plus `enrollment_request_id`, and its separate immutable `EnrollmentReceipt` supplies the same lost-result guarantee. -Stream-config v3 hard-bounds management-receipt count and bytes and reserves -enough slots/bytes for the complete worst-case closure path. Receipts remain for -the stream incarnation; they are not silently evicted. Non-closure management -stops before consuming the closure reserve, and a full stream can still reach -`SEALED` for rebuild. Before enrollment publishes `OPEN`, before each generation -is admitted, and before resume republishes `OPEN`, the engine proves and -reserves the full remaining management-receipt count/byte envelope for that -generation's fold/block/correction/abort/requiesce/seal path. A current fold is -part of that reserve, not an optional later allocation. Status reports receipt -used/reserved/limit values. A new root/stream incarnation makes an old request a -binding mismatch. Thus a delayed retry from an earlier drain/resume cycle can -only return its receipt or a stale-revision error; it can never begin a later -cycle. This bounded receipt set and its worst-case serialized growth are charged -by the graph-global history budget in §4.5. +Each management receipt remains individually bounded by its versioned schema, +the row/memory limits of the transaction that stores it, and bounded response +pagination. B2a retains those receipts for the stream incarnation without an +aggregate capacity preflight or eviction. A new root/stream incarnation makes +an old request a binding mismatch. Thus a delayed retry from an earlier +drain/resume cycle can only return its receipt or a stale-revision error; it can +never begin a later cycle. History growth is an accepted storage cost, not an +admission signal. + +A claim sidecar may be removed without a receipt only after exact no-effect +classification. Once a claim creates a manifest/sentinel authority effect, the +sidecar remains until the same lifecycle CAS that adopts the achieved epoch +also persists its complete `ClaimReceipt` and makes +`current_claim_receipt_id` name it. The same claim ID with another +`claim_operation_digest` conflicts; after terminal publication, another +`terminal_effect_digest` conflicts. `claim_operation_digest` is immutable +across recovery attempts and is canonical over the logical lifecycle +operation, claim kind/profile, binding, invariant initial authority, and +contract versions. Each caller-visible claim invocation has a fresh +`attempt_id` and an `attempt_plan_digest` over its authenticated authority +prestate/tail, permitted intermediates, and planned sentinel. B2b additionally +binds its storage envelope. Reusing an attempt ID with another plan or terminal +effect digest conflicts. + +`attempt_effect_chain` is the complete ordered sequence of caller-visible +claim invocations and exact authority classifications for that one management +operation. The operation's execution retries remain bounded and typed; B2a +does not impose a lifetime count/byte cap on the retained receipt history. A +stock-Lance claim or materialization still returns no substrate receipt. The +OmniGraph receipt records only the manifest/sentinel facts required to recover +the lifecycle transition; it is not a physical-object inventory and does not +claim to enumerate randomized generation materialization attempts. B2a retains +every such receipt indefinitely. B2b may compact older receipts only through +the authoritative checkpoint protocol in §4.5.2. `SEALED` is immutable with respect to shard retention state. A new ordinary -claim, reclaim, or checkpoint is refused while sealed; the only epoch movement -from that state is the recovery-covered `StreamResume`, which consumes the old -proof and publishes `OPEN`. Rebuild reads the unchanged proof. Retention work -must finish while `OPEN` or `DRAINING`, and quiesce resolves every pending -claim/reclaim/checkpoint plus its lifecycle publication before constructing the -final proof. +claim or profile-specific retention operation is refused while sealed; the only +epoch movement from that state is the recovery-covered `StreamResume`, which +consumes the old proof and publishes `OPEN`. Rebuild reads the unchanged proof. +Retention work must finish while `OPEN` or `DRAINING`, and quiesce resolves +every pending claim plus any B2b reservation/reclaim/checkpoint and its +lifecycle publication before constructing the final proof. For every `DRAINING` row, `drain.expected_current_head_witness == current_head_witness` is an invariant. @@ -782,38 +964,60 @@ Quiesce requires a caller-minted non-nil `drain_id` and expected lifecycle revision, acquires the common stream-admission lease exclusively, runs the recovery barrier, waits for every watcher/retirement owner, and CASes `OPEN -> DRAINING` before a new physical drain effect. The durable descriptor -is the restart plan. Under it the worker claims a higher epoch, classifies and -folds each exact cut through recovery-v12 `StreamFold` drain mode, independently -proves no active/frozen/replay tail plus shard/base merge agreement, then CASes +is the restart plan. Under it the worker advances the epoch through the +selected retention profile's claim hook, classifies and folds each exact cut +through recovery-v12 `StreamFold` drain mode, independently proves no +active/frozen/replay tail plus shard/base merge agreement, then CASes `DRAINING -> SEALED` with the exact proof. It never holds a table queue while waiting for a fold that needs that queue. The management-receipt rules above make the exact operation retry-safe and reject a stale request instead of retargeting it. Restart continues `DRAINING` and never auto-opens it. -Every B2 epoch advance—cold open, quiesce, resume/abort, reclaim, and -checkpoint—uses one patched Lance claim envelope. Lance first persists a -claim-attempt record, conditionally writes the decoded-empty successor sentinel, -then CASes a shard manifest that names the sentinel position/digest and advances -the epoch. It durably terminals the attempt with the exact sentinel/manifest -receipt before returning success. Ordinary claims, including checkpoint claims, -bind the complete authenticated WAL/sentinel tail and conditionally create their -sentinel at checked `max_authenticated_position + 1`; unknown tail, collision, -or overflow refuses. They -preserve the prior -`replay_after_wal_entry_position`. They must replay/classify every object after -that cursor and may advance it only through the existing flush/coverage proof; -they never skip data-bearing WAL merely because the later sentinel is empty. -The sole claim-time cursor exception is §4.5's reclamation successor: only after -its quiescent whole-cut proof establishes no data-bearing tail may its manifest -advance the cursor to the new sentinel. Only the exact manifest-named claim may -proceed to replay, and no new put is admitted until that classifier permits it. -A crash before the CAS is same-attempt recovery state; a foreign object or -movement fails closed. This replaces RC.1's manifest-before-sentinel gap for B2 -and gives every claim recoverable sentinel authority. The open/admission barrier -resolves or refuses pending claim attempts before another claim. -Claim IDs, terminal receipts, retry horizons, and checkpoint compaction follow -§4.5; they are counted durable substrate state rather than an unbounded audit -log. +Every B2 epoch advance—cold open, quiesce, and resume/abort—uses a +profile-neutral claim contract. Before writer open, existing manifest/recovery +authority must durably bind the claim operation, exact authoritative prestate, +complete authenticated WAL/sentinel tail, every permitted authority +intermediate, and exact terminal classification. B2b additionally binds its +physical-growth envelope. Unknown authoritative tail, collision, overflow, a +foreign effect, or ambiguous authority movement fails closed. +Claims preserve the prior `replay_after_wal_entry_position`, replay/classify +every object after it, and may advance it only through the existing +flush/coverage proof. The open/admission barrier resolves or refuses every +pending claim before another claim or put. +If recovery must invoke a higher-epoch claim, it first appends that invocation's +fresh `attempt_id`, `attempt_plan_digest`, exact authority prestate, and +permitted effect states to the durable sidecar, and only then calls Lance. The +immutable `claim_operation_digest` does not change. Invocation retries obey a +bounded runtime policy; exhaustion returns typed `RecoveryRequired` and keeps +the sidecar. Finalization projects the complete classified sequence into the +individually bounded `ClaimReceipt`. B2a neither reserves nor meters physical +storage for those invocations. + +The two retention profiles supply that hook differently. The selected B2a +retain-all implementation may consume stock RC.1's manifest-first, +sentinel-second claim only when an existing RFC-022 recovery sidecar, armed +before invocation, can classify and finish the exact no-effect, manifest-only, +manifest-plus-sentinel terminal, and lost-result authority states under the +single-live-writer boundary. The sidecar—not the stock shard manifest—binds the +planned successor sentinel position/digest. An unclassifiable authority gap is +`RecoveryRequired`; it is never papered over by path inference. Unreferenced +randomized generation output is not shard authority: B2a retains it forever +and never adopts it, but does not require an attempt receipt or storage charge +for it. + +B2b instead uses the patched Lance claim-attempt/terminal-receipt envelope in +§4.5.2: Lance conditionally creates the successor sentinel at checked +`max_authenticated_position + 1`, then names it in the exact shard manifest +that advances the epoch. Only that manifest-named claim may replay. B2b's sole +cursor exception is the reclamation successor: only after a quiescent whole-cut +proof establishes no data-bearing tail may its manifest advance the cursor to +the new sentinel. Claim IDs and individually bounded terminal receipts are common durable +state. For B2a, the OmniGraph graph-manifest-authoritative `ClaimReceipt` is the +durable projection of the already exactly classified RFC-022 sidecar proof for +stock RC.1's manifest-first/sentinel-second effects; only then may the sidecar +be finalized. For B2b it projects and binds the patched Lance terminal receipt +and manifest-named sentinel. Reclaim/checkpoint IDs, retry horizons, and history +compaction are B2b-only. Drain mode is not the implemented B1 `OPEN` fold with a relaxed check. Its sidecar binds the complete expected `DRAINING` row and `drain_id`; publication @@ -833,11 +1037,14 @@ configuration, base witness, graph-branch topology, fixed actor/operation, an `OPEN` template, management receipt, and minimum next epoch floor. The exact achieved epoch is unknowable before the writer claim. After claiming, the adapter durably records `EffectsConfirmed` with the exact sentinel/epoch, -sentinel-naming shard manifest/replay cursor, replay disposition, and final -`OPEN` row; only that row may publish. Recovery +profile-specific `ClaimReceipt`, achieved shard manifest/replay cursor, replay +disposition, and final `OPEN` row; only that row may publish. B2b's achieved +manifest names the sentinel directly; B2a's stock manifest does not, so its +persisted receipt carries the sidecar's already-classified binding. Recovery never compensates an epoch or fence sentinel: while admission remains closed it -may claim a still-higher epoch, then records a new exact confirmation before -publication. An already-visible byte-identical `OPEN` row finalizes the +may follow the pre-authorized rule above to claim a still-higher epoch, then +records a new exact confirmation before publication. An already-visible +byte-identical `OPEN` row finalizes the sidecar; a divergent `OPEN` row, binding, witness, topology, or uncovered residue fails closed. @@ -861,40 +1068,47 @@ continue to the sealed proof. This rule is identical in §8 and the B2 gates. Minimum status is one authoritative manifest row plus a bounded cut-consistent physical observation. In the supported B2 profile it takes stream admission exclusively, read-only lists/classifies relevant recovery intents, settles every -writer/watcher/flush/retirement/reclaim owner to the status deadline, reads -lifecycle/token authority plus the shard witness, performs exact inventory, -and rereads those authorities before release. It never resolves recovery. +writer/watcher/flush/retirement owner plus every selected-profile claim owner to +the status deadline, reads lifecycle/token authority plus the authoritative +shard witness, and rereads those authorities before release. It never resolves +recovery. Movement returns typed `StatusChanged`; failure to settle returns typed `StatusBusy`. It reports lifecycle/binding/configuration, `lifecycle_revision`, authoritative and observed epoch, drain ID/goal/phase, -pending generation -rows/bytes, replay and merge cut, retained WAL/generation/orphan/unknown bytes, -strict block token/code/revision, correction and management receipt -spare/use/reserve/limits, the persisted `LastFoldSummary`, graph-global manifest -history used/pending/per-stream reserve/aggregate reserve/limits/source-bound -version, relevant recovery operation, and exact rebuild-readiness reasons. +pending generation rows/bytes, replay and merge cut, strict block +token/code/revision, persisted receipt-history counts and pagination cursors, +the persisted `LastFoldSummary`, relevant recovery operation, and exact +rebuild-readiness reasons. Mutable WAL cursor statistics are labeled hints and never used as receipts. A listing error, overflow, or unknown classification is -an explicit status error; the read-only status call does not mutate lifecycle. -Every later admission independently requires its own under-gate token-spare, -management-receipt closure, graph-history, and storage-meter/inventory preflight and refuses before -`put_no_wait` when any proof is unavailable. +an explicit diagnostic error only when the caller requested advisory retained- +object detail; it is not an admission or lifecycle-authority failure. Advisory +retained bytes are labeled as current-listing observations, never quota or +billing truth. The read-only status call does not mutate lifecycle. Every later +admission independently reruns the recovery, lifecycle, token, row, memory, and +backpressure gates; there is no B2a storage-meter or receipt-capacity preflight. Rebuild preflight requires `SEALED` and re-runs under closed admission plus schema/branch/table gates. The sealed proof must still match the exact -base/shard state; no relevant sidecar, active/frozen/replayable tail, unmerged -generation, unattributed winner, unresolved token, or uncovered inventory may -remain. Export/cutover re-runs the proof. A prior status response is never an +base/shard state; no relevant sidecar, active/frozen/replayable authoritative +tail, unmerged generation, unattributed winner, or unresolved token may remain. +Unreferenced retained materialization output is inert and does not block the +proof. Export/cutover re-runs the proof. A prior status response is never an authority token. `verified_empty_digest` is domain-separated over the exact binding, -configuration, stream incarnation, base witness, ordered shard-manifest/ -generation inventory, replay/merge cursors, retained/unknown classes, and every -decoded empty WAL fence sentinel. A sentinel may remain beyond the authoritative +configuration, stream incarnation, base witness, ordered authoritative shard- +manifest/referenced-generation state, replay/merge cursors, and every decoded +empty WAL fence sentinel. It excludes advisory listings, unreferenced retained +objects, and storage-byte estimates. A sentinel may remain beyond the authoritative replay cursor only when its exact position/digest/epoch is authenticated by the -retention checkpoint plus retained claim/manifest successor chain. Exactly one -sentinel for the achieved current epoch must remain, be decodable, and be named -by the latest shard manifest. Older authenticated sentinels from the same +selected profile's durable chain: B2a uses its OmniGraph +graph-manifest-authoritative `ClaimReceipt` set plus the stock manifest +sequence, while B2b uses its +retention checkpoint plus retained claim/manifest successor sequence. Exactly +one sentinel for the achieved current epoch must remain, be decodable, and be +named by `current_claim_receipt_id`; B2b additionally requires the latest shard +manifest to name it. Older authenticated sentinels from the same shard/enrollment and an epoch no greater than that floor are harmless fence history and remain included in the digest until reclamation removes them. A data-bearing entry beyond the cursor, malformed or unauthenticated sentinel, @@ -963,32 +1177,17 @@ correction_id)` with plan digest, actor, fixed graph commit/result, and final lifecycle/token digest. A retry checks that receipt before declaring the block stale: exact ID/digest returns the recorded result, while the same ID with a different digest is `StreamIdempotencyConflict`. Receipts are rare operator -records, not per-ack admission history. Stream-config v3 contains hard -`max_correction_receipts`, `max_correction_receipt_bytes`, and source-derived -`max_single_correction_receipt_bytes` values. Before the first WAL effect of -each one-live-generation admission, the engine checks under stream admission -and the token gate that persisted receipt usage plus one slot and the full -single-receipt byte bound still fit. That spare cannot be consumed by another -operation while the generation is live: correction requires its exact blocked -generation and admission is closed. Cold replay reconstructs the same spare -from the persisted generation plus current receipt usage. If it cannot be -proved, new admission returns typed `CorrectionReceiptCapacityExceeded` before -`put_no_wait`, while fold/quiesce/rebuild remain available. - -Under the token gate, a correction transfers that generation's spare to the -exact canonical receipt before sidecar arm. The source-derived per-receipt -maximum must cover every legal canonical receipt; exceeding it is a fail-closed -format/invariant violation that the activation matrix proves unreachable, never -an ordinary post-admission capacity outcome. A successful fold releases the -unused logical spare before another generation may admit. -The already-reserved blocked generation therefore always has room for one -terminal correction even when the retained-receipt totals then reach their -caps. Receipts are retained for the stream incarnation. Once no generation or -block remains, full totals stop new admission and a quiesce can still reach -`SEALED` for export/rebuild into a new root/new stream incarnation. Status -reports used, reserved, and limit values. Public defaults require storage -evidence before activation; an unspecified, advisory, or admission-only check -without the terminal spare is invalid. +records, not per-ack admission history. Their canonical schema and the bounded +correction operation impose a source-derived maximum on each individual +receipt, and the token-table transaction that stores one remains subject to the +ordinary row/memory limits. B2a retains every receipt for the stream +incarnation, indexes it for exact idempotency lookup, and paginates history; it +does not reserve a slot, enforce aggregate receipt count/bytes, or stop row +admission because history has grown. A correction arms its exact receipt and +token effects in recovery before either can become manifest authority. If the +backing store cannot persist that bounded record, the operation follows normal +typed storage/recovery semantics and admission remains closed around the +blocked cut; no row is silently discarded. Correction targets only keys whose LWW winner is in that blocked cut; adding a new key is out of B2. It supports two explicit actions: @@ -1027,7 +1226,7 @@ applies one bounded keyed-upsert batch with at most one image per resulting then publishes at most 8,192 resulting `PRESENT | WITHDRAWN` current-token rows in deterministic chunks and the one `CorrectionReceipt` row in a final separate transaction on that same dataset. Each transaction independently obeys the -sealed 8,192-row/32-MiB bound; the receipt transaction also obeys the stricter +sealed 8,192-row/32-MiB bound; the receipt transaction also obeys the source-derived single-receipt byte bound. Stream-config v3 fixes a source- derived `max_correction_token_transactions`, and admission proves that the worst-case token projection for the complete legal generation plus the receipt @@ -1055,7 +1254,42 @@ plain `stream resume` continues to accept only `SEALED`. The fold/correction CAS never opens admission itself. Skip-invalid, implicit drop, whole-generation discard, base-row delete, and direct `DRAINING -> OPEN` CAS are forbidden. -### 4.5 Reclamation decision: own the missing Lance primitive +### 4.5 Retention profiles + +#### 4.5.1 B2a unbounded retain-all/no-GC profile (selected) + +B2a deletes no MemWAL object after fold, optimize, cleanup, restart, +correction, quiesce, or `SEALED`. Referenced generations, WAL files, historical +fence sentinels, and unreferenced partial or complete randomized +materialization subtrees all remain in the source root indefinitely. The +profile intentionally has no MemWAL file/object/byte limit, retained-storage +ledger, quota, admission watermark, physical-output estimator, storage +reservation, or materialization-attempt receipt requirement. It does not +promise bounded disk/object-store usage or indefinite service under finite +provider capacity. + +This changes only the physical-retention contract. Row and memory admission +remain bounded at one no-roll generation of at most 8,192 rows and 32 MiB of +logical Arrow data; worker count, queues, deadlines, execution retries, and +fold/correction transaction shapes remain explicitly bounded. Every +acknowledgement still requires watcher durability plus the same-writer fence +check. Every authority-changing effect still uses the existing manifest and +recovery protocols, and unresolved authority ambiguity still blocks progress. +No path, listing result, timestamp, or compatible-looking object may establish +ownership. Unreferenced physical residue is simply inert: it is retained, +never scanned as a generation, never adopted, and never deleted. + +Gate R0's first two findings in §0.2 therefore become accepted operational +limitations rather than activation blockers. Its third finding is closed by +the dense-scanner compaction repair. Current-object inventory remains useful as +an advisory regression instrument, but it is not correctness authority, +provider billing truth, or admission input. A store-capacity failure before a +put is a typed storage refusal; after invocation it follows the existing +`AckUnknown`/recovery classification. Neither case permits data loss or false +acknowledgement. Export/rebuild into another root remains an operator tool, not +a capacity promise or mandatory automatic lifecycle. + +#### 4.5.2 B2b managed reclamation: own the missing Lance primitive The RC.1 audit produces an explicit no-go for physical reclamation through the stock public surface: @@ -1076,7 +1310,7 @@ stock public surface: though an explicit fence check returns `PeerClaimedEpoch`; and - `FlushedGeneration` carries generation/path but no per-generation WAL range. Only the shard-wide replay cursor exists, so a WAL prefix is reclaimable only - at B2's quiescent whole-cut boundary. `wal_entry_position_last_seen` is never + at B2b's quiescent whole-cut boundary. `wal_entry_position_last_seen` is never substituted. Private B1 contains the clean-acknowledgement face of that RC.1 gap at the @@ -1088,15 +1322,13 @@ post-invocation `AckUnknown`, and the worker retires while its possible durable residue remains replayable. This containment is deliberately narrower than the required Lance change. It does not erase stale WAL bytes, make a deleted fence sentinel safe, protect raw Lance MemWAL callers, provide cross-process -seal/failover, or authorize any `_mem_wal` deletion or public B2 surface. +seal/failover, or authorize any `_mem_wal` deletion or public B2b surface. OmniGraph therefore does not raw-delete `_mem_wal` objects, route them through -`delete_unverified`, compact shard-manifest history, or wait passively for an -upstream release. The next substrate slice authors a narrow Lance patch, -proposes it upstream, and may pin OmniGraph immediately to the exact reviewed -commit while upstream review proceeds. Deletion stays inside Lance; when the -change is released upstream, the pin returns to the canonical revision rather -than preserving a product fork. +`delete_unverified`, or compact shard-manifest history. B2b requires the narrow +Lance-owned primitive below; an upstream proposal remains useful, but its +calendar does not block the separate OmniGraph-only successor evidence gate for +B2a. No temporary B2a work may impersonate reclamation or delete a path. Stock RC.1 exposes only evidence-level raw object listing plus manifest reads, not the owned classified inventory below. The required public Lance shape is @@ -1177,7 +1409,7 @@ from the attempt/successor/receipt plus fresh inventory, never optimistic process memory. Attempt and receipt records remain counted and retained through the supported idempotency/recovery horizon. -Pending reclaim attempts are part of the mandatory B2 open/admission barrier, +Pending reclaim attempts are part of the mandatory B2b open/admission barrier, not optional maintenance metadata. Before any stream open, status, put, fold, quiesce, resume, correction, rebuild preflight, or later reclaim may claim a writer or touch shard state, it classifies the exact attempt set. Mutating @@ -1193,8 +1425,9 @@ complete expected lifecycle row, exact Lance attempt/plan, and fixed post- receipt shard epoch/manifest/cursor/inventory outcome before the first physical effect. After Lance terminals the receipt, the operation records `EffectsConfirmed` and conditionally publishes the exact epoch floor, shard -witness, retained-byte classes, lifecycle revision, and graph-history charge -while preserving the lifecycle state and any drain identity/goal. A crash in between is roll- +witness, retained-byte classes, and lifecycle revision while preserving the +lifecycle state and any drain identity/goal. A separately accepted bounded-root +amendment may add its own graph-history charge; base B2b does not imply one. A crash in between is roll- forward-only and blocks another writer or final sealed proof. Quiesce constructs `SealedProof` only after every such sidecar and substrate attempt is settled, so the proof cannot be invalidated by later retention work. @@ -1244,13 +1477,13 @@ new checkpoint epoch expires every old reclaim and ordinary-claim ID: retry returns typed `ReceiptExpired` or `ClaimReceiptExpired`, respectively, and the same UUID in the new epoch is a different identity. -There is no absent-pointer bootstrap mode. B2 enrollment or sealed rebuild asks +There is no absent-pointer bootstrap mode. B2b enrollment or sealed rebuild asks the patched Lance initializer to create an immutable genesis checkpoint body with empty claim/reclaim summaries and conditionally install its bootstrap -pointer **before** committing the B2 MemWAL details/index kind. The enrollment +pointer **before** committing the B2b MemWAL details/index kind. The enrollment recovery envelope binds the genesis body, pointer, details commit, empty shard, and final lifecycle publication; a crash before graph publication either -completes that exact prefix or refuses foreign/malformed residue. The first B2 +completes that exact prefix or refuses foreign/malformed residue. The first B2b reader therefore always starts from an authoritative body, while a body/pointer left before the details commit is classified enrollment-owned state rather than silently adopted. @@ -1263,8 +1496,9 @@ shard state. Checkpoint activation never strands an already-open old reader on the legacy kind. Genuine patched↔RC.1 and graph-v8↔v9 tests pin genesis body/pointer/details publication, every later checkpoint crash boundary, ID expiry, and refusal to fall back to `version_hint`. If the bootstrap, complete -summary, ID expiry, or bounded compaction cannot be proved, public B2 remains -inactive; periodic rebuild is not a substitute for an unbounded or +summary, ID expiry, or bounded compaction cannot be proved, public activation +under B2b remains inactive; periodic rebuild is not a substitute for an +unbounded or unrecoverable MemWAL namespace. Even after that namespace is bounded, the separate whole-root lifetime limit below still applies. @@ -1312,22 +1546,22 @@ not reconstruct them from paths. Exact orphan/unknown accounting additionally requires a backend capability that guarantees HEAD/GET and LIST visibility after successful PUT **and** DELETE for -the `_mem_wal` namespace. If that capability is absent, public streaming is -refused unless the Lance patch provides its own durable complete object +the `_mem_wal` namespace. If that capability is absent, public activation under +B2b is refused unless the Lance patch provides its own durable complete object accounting that does not depend on a possibly stale listing. That accounting includes incomplete multipart uploads, which ordinary object LIST does not return. Current-key visibility is still insufficient on a versioned, soft-delete, or Object-Lock/retention-enabled namespace: noncurrent versions, delete markers, retained locked objects, and incomplete uploads may continue to -consume billed storage after the current key disappears. The initial public B2 +consume billed storage after the current key disappears. The initial public B2b profile refuses those configurations unless the patched backend can enumerate and byte-account every retained version/marker/lock, permanently delete each eligible version, and keep ineligible retained bytes charged until verified removal or expiry. A successful best-effort current-object list is evidence for the negative RC.1 guards, not a production completeness or cost proof. -Public activation also requires a hard **admission watermark**, not a maximum -observed in a benchmark and relabeled as a guarantee. Exact cold inventory +Public activation under B2b also requires a hard **admission watermark**, not a +maximum observed in a benchmark and relabeled as a guarantee. Exact cold inventory counts every object/byte under `_mem_wal`: WAL, referenced generations and PK/Bloom sidecars, recognized orphans, shard-manifest history/checkpoints, ordinary claim and reclaim attempt/receipt records, and malformed/unknown @@ -1392,7 +1626,7 @@ durable upload identity and charged byte/object allowance. Same-plan recovery must complete or explicitly abort it before another materialization attempt; the retained-state inspector must list/account for it through the backend's multipart API even though ordinary LIST cannot. A backend without exact -multipart accounting/abort semantics cannot enable public streaming. The +multipart accounting/abort semantics cannot enable B2b. The local/RustFS/S3-compatible matrix repeatedly crashes before multipart complete and proves retained charged bytes plus attempts remain within the envelope. @@ -1418,116 +1652,40 @@ binding ledger but from the non-admission pool. Row admission stops with typed `ControlHeadroomExceeded` before it consumes the emergency floor; same-ID recovery, status, fold/correction, quiesce, reclaim, checkpoint, and sealed rebuild stay available. If Lance cannot prove these history/orphan limits, the -reserve-first ledger, or forward-progress headroom, B2 must stop before public +reserve-first ledger, or forward-progress headroom, B2b must stop before public activation rather than discover at runtime that a full WAL cannot be reclaimed. Local and RustFS measurements across row/byte caps, wide schemas, blobs, PK/Bloom output, fragmented batches, crashes, and retries validate the estimator/quota and catch regressions; they do not create the bound. Arrow or RSS bytes are not disk bytes. A patch that can only report measured growth, or -that cannot checkpoint its own history, does not satisfy B2. - -The per-binding watermark bounds only the physical MemWAL namespace. It does -not bound logical base/token data or their Lance version histories. The shared -`__manifest` history is a separate graph-global finite resource: another stream -or an ordinary graph writer can grow it, so a per-stream publication counter -cannot guarantee that every enrolled stream still has an escape. - -Internal schema v9 therefore adds one manifest-authoritative singleton -`GraphHistoryBudget` for the root. It carries the root/history epoch, every -manifest publication charged since rebuild, a conservative source-derived -physical-growth charge, hard publication/byte limits, the sum of each enrolled -stream's remaining closure obligation, the last finalized budget revision, and -the last finalized budget operation/delta digest. -Each state-v2 stream row carries its own remaining closure reservation; the -singleton aggregate and the set of stream rows must agree in the same CAS. -There is no in-memory or side-dataset budget authority. - -Every publisher of `__manifest` participates—not only stream code. Mutation, -Load, SchemaApply, BranchMerge, branch controls, EnsureIndices, Optimize, -Repair, Cleanup, recovery, enrollment, fold/correction/lifecycle/retention, and -future writers all acquire the graph-history gate and charge the exact logical -publication plus its conservative physical-growth maximum. The complete crash- -safe authority is the committed singleton **plus** globally relevant standard -RFC-022 recovery sidecars. Under the history gate, a writer that may need a -later manifest CAS writes its sidecar with the exact pending budget charge -before its first data effect. The charge binds the history epoch, expected -budget revision, estimator/limits version, operation identity, exact singleton -delta, per-stream reserve delta if any, and conservative object/byte envelope; -same operation with another digest conflicts. This does not require or pretend -there was an extra budget-only manifest CAS. Every publisher's root recovery -barrier sums or resolves those pending charges exactly once. Final manifest -publication applies the bound delta and records its operation/digest in the -singleton; a lost CAS result classifies that exact marker and never charges it -twice. Effect-free finalization removes the pending charge. A normal publication -is admitted only when its charge leaves -the aggregate closure floor untouched. A closure publication may consume only -the named stream's reservation, atomically reducing that stream and the -singleton aggregate; it cannot borrow another stream's reserve. - -The initial B2 profile holds the graph-history gate from that final -reserve/sidecar arm through every owned physical effect, the manifest CAS, and -budget-charge finalization. There is therefore at most one live expected budget -revision; another publisher waits and then restarts its final validation from -the new singleton, while crash recovery resolves the sole pending charge before -releasing the gate. This deliberately serializes graph publication. A future -concurrent-reservation protocol needs a separate accepted composition proof; B2 -does not let multiple pending deltas speculate from one revision. - -The byte charge is a physical retained-storage claim, so it includes every -Lance object a `__manifest` attempt can create: data fragments, version/ -transaction manifests, maintained-index objects, multipart uploads, standard -recovery-sidecar overhead, and failed-CAS/materialization residue. The manifest -adapter obtains a source-derived maximum before materialization, fixes the -Lance transaction/output identity and a hard maximum attempt count in the -sidecar, and either reuses deterministic output or retains the full next-attempt -allowance. Even a manifest-only metadata writer must persist this minimal -standard reservation sidecar before Lance can materialize its CAS attempt, -unless Lance itself supplies an equivalent durable attempt/receipt consumed by -the same barrier. Cold open inventories and classifies the complete manifest namespace -plus pending multipart state; unknown or unreserved output closes all writers. -Settlement moves only observed materialized bytes from reserved to retained, -without double counting, and releases bytes only after an exact Lance-owned -cleanup/rebuild receipt plus fresh inventory. A successful-publication counter -without this reserve-first materialization envelope is depth evidence, not byte -headroom, and does not activate B2. - -Enrollment reserves the complete worst-case closure path for the new stream. -Before a stream admits a generation, resumes to `OPEN`, or begins non-emergency -management, it must replenish and prove that stream's full remaining obligation: -fold or strict block, one terminal correction, abort-drain when the goal is -`OPEN_AFTER_FOLD`, a second quiesce/claim, and final `SEALED` publication, -including the required management receipts and pending recovery charges. The -aggregate is dynamic across **all** enrolled streams, so two blocked tables at -the floor each retain their own correction and seal path. Closing one cannot -consume the other's reserve. The worst-case number/bytes are derived from the -fixed state machine and source-level maximum manifest-row/transaction growth; -configuration cannot lower them. - -A successful `SEALED` publication releases that stream's unused closure -reservation in the same CAS. Any transition back to `OPEN` must first reserve -the full applicable OPEN obligation; if it cannot, it remains closed and -returns `GraphRebuildRequired`. A successful fold may leave an idle stream -`OPEN` with only its shorter no-generation closure reserve, but no next -generation is admitted until the full generation envelope is replenished. - -When an ordinary charge would cross that shared floor, new work returns typed -`GraphRebuildRequired` before effect. Status reports graph-global used, -pending, per-stream reserved, aggregate reserved, limit, and source-bound -version. Operators close every enrolled stream and export/rebuild into a fresh -root. In-place SchemaApply, Optimize, and Cleanup on an enrolled table remain -refused until a later accepted drain/maintenance strand. Indefinite in-place -service requires Phase D or another accepted manifest-retention design; B2 does -not market a bounded MemWAL as a whole-graph storage bound. Public B2 remains -closed until this graph-global charge and its physical-growth upper bound are -implemented and evidence-green; an advisory counter or per-stream-only reserve -does not satisfy the escape invariant. - -These contracts close the design question without pretending RC.1 can already -execute it. B2 implementation and every public surface remain closed until the -reviewed Lance primitive, post-success fence fix, enforced admission watermark, -bounded MemWAL history checkpoint, graph-global manifest-history budget, and -crash matrix are pinned and green. +that cannot checkpoint its own history, does not satisfy B2b. + +#### 4.5.3 Deferred bounded-storage accounting (not a B2a gate) + +The earlier B2-0 design proposed a graph-global `GraphHistoryBudget`, physical- +growth reservations for every manifest publisher, per-stream closure reserves, +and hard aggregate receipt/storage caps. That design answered a different +product promise: continue operating indefinitely inside a finite, enforced +storage envelope. The selected unbounded retain-all profile makes no such +promise, so none of those mechanisms is part of B2a's schema, gate order, +admission path, recovery payload, error surface, or activation criteria. + +This is not permission to make process memory or individual operations +unbounded. B2a keeps the row, logical-Arrow-byte, worker, queue, deadline, +transaction-chain, response-page, and retry bounds specified elsewhere. It +also keeps every correctness obligation: durable acknowledgement, exact +recovery ownership for authority effects, one manifest visibility point, +compare-and-chain tokens, trusted attribution, revision-fenced lifecycle, and +strict correction. Only cumulative physical storage and append-only protocol +history are deliberately unmetered. + +If OmniGraph later promises bounded retained storage, automatic reclamation, +or indefinite in-place operation under a configured quota, that feature needs +a new accepted amendment and evidence at that boundary. B2b's Lance-owned +reclamation/accounting work in §4.5.2 is the current research direction; it is +not a prerequisite for B2a and cannot be smuggled into B2a through raw deletes +or an advisory counter. ### 4.6 Public surface after the gates close @@ -1569,8 +1727,6 @@ new admission attempt. Exact JSON `status` values are `durable`, `stream_input_too_large`, `stream_binding_changed`, `stream_lifecycle_changed`, `stream_authority_changed`, `stream_sequence_conflict`, `stream_idempotency_conflict`, `stream_fold_required`, -`retained_storage_limit_exceeded`, `control_headroom_exceeded`, -`correction_receipt_capacity_exceeded`, `graph_rebuild_required`, `stream_backpressure`, `recovery_required`, and `stream_retry_required`; CamelCase names below denote the corresponding engine error/disposition types. An unresolved recovery found by the request-level barrier before any body line @@ -1609,15 +1765,11 @@ binding. OpenAPI makes presence variant-specific rather than nullable-by-habit. corresponding typed error, with no fabricated attempt. Lifecycle change covers `OPEN -> DRAINING/SEALED` between request runs; authority change covers an exhausted pre-invocation reprepare after schema/main/token movement; -- `stream_fold_required`, `retained_storage_limit_exceeded`, - `control_headroom_exceeded`, `correction_receipt_capacity_exceeded`, and - `graph_rebuild_required` are effect-free admission refusals. They carry the - exact relevant usage/limit snapshot and no attempt; `stream_fold_required` - means the next legal row/run fits an empty generation but not the bounded - resident generation and must be folded before retry, while - `graph_rebuild_required` means the shared root publication budget has reached - the aggregate reserved closure floor. The management/rebuild escape in - §4.3–§4.5 remains available for every capacity result; +- `stream_fold_required` is an effect-free admission refusal with no attempt. + It means the next legal row/run fits an empty generation but not the bounded + resident generation and must be folded before retry. B2a defines no retained- + storage, control-headroom, aggregate-receipt-capacity, or graph-history-budget + refusal; - `stream_backpressure` is an effect-free admission/queue deadline reached before `put_no_wait`; it carries retry guidance but no durability claim or attempt; @@ -1744,28 +1896,14 @@ enroll, block inspection, correct, and rebuild-preflight use the separate `stream_manage` action. Status is authorized like other graph operational metadata. The same engine gates apply to embedded and remote CLI use. -`GraphRebuildRequired` is graph-wide, so it is part of the ordinary writer -contract too. The engine/SDK error carries history epoch and budget revision, -used and pending publication/byte charges, a bounded per-stream reservation -summary for streams relevant to this operation (empty when none, truncated only -with an explicit summary digest/count), aggregate closure reserve, hard limits, -source-bound version, and the explicit -`quiesce_all_and_rebuild` action; it never suggests retrying ordinary work in -the same root. A streaming request that already emitted rows uses the per-line -`graph_rebuild_required` variant above. Any other HTTP write refuses before its -first durable effect with HTTP 409 and structured `graph_rebuild_required`; -CLI JSON prints the same payload and human output explains that reserved -quiesce/correction/`SEALED` operations remain available. SDK, server, CLI, and -OpenAPI parity tests pin both request-level and partial-NDJSON forms. - Phase B2 initially exposes only strict compare-and-chain upsert. Public activation requires the minimum `status`, explicit `fold`, persistent `quiesce`, `resume`/abort-drain, rebuild preflight, a bounded strict-correction workflow, durable authenticated-contributor attribution, a same-key `AckUnknown` -sequencing/idempotency contract, and a proved WAL/generation reclamation rule. +sequencing/idempotency contract, and one proved physical-retention profile. Minimum status includes lifecycle and binding/revision, current epoch, pending generations/bytes, last fold error, -receipt and graph-history budgets, and whether strict fold is blocked. It never +receipt-history pagination, and whether strict fold is blocked. It never claims to resolve one caller's `AckUnknown`. Dead-letter row identity and operation, richer status, and configurable policy remain Phase C. Automatic operation-scoped drain, SchemaApply/branch @@ -1784,12 +1922,15 @@ acknowledgement. B1 does not call an external embedding provider or synthesize unspecified fold-derived fields. Any future derivation step requires its own contract and evidence and must complete before the one bounded table effect. -B2 adds one mandatory substrate prelude before any writer claim: under stream -admission it classifies the complete Lance reclaim-attempt set from §4.5. -Mutating entry points finish the sole exact pending plan or refuse, while -read-only status reports it; patched Lance independently refuses -`mem_wal_writer` when an attempt remains pending. No token check, same-key queue, -or final pre-put gate can bypass that prelude after restart. +B2 adds one mandatory recovery prelude before any writer claim: under stream +admission it resolves or refuses every pending claim/recovery operation relevant +to the binding. B2a has no reservation or physical-inventory prelude. B2b must +additionally resolve the Lance claim/reclaim/checkpoint attempts from §4.5.2. +Mutating entry points finish the sole exact pending authority plan or refuse, +while read-only status reports it. Under B2b, patched Lance also refuses +`mem_wal_writer` while a substrate attempt remains pending. No token check, +same-key queue, or final pre-put gate can bypass this recovery prelude after +restart. Phase B1 admits exactly one non-empty normalized `RecordBatch` per Lance `put_no_wait`. Each call and the **entire active generation**, including any @@ -2150,9 +2291,10 @@ inventory classifies an unreferenced recognized `{hash}_gen_N/` subtree under the bound shard—complete or partial—as derived orphan output: it is never adopted as a generation, never scanned or folded, and never deleted by B1. It does not make the durable WAL residue unrecoverable. Any loose object outside -that recognized generation-output subtree shape still fails closed. B2's -accepted reclamation protocol must prove when these orphans can be removed and -count them toward its enforced retained-storage admission watermark. +that recognized generation-output subtree shape still fails closed. B2a keeps +these orphans inertly and unmetered for the root's lifetime. Only a future B2b +Lance-owned reclamation protocol may account for them and prove when they can +be removed. The fold excludes generations already covered by the base table's exact `merged_generations` and requires exactly one remaining B1 generation. It @@ -2212,9 +2354,9 @@ generation into multiple base-table keyed transactions. Because `_stream_tokens.lance` is one graph-global physical participant, B2 adds one root-shared stream-token gate. The universal order for a manifest -publisher is `sorted relevant stream admission -> graph history -> schema -> -main branch -> stream token -> sorted graph tables`; a writer with no relevant -stream admission starts at graph history, and no caller acquires a newly +publisher is `sorted relevant stream admission -> schema -> main branch -> +stream token -> sorted graph tables`; a writer with no relevant stream +admission starts at schema, and no caller acquires a newly discovered stream lease after that gate. Every fold or correction captures the token dataset's exact manifest-selected version, transaction UUID, and e_tag in its `ReadSet`; it never opens raw token HEAD as @@ -2227,7 +2369,7 @@ EnsureIndices, Optimize, Repair, Cleanup, recovery, and future writers all resolve or refuse it before base capture even when its graph table is disjoint. Quiesce and rebuild preflight use the same barrier. If final relisting discovers a late global sidecar, stream lifecycle, or required admission lease, the caller -releases every admission/history/schema/branch/token/table gate and restarts +releases every admission/schema/branch/token/table gate and restarts from the root barrier; it never recovers a disjoint global sidecar while retaining one caller table's gate. This prevents an unmanifested token HEAD from being buried by an ordinary graph commit. @@ -2259,9 +2401,10 @@ MemWAL generation GC can start only after the exact fold and token outcome are graph-visible, its sidecar is resolved, index catchup permits reclamation, and no `FreshReadCut` or retained-version guard references the generation. Data-HEAD merge progress alone is never permission to delete the only -fresh-tier copy. B1 deliberately performs no GC; B2 consumes only the -Lance-owned inspect/plan/execute primitive and enforced admission watermark -specified in §4.5. +fresh-tier copy. B1 deliberately performs no GC. B2a likewise deletes nothing +and retains the residue indefinitely without metering it; B2b consumes only the Lance-owned +inspect/plan/execute primitive and enforced admission watermark specified in +§4.5.2. Concurrent folders reload `merged_generations`: a generation already committed is skipped; otherwise the fold is replanned from current state. The explicit @@ -2499,16 +2642,17 @@ those limits wherever the tier is exposed. Phase B1 keeps observability internal and does not create a public status contract. The private implementation exposes test seams at its durability, replay, fencing, resource, cut, fold, visibility, and recovery boundaries. The -accepted local cost/RSS and configured-RustFS cost evidence in §12.3 supports -the bounded private profile; it is not a public latency SLO, group-commit -multiplier, or claim that every metadata term is history-flat. +2026-07-21 dense-scan repair and near-cap/RSS cell in §12.3 re-prove closure for +the widest admitted shape. That evidence is not a public latency SLO, +group-commit multiplier, current object-store result, physical-storage bound, +or claim that retained metadata work is history-flat. B2's minimum status surface is the authority-plus-observation contract in §4.3. It includes the exact lifecycle/binding, active epoch, drain operation, -pending generation/row/byte totals, retained/unknown bytes, merged progress, -last fold outcome, strict block, correction/management-receipt spares, -lifecycle revision, graph-global history used/pending/per-stream and aggregate -closure reserves, relevant recovery, and rebuild readiness. +pending generation/row/byte totals, merged progress, last fold outcome, strict +block, receipt-history pagination, lifecycle revision, relevant recovery, and +rebuild readiness. Optional retained-object counts/bytes are advisory current- +listing diagnostics, not admission or provider-billing truth. Richer index-catchup, age, and reject detail may follow in Phase C. Lance's `wal_entry_position_last_seen` and next-position statistics are explicitly labeled hints; neither is presented as a durable per-row receipt or a way to @@ -2518,11 +2662,11 @@ Before B2 activation, internal metrics must cover ack latency, durability-wait batching, fenced writers, replayed entries, fold rows/bytes/generations, fold retries, lag, reject counts, and sidecar recovery. They also cover compare-and-chain conflicts/idempotent hits, token-table -effects, correction outcomes, retained object/byte classes, reclaim -attempt/receipt outcomes, reservation/control-ledger state, admission-watermark -refusals, and finite-lifetime rebuild-required refusals. -Defaults for every byte/count/time bound must be documented, and configuration -changes are observable behavior. +effects, correction outcomes, retained object/byte diagnostics, and selected- +profile claim outcomes. B2b additionally covers its reservation ledger, +admission watermark, reclaim/checkpoint attempts, and receipts. Defaults for +every active row/memory/count/time bound must be documented, and configuration +changes are observable behavior. B2a has no storage-limit default to report. The accepted acknowledgement-cost instrument scopes a warm, already-claimed writer in steady state and includes both WAL append and synchronous in-memory @@ -2571,36 +2715,18 @@ now passes (§12.3). V8 remains a private-core format with no public B2 caller. B2 is a third strict strand: internal schema v9, stream-state protocol v2, stream-config v3, and recovery-v12. V9 adds the reserved nullable stream-row metadata to stream-capable physical schemas and initializes the -manifest-selected `_stream_tokens.lance` authority. Root initialization also -creates the singleton `GraphHistoryBudget` at history epoch zero with the exact -source-bound version, baseline retained manifest inventory, zero pending -charges, and closure reserves for every enrollment published in that same -initial outcome. The genesis publication is self-accounting: before any Lance -materialization, the initializer writes a deterministic bootstrap attempt with -the precomputed conservative maximum for its own sidecar, fragments, -manifest/version/index objects, multipart state, and bounded retries. Configured -limits must fit that maximum plus the initial closure aggregate or init refuses -without opening the root. The genesis row records publication one and charges -the **full conservative bootstrap maximum** as retained use; it never attempts -to observe and settle its own publication downward inside that same CAS. Only a -later exact cleanup/rebuild receipt may reduce the charge. Crash/lost-result -recovery either proves that exact singleton/transaction or keeps the root -unopened. No other v9 manifest -publication may precede that authority. A -compatible open validates singleton uniqueness, internal-schema/config/source- -bound versions, committed counter/inventory, unresolved sidecar charges, and -the exact sum of all per-stream closure reservations before any writer runs; -missing, duplicate, or mismatched budget state is uncovered format corruption, -not a default. V8 rows, lifecycle payloads, -and private acknowledgements are never reinterpreted or assigned contributor/ -token evidence with serde defaults. The implementation must prove genuine -v8↔v9 refusal plus export/init/load rebuild before any production caller exists. -Graph-history publication/byte limits and their estimator version are root-init/ -export-rebuild choices in B2; they cannot be lowered or reinterpreted in place. -A later dynamic configuration change requires its own accepted recovery-covered -design that re-proves every committed/pending charge and closure reservation. -These version assignments are part of the B2 contract; this design amendment -does not activate them in code or change the currently served v8 format. +manifest-selected `_stream_tokens.lance` authority. It does **not** add a +`GraphHistoryBudget`, storage-meter singleton, aggregate receipt cap, or +physical-retention quota. Root initialization and first enrollment still use +the normal recovery-covered publication protocol for their actual authority +effects; unreferenced physical residue remains retained and inert. + +V8 rows, lifecycle payloads, and private acknowledgements are never +reinterpreted or assigned contributor/token evidence with serde defaults. The +implementation must prove genuine v8↔v9 refusal plus export/init/load rebuild +before any production caller exists. These version assignments are part of the +future B2 contract; this design amendment does not activate them in code or +change the currently served v8 format. Old binaries refuse a stream-capable graph before reading or writing any table. A stream-capable binary refuses an older graph before running recovery. On a @@ -2787,8 +2913,11 @@ The Phase-B1 implementation landed on 2026-07-19. It is reachable only through one `#[doc(hidden)]`, feature-gated engine seam; there is no schema syntax, ordinary SDK method, HTTP/CLI/OpenAPI route, or production first-use caller. This section distinguishes implemented behavior from checked evidence and -the accepted private boundary. All Phase-B1 acceptance gates passed on -2026-07-19. “Green” does not mean public activation, and the RFC remains draft. +the then-accepted private boundary. All then-declared Phase-B1 gates passed on +2026-07-19. Gate R0 then exposed one legal near-cap closure failure; the +2026-07-21 dense-scan repair and remeasurement close that failure and requalify +the one-generation B1 boundary. “Green” never meant public activation, and the +RFC remains draft. **Implemented private core** @@ -2846,7 +2975,10 @@ the accepted private boundary. All Phase-B1 acceptance gates passed on base-dependent validation, and stages one exact-ID upsert with `MergedGeneration` in the same Lance transaction. It folds already-normalized physical rows—including physical vector columns—and performs no external - embedding call or unspecified fold-derived-field materialization. One + embedding call or unspecified fold-derived-field materialization. Scanner + output is row/byte checked and copied into dense owned Arrow arrays before it + is retained, so sparse variable-width slices cannot charge or pin their full + backing buffers through the fold. One manifest CAS publishes the achieved table pointer, next lifecycle witness, and fixed lineage; default reads remain old until that CAS. - Resource reservations cover queued input, the resident generation, and the @@ -2859,7 +2991,9 @@ the accepted private boundary. All Phase-B1 acceptance gates passed on budget is a B2 requalification, not an implied default. A failed or stalled abort retains the original registry retirement and admission authority. B1 performs no generation GC and admits no correction generation beside - strict-blocked input. + strict-blocked input. The isolated fold RSS measurement and its 384-MiB + remeasurement tripwire are evidence for this one exclusive fold, not a + runtime allocator limit or storage quota. **Checked evidence** @@ -2932,8 +3066,13 @@ the accepted private boundary. All Phase-B1 acceptance gates passed on from 74,334,208 to 206,471,168 bytes (+132,136,960), explicitly including Arrow, PK-index, runtime, and allocator overhead. That result qualifies one resident writer with one 32-MiB aggregate Arrow reservation. Queued batches - share that reservation rather than accumulating outside it; B2 must measure - concurrent residents before raising either limit. + share that reservation rather than accumulating outside it. The repaired + high-entropy widest fold publishes all 8,192 rows and measures an isolated + whole-process peak-RSS delta of 284,934,144 bytes (about 272 MiB / 285 MB) + over the small-fold baseline. CI uses 384 MiB only as a remeasurement + tripwire for that one exclusive fold. It is not enforced allocator admission, + and concurrent residents must be measured before either concurrency or the + logical generation limit changes. **Evidence qualification** @@ -2959,33 +3098,97 @@ the accepted private boundary. All Phase-B1 acceptance gates passed on retract a WAL effect, make raw sentinel deletion safe, fence raw Lance callers, or provide a reclamation receipt, cross-process seal, or failover protocol. The Lance-owned post-success fence and retention patch therefore - remains a public-B2 gate. + remain B2b managed-reclamation and broader-topology gates. They do not block + public activation of the no-delete, single-live-writer-process B2a profile, + which retains the wrapper's post-watcher fence check. -**Phase B1 acceptance disposition — green 2026-07-19** +**Phase B1 disposition — accepted 2026-07-19, closure gap found 2026-07-20 and +repaired 2026-07-21** -The genuine cross-version/rebuild gate and complete graph-level B1 suite pass. +The genuine cross-version/rebuild gate and the then-declared graph-level B1 +suite passed. The post-watcher epoch-loss cell closes the clean-ack stale-epoch case at the adapter boundary, and the post-containment local cost cell remains history- flat. The configured-RustFS pass is still pre-containment and awaits rerun, so -it supports no current object-store ack-cost claim. B1 remains accepted at its -private main-only, unsharded, one-live-writer-process, one-resident-writer, -one-generation correctness boundary. +it supports no current object-store ack-cost claim. This evidence, together +with the dense-scan closure/RSS cell above, covers the private main-only, +unsharded, one-live-writer-process, one-resident-writer and one-exclusive-fold +fixture set. The evidence does not claim history-flat uncompacted manifest work, a public latency SLO, group commit, Lance-internal failure injection, or the unavailable forged pre-claim/pre-put stream-sidecar cell. -Phase B2 remains inactive despite B1 being green. §4.1–§4.6 now specify its -attribution, token, reclamation/enforced-admission-watermark, correction, and -persistent operator contracts. None is implemented by this amendment. - -### 12.4 Phase B2 — public strict activation gates - -Phase B2 implementation may begin from the contracts in §4.1–§4.6, but public -activation waits until every implementation/evidence item below is green. The -design does not waive the persistent escape requirement: a user must never be -left with a table that ordinary writers refuse but cannot be corrected, -quiesced, or rebuilt. +Gate R0 correctly found that the old scanner retained sparse backing buffers. +The dense copy releases those buffers and restores the intended logical +32-MiB closure check; the near-cap cell now closes. This amendment changes no +product surface. The future contracts in §4.1–§4.6 remain unimplemented. + +### 12.4 Gate R0 — historical bounded-retention no-go; closure repaired + +The checked-in `memwal_stream_cost.rs` Gate-R0 cells: + +- pin the result to surveyed Lance revision + `cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d`; +- strictly inventory every currently listed MemWAL object into a known class, + compare the set with the latest shard-manifest generation references, and + prove every earlier listed immutable path retains the same class and size; +- sweep one/four/eight success-only folds and preserve their cumulative + current-object growth; +- prove a retry after the referenced cut reuses the same generation root; and +- use deterministic high-entropy data for the legal near-cap physical-output + cell instead of a compression-friendly repeated value. + +The 2026-07-20 result remains historical evidence that stock RC.1 cannot support +a *bounded* retain-all promise: ordinary listing is not provider billing or +incomplete-multipart inventory, and Lance supplies neither a durable cross-open +materialization-attempt receipt/cap nor a source-derived physical-output +envelope. Those findings must be revisited before any future storage bound is +claimed. They are not blockers for unbounded B2a. + +The high-entropy cell now expects successful closure, complete row visibility, +one base-table effect, one `__manifest` visibility point, and no remaining +recovery sidecar. It retains the historical acknowledgement/materialization +byte observations and the invariant that every earlier listed immutable path +keeps the same path/class/size. The local RSS child's recorded reference run +measured a 284,934,144-byte fold delta and enforces a 384-MiB CI +**remeasurement** tripwire +for one exclusive fold. That tripwire is not a runtime hard allocator limit. +Configured RustFS remains a post-merge/tag regression signal and is not used to +invent a storage bound. + +### 12.5 Phase B2a — unbounded retain-all/no-GC implementation gate + +B2a is the selected next profile. Its implementation gate is deliberately +narrow: + +- retain every `_mem_wal` object and make every OmniGraph cleanup/repair path + structurally incapable of deleting or reclassifying it; +- preserve the existing 8,192-row/32-MiB logical generation limit, one resident + writer, one exclusive fold, bounded queues/deadlines/retries, and dense near- + cap closure evidence; +- resolve or refuse every relevant manifest/recovery authority operation before + claim, put, fold, correction, quiesce, or resume; never infer ownership from + listings or path shape; +- prove cold replay, fold-only retry, strict-block retention, and every existing + acknowledgement/fold crash boundary while unreferenced materialization + residue remains inert and untouched; and +- expose retained-object measurements only as advisory diagnostics, with no + file/object/byte limit, quota, reservation, aggregate receipt cap, + `GraphHistoryBudget`, or materialization-attempt receipt requirement. + +This slice adds no schema v9, production caller, SDK/HTTP/CLI/OpenAPI/Cedar +surface, or lifecycle/correction implementation. Those remain §12.6 work. + +### 12.6 Common public and B2b managed-reclamation activation gates + +The common public contracts in §4.1–§4.4 and §4.6 proceed only after the private +§12.5 B2a gate passes. This section owns those common gates and B2b's optional +managed-reclamation gates. B2a does not need any B2b-only bullet. Public +activation waits for every common implementation/evidence item to be green. +The design does not waive the +persistent escape requirement: a user must never be left with a table that +ordinary writers refuse but cannot be corrected, quiesced, or rebuilt. - `@stream(mode="upsert", on_reject="strict")`, production first use, SDK, HTTP, CLI, Cedar, and OpenAPI all route through the same private B1 core. The @@ -2994,12 +3197,13 @@ quiesced, or rebuilt. type requiring `@embed` or any external/provider-derived field; caller- supplied physical vectors round-trip without provider invocation. - Explicit enrollment tests cover `UNENROLLED` status, request-ID same/different - intent, every genesis/body/pointer/details/shard/lifecycle crash boundary, + intent, every selected-profile bootstrap/shard/lifecycle crash boundary, lost result after sidecar deletion through the persisted request-ID/intent receipt, `already_enrolled`, `stream_manage` authorization with Cedar/ embedded/remote parity, first returned stream incarnation, and request-level `StreamNotEnrolled` before ingest body admission. Ingest never - creates physical enrollment implicitly. + creates physical enrollment implicitly. B2b additionally covers every + genesis body/pointer/new-details crash boundary. - The public acknowledgement is status-only and caller-ordered. It maps the private durable batch result back to each caller ordinal and reports the stream incarnation, write ID, and current binding, not a WAL position, @@ -3008,10 +3212,10 @@ quiesced, or rebuilt. its candidate unconfirmed, while invalid/conflict/not-invoked outcomes mint neither. Every exact response variant in §4.6, including `invalid`, `stream_input_too_large`, lifecycle/authority/recovery blockers, and - `stream_retry_required`, has a schema. The B2 adapter owns contiguous + `stream_retry_required`, has a schema. The public adapter owns contiguous physical-run boundaries around invalid lines and token dispositions plus its - bounded reorder buffer. Tests cross a partially full generation, retained- - storage/control/correction/management-receipt spare and graph-history limits, + bounded reorder buffer. Tests cross a partially full generation, row/logical- + memory and queue/deadline limits, an intrinsically oversized row on an empty generation, authority/lifecycle movement or `RecoveryRequired` between runs, and a pre-invocation queue deadline after earlier lines are already durable; the @@ -3023,8 +3227,8 @@ quiesced, or rebuilt. - Internal schema v9/config-v3/state-v2/recovery-v12 provisions the hidden row metadata and manifest-selected token dataset. Every base/token effect is one recovery-covered publication. The graph-global token participant follows - canonical `sorted relevant stream admission -> graph history -> schema -> - main branch -> stream token -> sorted graph tables` gate order; folds capture + canonical `sorted relevant stream admission -> schema -> main branch -> + stream token -> sorted graph tables` gate order; folds capture its exact manifest-selected witness in the read set and never use raw token-table HEAD. If either the base or token participant has an effect or ambiguous outcome, recovery owns both; @@ -3044,9 +3248,9 @@ quiesced, or rebuilt. - Minimum operator controls expose authoritative status, explicit fold, plus persistent quiesce, resume/abort-drain, correction, and rebuild preflight. Status takes an exclusive cut, settles owners without mutating recovery, and - includes exact lifecycle/binding/revision, epoch, pending generations/bytes, - correction and management receipt usage/reserves/limits, graph-history - budget, last fold summary, and strict-blocked state. Every mutating call after + includes exact lifecycle/binding/revision, epoch, advisory pending + generations/bytes, paginated correction and management/claim receipt + summaries, last fold summary, and strict-blocked state. Every mutating call after enrollment carries an operation ID and expected revision; same-ID/same-digest returns the complete bounded terminal receipt, same-ID/different-digest conflicts, and a stale revision never retargets. Quiesce requires a caller @@ -3059,33 +3263,25 @@ quiesced, or rebuilt. or strict block remains. Correction clears a matching block but preserves `DRAINING` and the original drain goal; it is never itself a reopen. Tests delay a quiesce/resume retry across a later cycle and prove receipt/stale- - revision handling. They drive the graph-global budget to its closure floor, - prove new ordinary work returns `GraphRebuildRequired`, hold two streams - strict-blocked simultaneously, then close one while preserving the other's - independent block/correction/abort-drain/requiesce/`SEALED` reserve. The - cap-minus-full-closure case also reserves every required management receipt - and both sequential claim envelopes/terminal records. - The WAL watermark is never reported as a whole-graph history bound. -- Graph-history tests force every manifest writer through the universal gate, - verify late sidecar/lifecycle discovery releases all gates and restarts, and - exercise exact used/pending/finalized accounting. The reserve-first physical - envelope crashes before/after fragment, multipart, failed CAS, manifest/index - output, publication, and cleanup receipt; bounded retries cannot create - uncharged orphans, cold inventory reconstructs every charge, unknown output - closes writers, and two racing publishers prove the globally serialized gate - lifetime, fresh-revision restart, and absence of reverse-order deadlock. Two - simultaneous stream closure reservations cannot be spent by ordinary commits - or by each other. + revision handling. They hold two streams strict-blocked simultaneously, then + close one while preserving the other's independent block/correction/ + abort-drain/requiesce/`SEALED` authority. The WAL watermark is never reported + as a whole-graph history bound. +- **Future bounded-root profile only:** a separately accepted + `GraphHistoryBudget` amendment must force every manifest writer through one + reserve-first gate, account for complete physical effects and recovery, and + prove crash-safe settlement and independent stream closure reserves. None of + those tests gates unbounded retain-all. - Every public batch durably records its authenticated contributor before rows can become unattributable, and the fold audit consumes that evidence. A - current `WITHDRAWN` token remains durable even when the graph row is absent. - The reviewed Lance inspect/plan/execute patch, durable attempt/receipt, + current `WITHDRAWN` token remains durable even when the graph row is absent; + neither attribution nor sequencing is inferred from MemTable positions, + mutable WAL statistics, age, raw path shape, or benchmark maxima. +- **B2b only:** the reviewed Lance inspect/plan/execute patch, durable attempt/receipt, post-success WAL fence check, bounded history checkpoint, and enforced retained-byte/object admission watermark bound growth. Exact inventory requires strong HEAD/GET/LIST visibility after PUT/DELETE plus exact multipart - accounting/abort, or Lance-owned durable accounting. Neither contract is - inferred from MemTable positions, mutable WAL statistics, age, raw path - shape, or benchmark maxima. + accounting/abort, or Lance-owned durable accounting. - Admission cancellation and server shutdown cannot abandon an in-flight durability waiter. A background-owned admission task reaches a durable outcome or typed `AckUnknown`. Status/replay preserves possible residue but @@ -3107,10 +3303,8 @@ quiesced, or rebuilt. planning bound, engine-derived canonical plan digest plus actor binding, same/different correction-ID replay, stale block token, unknown key, exact row/byte and - receipt-count/receipt-byte cap/+1, admission at cap-minus-one with one full - terminal-receipt spare, cold-replay reconstruction of that spare, a blocked - correction consuming the final reserved slot/bytes at the exact cap, and - no-generation-at-full-cap quiesce through `SEALED` rebuild. They also cover + response-page bounds, cold-replay reconstruction of terminal receipts, and + no-generation quiesce through `SEALED` rebuild. They also cover 8,192 distinct corrected current-token rows plus the separately bounded one- row receipt transaction, depth-8,192 replacement to exact maximum 8,193, concurrent fold/resume, and @@ -3118,21 +3312,26 @@ quiesced, or rebuilt. skip-invalid path, or graph delete is admitted. - Quiesce tests cover crashes before/after `OPEN -> DRAINING`, epoch claim, seal/drain, each fold, equal descriptor/top-level witness advancement, - historical and reclaim-successor fence sentinels, strict-block summary, + historical and selected-profile-authenticated current fence sentinels, + strict-block summary, engine-minted failure-drain identity/actor before and after its conditional CAS, empty proof, and `SEALED` publication. Status covers concurrent - put/flush/reclaim exclusion. Resume and abort-drain cover + put/flush/selected-profile retention exclusion. Resume and abort-drain cover arm/claim/confirm/CAS/finalize, caller operation-ID/expected-revision retry, same-ID/different-intent, delayed retry after a later drain/resume cycle, named-branch refusal, strict- - block refusal, reopen, and lost replies. Cold open, quiesce, resume/abort, and checkpoint also crash at - claim-attempt/sentinel/manifest boundaries and prove that their ordinary - sentinel-first claims preserve the prior replay cursor and classify its full - tail before admitting a put. Only the whole-cut reclaim case may advance that - cursor to its new sentinel. Quiesce settles and publishes every retention - sidecar before the final proof; reclaim/checkpoint refuse while `SEALED`, and - resume consumes rather than mutates the old proof. `SEALED` never auto-opens. -- Reclamation tests cover stock-cleanup non-ownership, the stale-writer deleted + block refusal, reopen, and lost replies. Every selected-profile claim crashes + at its declared authority/effect boundaries and proves exact classification + before admitting a put. Under B2b, cold open, quiesce, resume/abort, and + checkpoint additionally cover reclaim-successor sentinels and crash at + patched claim-attempt/sentinel/manifest + boundaries and prove that their ordinary sentinel-first claims preserve the + prior replay cursor and classify its full tail; only the whole-cut reclaim + case may advance that cursor to its new sentinel. Quiesce settles and + publishes every retention sidecar before the final proof; B2b + reclaim/checkpoint refuse while `SEALED`, and resume consumes rather than + mutates the old proof. `SEALED` never auto-opens. +- **B2b only:** reclamation tests cover stock-cleanup non-ownership, the stale-writer deleted successor-sentinel slot, exact local/RustFS inventory, HEAD/GET/LIST and multipart capability refusal or durable accounting, and explicit refusal of versioned/soft-delete/Object-Lock storage unless every retained version, @@ -3169,19 +3368,16 @@ quiesced, or rebuilt. lifecycle, including `SEALED`, until Phase D defines a token-aware direct- write/witness/rebind transition. - Genuine cross-version tests prove old-binary/new-format and - new-binary/old-format refusal. They prove stock RC.1 rejects B2's new MemWAL - details type URL/system-index kind rather than ignoring a field or falling - back to a latest-version hint. V9 root-init/open tests prove the singleton - graph-history budget is established before any other v9 publication, its - baseline inventory/source-bound version and aggregate stream reserves match, - its bootstrap publication self-charges under a precomputed maximum, and - cap-too-small, every bootstrap crash/lost-result boundary, changed estimator/ - limits, and missing/duplicate/mismatched state refuse rather than defaulting. + new-binary/old-format refusal. Under B2b, they prove stock RC.1 rejects the + patched new MemWAL details type URL/system-index kind rather than ignoring a + field or falling back to a latest-version hint. A future bounded-root + amendment must separately own `GraphHistoryBudget` bootstrap, mismatch, + crash, and cap-too-small refusal tests; those are not v9/B2a activation gates. Rebuild tests prove acknowledged rows survive only after quiesce/fold, and fail closed when an acknowledged row remains unfolded. -### 12.5 Later expansion gates +### 12.7 Later expansion gates - Phase C defines restart-stable reject-row identity, using B2's durable contributor attribution, before adding atomic dead letters, reject @@ -3206,9 +3402,11 @@ quiesced, or rebuilt. |---|---|---| | E0 | production-neutral public-surface enrollment/witness classifier; no schema, API, sidecar, or format activation | **Passed 2026-07-18:** 14 substantive local cells, complete six-attempt zero-list 8/80 cost shape, Unix no-list/error tripwire, and one non-vacuous configured RustFS positive-plus-negative cell (§12.1) | | A | bounded main/unsharded/single-live-writer enrollment adapter, all-lifecycle effect exclusion with only the `SEALED` native-branch exception, lifecycle/admission lease, then graph-format capability/refusal and strict rebuild | **Implemented 2026-07-18 (§12.2):** internal schema v7, recovery-v10 enrollment, durable lifecycle CAS, process-local exclusion, crash/partial-format refusal, and genuine v6↔v7 strand evidence; no public enrollment or row path | -| B1 | **Implemented privately 2026-07-19; acknowledgement containment added 2026-07-20:** internal schema v8/config-v2, root-scoped one-generation admission worker, durability-watcher success followed by a same-writer post-durability epoch check, conservative active-state reopen/replay, the pinned RC.1 replay-watermark bridge, and one explicit strict RFC-022 fold; no production caller | **Green privately at the bounded correctness boundary (§12.3):** genuine v7↔v8 binary refusal/rebuild and the graph-level B1 behavior/crash/race suite, including post-watcher epoch loss, pass. The post-containment local ack probe remains history-flat at 9 reads / 219 bytes; configured-RustFS retains only the 2026-07-19 pre-containment baseline and must be rerun before a current object-store ack-cost claim. The non-flat uncompacted-manifest term is retained explicitly; upstream-internal flush/channel failure and forged pre-claim/pre-put stream-sidecar injection remain unclaimed seam limitations, not public guarantees | -| B2-0 | contract closure for compare-and-chain tokens, trusted row attribution, manifest-selected current-token state, revision-fenced protocol-v2 lifecycle/management receipts, bounded `REPLACE`/`WITHDRAW` correction, Lance-owned reclamation, enforced retained-storage admission, and graph-global manifest-history/closure reservations; no product surface | **Specified 2026-07-19 (§4.1–§4.6):** two stock-RC.1 negative/ownership guards are checked in. Implementation remains inactive; the owned Lance durable inspect/plan/execute, Lance-owned post-success-fence check, bounded MemWAL checkpoint, strong-inventory proof, enforceable MemWAL growth reservation, and reserve-first graph-manifest materialization envelope are next. B1's adapter recheck does not satisfy those substrate/retention gates | -| B2 | public strict activation across schema v9/config-v3/recovery-v12, SDK, HTTP, CLI, Cedar, and OpenAPI, with the B2-0 contracts implemented | B1 green; reviewed Lance patch pinned; token/attribution/correction/revision-fenced lifecycle/reclamation and graph-history crash matrices; local and RustFS enforced-bound validation; genuine v8↔v9 refusal/rebuild; API compatibility; cancellation/shutdown ownership; public parity | +| B1 | **Implemented privately 2026-07-19; acknowledgement containment added 2026-07-20; widest-shape closure repaired 2026-07-21:** internal schema v8/config-v2, root-scoped one-generation admission worker, durability-watcher success followed by a same-writer post-durability epoch check, conservative active-state reopen/replay, the pinned RC.1 replay-watermark bridge, and one explicit strict RFC-022 fold; no production caller | The graph-level behavior/crash/race suite and genuine v7↔v8 refusal/rebuild remain green. Fold now charges logical slices and copies each scanner emission into dense owned arrays. The legal 8,192-row high-entropy near-cap generation folds and publishes exactly once without changing the 32-MiB admission cap; its isolated fold RSS delta is guarded by the 384-MiB remeasurement tripwire (§12.4) | +| R0 | production-neutral retained-growth/source audit; current-object census; referenced-cut retry; legal high-entropy near-cap materialize/fold cell; no schema, public caller, or deletion | **Historical bounded-retention no-go 2026-07-20; disposition amended 2026-07-21 (§0.2/§12.4):** RC.1 still exposes neither a complete reserve-first physical envelope/receipt nor a durable cross-open randomized-attempt cap. Those facts prohibit a finite storage promise but do not block selected unbounded retain-all. The formerly red widest cell is now green locally and on the configured-RustFS CI path; current-object observations remain advisory retention evidence, not provider billing/accounting | +| B2a | selected unbounded retain-all/no-GC profile on stock Lance | **Selected 2026-07-21; implementation gate remains (§12.5):** no OmniGraph byte/object/file/history quota, no raw `_mem_wal` deletion, and loud provider exhaustion. Keep row/memory/deadline/retry/ambiguity bounds and prove no-delete, recovery, correction, lifecycle, token, attribution, authorization, and product parity before activation. No schema v9 or product surface is active yet | +| B2b | candidate managed-reclamation retention profile | Inactive. Requires the Lance-owned durable inspect/plan/execute + receipt, post-success fencing, bounded checkpoint/inventory/accounting, local/RustFS enforced-bound validation, and the profile-specific crash matrix (§4.5.2/§12.6). Passing it alone activates no product surface | +| B2-common | explicit enrollment, compare-and-chain token/attribution, revision-fenced lifecycle/correction, schema v9/config-v3/recovery-v12, SDK, HTTP, CLI, Cedar, and OpenAPI | Inactive. Public activation requires the common crash/no-delete/provider-failure/cross-version, authorization, cancellation/shutdown, API-compatibility, and product-parity gates in §12.6. `GraphHistoryBudget` belongs only to a future bounded/managed profile, not selected retain-all | | C | restart-stable reject-row identity, atomic dead letter, richer status, and evidence-backed configurable bounds | reject crash matrix; reject-retention proof; backpressure and RSS/latency evidence | | D | automatic operation drain, schema/branch/upgrade integration, and rematerialization rebind | two-coordinator race, old/new physical-binding crash matrix, and format-transition suite | | E | fresh cuts and maintained-index reads; cross-process `Fresh` ships only if the substrate generation-retention guard exists (§9), otherwise same-process only | cut consistency; merged-generation exclusion | diff --git a/docs/user/concepts/storage.md b/docs/user/concepts/storage.md index 2bee7b8b..555496cc 100644 --- a/docs/user/concepts/storage.md +++ b/docs/user/concepts/storage.md @@ -56,7 +56,7 @@ place. | v5 | RFC-028 SchemaIR v2 plus graph-domain stable schema IDs; manifest table rows, OCC, recovery ownership, and physical paths keyed by stable table ID + incarnation. | | v6 | Preserves v5 identity and activates RFC-023: every graph node/edge dataset has exact non-null physical `id` as Lance's unenforced PK, and every production strict insert/upsert uses the exact-`id` filter-bearing adapter. | | v7 | Preserves v5/v6 identity and keyed-write contracts, and adds RFC-026 identity-keyed `stream_state` authority: physical enrollment binding, mutable current-HEAD witness, lifecycle, and per-shard epoch floor. Phase A can recover one exact empty private enrollment; no production row-streaming API is active. | -| v8 | Preserves the v5/v6/v7 contracts and activates RFC-026 stream-config v2 plus recovery-v11 for the private B1 row/fold core: one no-roll generation, watcher-only durable acknowledgement, and one exact fold whose manifest publish advances the table pointer, lifecycle witness, and graph lineage together. Public row streaming remains inactive. **The only version this binary serves.** | +| v8 | Preserves the v5/v6/v7 contracts and activates RFC-026 stream-config v2 plus recovery-v11 for the private B1 row/fold core: one no-roll generation, watcher success plus a same-writer post-durability epoch check before clean acknowledgement, and one exact fold whose manifest publish advances the table pointer, lifecycle witness, and graph lineage together. The legal near-cap closure cell is green after logical-slice accounting and dense scanner-batch copies. Unbounded retain-all/no-GC is selected for the first profile, but schema-v9 and every public row-streaming surface remain inactive. **The only version this binary serves.** | ## On-disk layout diff --git a/lance-memwal-pr.md b/lance-memwal-pr.md new file mode 100644 index 00000000..e1d2acf5 --- /dev/null +++ b/lance-memwal-pr.md @@ -0,0 +1,448 @@ +# Lance MemWAL PR + +**Status:** deferred optional managed-reclamation proposal; not an RFC-026 +retain-all activation dependency + +**Surveyed substrate:** Lance 9.0.0-rc.1 at +`cec0b7dffe2d85c7e66dbe9d1f3891c297903a1d` + +**Related:** [RFC-026](docs/rfcs/0026-memwal-streaming-ingest.md), +[write-path state of affairs](docs/dev/writing-path-state-of-affairs.md), and +[WAL thinking](WAL-thinking.md) + +## Decision in one paragraph + +The selected first profile is unbounded retain-all on stock Lance: OmniGraph +performs no MemWAL GC and promises no retained-byte or file-count limit. +Therefore OmniGraph does **not** need this Lance change to activate that +profile. If retained-storage cost later justifies managed reclamation, the safe +route is still an upstream-shaped Lance PR: keep the change isolated and +generally useful, pin only an exact reviewed commit if OmniGraph must consume it +before release, and never turn the temporary branch into a permanent product +fork. + +## What “without forking Lance” means + +There are two different questions hidden in that phrase: + +1. **Can we avoid a permanent, independently maintained Lance distribution?** + Yes. That is the recommended route. +2. **Can we deliver safe MemWAL reclamation without changing Lance at all?** + No, not on the surveyed RC.1 public surface. Retain-all avoids that operation + entirely, so this is a future optimization boundary rather than a blocker. + +A contributor GitHub fork or branch may be needed mechanically to open the +upstream PR. Pinning its exact reviewed commit while the PR is open is also a +temporary divergence. Neither is a product fork if we keep one upstream-shaped +patch, do not add OmniGraph-only behavior, and remove the pin after upstream +merge. Vendoring Lance or carrying an indefinite private patch series would be +a product fork, even if we avoided calling it one. + +## Current state + +### OmniGraph's direct write path is correct, but expensive + +The RFC-022 write path is implemented. A graph mutation stages its affected +Lance transactions, arms durable recovery, commits the physical table effects, +and makes the complete graph change visible through one `__manifest` publish. +That preserves graph-wide atomic visibility and recovery, but every small +interactive write pays the full graph commit protocol. + +This is the correct path for an interactive write whose acknowledgement means +“the graph commit is visible.” It is not the desired latency shape for a large +number of agents continuously submitting small changes. + +### The private MemWAL core exists + +RFC-026 Phase A and private Phase B1 are implemented and evidence-green at a +deliberately narrow boundary: + +- one main-branch, unsharded MemWAL binding per enrolled table; +- one live OmniGraph writer process for the graph; +- one bounded generation of at most 8,192 rows and 32 MiB; +- acknowledgement only after Lance's durability watcher succeeds; +- replayed or flushed-but-unmerged state routes to fold only; +- one strict fold stages a normal RFC-022 table effect and publishes it through + `__manifest`; and +- current internal schema v8, stream-config v2, and recovery-v11 describe this + private implementation. + +The core is intentionally reachable only through a feature-gated, doc-hidden +engine seam. There is no `@stream` schema intent, public enrollment, SDK +method, HTTP route, CLI command, OpenAPI surface, or operator drain/resume +workflow. An acknowledgement means that the submitted batch is durable and +replayable; it does not mean that the graph already exposes the rows. + +MemWAL is the strategic streaming substrate. OmniGraph is not designing a +replacement WAL, buffer pool, transaction manager, or LSM tree. The remaining +risk is in still-maturing public lifecycle and operational APIs, not in a +decision to build a different storage architecture. + +The prior near-cap closure failure is fixed without changing admission: fold +scanning charges logical Arrow slices and densifies selected rows before +retention. The measured one-exclusive-fold RSS delta was 284,934,144 bytes +(about 272 MiB); CI's 384-MiB threshold is a remeasurement tripwire, not a +runtime allocator limit. + +### Public streaming is still specified but inactive + +RFC-026 Phase B2-0 specifies the remaining public contract: + +- compare-and-chain write tokens for safe same-key retries; +- trusted durable contributor attribution; +- a manifest-selected current-token participant; +- revision-fenced persistent lifecycle management; +- bounded strict correction; +- unbounded retain-all with no OmniGraph MemWAL deletion; and +- explicit acceptance of loud provider-capacity exhaustion rather than a hard + retained-storage admission promise. + +Those contracts are not implemented. Schema v9, stream-config v3, state v2, +recovery-v12, and all public streaming surfaces remain inactive. + +## The missing Lance capabilities + +Stock Lance RC.1 owns the MemWAL layout and write machinery, but it does not +expose the complete lifecycle needed for safe long-running use. + +### Generic cleanup does not own MemWAL + +`cleanup_old_versions` removes eligible ordinary Lance table history. The +surveyed RC.1 implementation does not traverse or reclaim `_mem_wal`. +OmniGraph has a checked-in negative guard that creates reclaimable table +history plus one durable MemWAL fixture, runs generic cleanup, and proves that +the fixture's MemWAL object names and bytes remain unchanged. The source audit, +not that one fixture alone, establishes the missing owned public reclamation +surface. + +This is an ownership boundary, not a bug to work around with +`delete_unverified`. + +### There is no safe public MemWAL reclamation operation + +The surveyed `DatasetMemWalExt`, `ShardWriter`, and `ShardManifestStore` +surfaces do not provide a classified inventory, a reclaim plan, a delete +operation, or a durable result receipt. Raw listing can show object names, but +it cannot prove which generation outputs were referenced, which objects are +attempt-owned or orphaned, whether an unknown object is safe to delete, or +whether a lost response completed the operation. + +OmniGraph cannot safely reconstruct that authority from paths, age, mutable +hints, or matching versions. + +### Successful WAL writes are not re-fenced + +RC.1 checks epoch ownership after an atomic PUT conflict or error, but returns +immediately after a successful WAL PUT. If a successor's empty fence sentinel +is removed, a stale predecessor can write into that slot and receive watcher +success even though the shard manifest already records a higher epoch. + +The checked-in +`mem_wal_deleted_fence_slot_allows_stale_writer_success_on_pinned_lance` +regression demonstrates this exact failure. Safe reclamation therefore requires +Lance to recheck epoch authority after a successful WAL PUT and return a typed +fence or unknown outcome when the writer lost ownership. + +### Epoch claims are not one recoverable effect + +The current writer-open path claims the next epoch in the shard manifest and +then writes the empty successor fence sentinel. A crash can occur between those +effects. There is no caller-owned durable claim attempt and terminal receipt +that can prove whether to finish, retry idempotently, or refuse foreign state. + +Reclamation also needs to advance the epoch and install a successor sentinel. +It cannot safely build on a claim sequence whose partial effects have no +durable owner. + +This is distinct from the initial enrollment receipt/seal gap. Phase A and B1 +work around enrollment within the main-only, one-shard, +one-live-writer-process boundary. The proposed retention protocol does not by +itself make overlapping writers, failover, or general enrollment safe. + +### Only a quiescent whole cut is currently provable + +`FlushedGeneration` identifies a generation and path but does not persist its +exact WAL position range. The authoritative shard-wide replay cursor is the +available cut. Consequently, the initial reclamation primitive may delete only +one fully quiesced whole prefix after every covered generation is merged and +every writer, watcher, flush, recovery, and read guard is settled. It must not +pretend that an arbitrary per-generation prefix is safe. + +### Reclamation metadata must also remain bounded + +Adding attempt and receipt objects without a checkpoint would merely move the +unbounded-growth problem. Claim, reclaim, manifest, and receipt history needs +an authoritative bootstrap/checkpoint chain, deterministic recovery, and typed +expiry of old operation IDs. + +## Why the implementation belongs in Lance + +Lance is the only layer that owns all the facts required to delete safely: + +- the `_mem_wal` namespace and object encodings; +- shard-manifest versions, epochs, and replay cursors; +- fence-sentinel encoding and writer checks; +- generation datasets, PK/Bloom sidecars, and maintained indexes; +- object-store conditional-write and delete behavior; and +- the relationship between WAL persistence, generation flush, and base-table + merge progress. + +Keeping reclamation in Lance gives us four important properties. + +1. **Opaque ownership.** The caller supplies logical witnesses and budgets, not + object paths to delete. +2. **One fencing protocol.** The writer, claimant, and reclaimer use the same + epoch and sentinel implementation. +3. **Durable lost-result recovery.** Lance can classify its own attempt, + sentinel, manifest, deletion, and receipt states. +4. **Upstream compatibility.** Other MemWAL users can use and test the same + lifecycle instead of OmniGraph depending on private layout details. + +An OmniGraph-side collector would be a second, incomplete implementation of +Lance's storage state machine. It would violate the architectural rule that +Lance owns MemWAL and would make every Lance upgrade a raw-format migration +risk. + +The division of responsibility remains exact: OmniGraph decides whether a +graph-visible whole cut is eligible and keeps `__manifest` as the only graph +publication door; Lance classifies and reclaims the physical MemWAL state. The +Lance PR does not create another way to publish graph data. + +## Deferred upstream PR + +If managed reclamation becomes a measured priority, the logical slice is +**Lance-owned MemWAL retention authority**. It should be implemented in Lance, +proposed upstream as a ready-for-review PR, and qualified through a thin +OmniGraph pin PR. It is not the next retain-all slice. + +This would be the first of two Lance-side reclamation tranches. The retention-authority PR +below makes deletion and its recovery safe. A following Lance PR adds the +reserve-first physical-growth ledger, materialization-attempt limits, and +emergency control headroom required before public **bounded managed- +reclamation** admission. They are not prerequisites for the selected +unbounded, no-delete B2a profile. Keeping those contracts separate gives the +first PR one reviewable safety outcome without mistaking it for complete B2b +activation. + +### 1. Opt-in, fail-closed details kind + +Introduce a new versioned MemWAL details type or system-index kind for the +retention protocol. Do not add an unknown field to RC.1's accepted details +shape: an old binary might ignore it and open state it cannot manage safely. + +Initialization writes an immutable genesis checkpoint body and conditionally +installs its bootstrap pointer before publishing the new details kind. There is +no “missing pointer means empty” fallback, and readers never use a best-effort +latest hint as authority. + +The legacy details kind remains available for existing users and for +OmniGraph's private schema-v8 B1 state. Enabling the new kind is explicit. + +### 2. Post-success writer fencing + +After every successful WAL atomic PUT, re-read the authoritative shard epoch. +If a successor has claimed a higher epoch, return the existing typed peer-fence +classification or a distinct outcome that tells the caller the durable effect +may exist but must not be acknowledged as an unfenced success. + +This fix applies independently of reclamation and should carry local and +object-store versions of the deleted-successor-sentinel regression. + +### 3. Durable sentinel-first claims + +Represent ordinary epoch claims with a caller-minted, checkpoint-scoped claim +ID, an exact intent digest, a durable attempt, and a terminal receipt: + +```text +attempt + -> empty successor sentinel at the authenticated tail + 1 + -> shard-manifest CAS naming that sentinel and the new epoch + -> complete receipt +``` + +Once the sentinel exists, the claim cannot abort. Same-ID/same-digest recovery +must finish or return the existing terminal receipt; the same ID with another +digest conflicts. Pending or unknown claim attempts block another writer claim. +Ordinary claims preserve the existing replay cursor and classify the complete +tail; they do not turn a writer reopen into reclamation. + +### 4. Opaque whole-cut reclamation + +Expose a narrow public protocol along these lines: + +```text +inspect_mem_wal_retention(...) +plan_mem_wal_reclaim(reclaim_id, exact_witnesses, approved_whole_cut, budgets) +execute_mem_wal_reclaim(opaque_serializable_plan) -> exact_receipt +classify_mem_wal_reclaim(reclaim_id, plan_digest) + -> pending | aborted_no_effect(receipt) | complete(receipt) +``` + +The plan binds the exact base-table and shard witnesses, current generation, +replay cursor, complete authenticated WAL/sentinel tail, referenced and orphan +generation classifications, unknown inventory, and object/byte deletion +budgets. The caller cannot add or change object paths. + +Execution must: + +1. persist the exact attempt before any effect; +2. revalidate the plan byte-for-byte; +3. install an attempt-owned successor sentinel at the authenticated tail + 1; +4. CAS a successor manifest that names the sentinel and preserves the monotonic + `current_generation` authority; +5. only then delete the plan-owned whole cut; +6. retain malformed, unknown, or unproved objects; and +7. persist the exact receipt before returning. + +A stale plan that lost authority before any effect terminates as +`aborted_no_effect`. After any owned effect, abort is forbidden: recovery must +finish the same plan or fail closed with the pending attempt retained. Partial +deletion and a lost final response are normal recovery cases, not permission to +guess success. + +### 5. Bounded checkpoint and receipt history + +Checkpoint only terminal attempts whose declared retry horizon has elapsed. +Write a deterministic immutable checkpoint body over the post-claim state, then +conditionally swap the bootstrap pointer. Only after that pointer is +authoritative may older summarized metadata be removed idempotently. + +Old operation IDs return typed expiry results. Reusing the same UUID in a new +checkpoint epoch is a distinct identity; the implementation does not retain an +unbounded tombstone set. + +### 6. Exact inventory and backend refusal + +The supported backend must provide inventory semantics strong enough to prove +the result after successful PUT and DELETE, including incomplete multipart +uploads, or Lance must keep an equivalent durable complete object ledger. + +Versioned, soft-delete, retention-lock, or Object-Lock configurations must be +refused unless Lance can account for every retained version, delete marker, +locked byte, and incomplete upload and can permanently remove every eligible +version. A successful current-key listing alone is not proof of reclaimed +physical storage. + +The first acceptance boundary is local storage plus configured RustFS. Other +backends remain refused until their capabilities pass the same contract. + +## OmniGraph companion change + +The OmniGraph PR should remain deliberately thin: + +- pin every Lance workspace dependency to one exact reviewed PR-head commit; +- turn the stale-writer negative regression into an assertion of typed + fence/unknown behavior; +- keep the negative guard proving generic cleanup still does not own MemWAL; +- add compile/runtime guards for the new opaque APIs and fail-closed details + kind; +- prove stock RC.1 refuses the new details kind rather than silently ignoring + it or falling back to a latest hint; +- preserve structural checks forbidding raw `_mem_wal` deletion in OmniGraph; +- rerun the complete private B1 behavior, crash, race, cost, and cross-version + suites; and +- update RFC-026 and the write-path state ledger with the exact reviewed Lance + revision and the boundary that has become green. + +OmniGraph schema v8 remains the active format during this slice. The new Lance +retention kind is qualified but not yet activated by a production stream. + +## Acceptance evidence + +The Lance PR is not complete because the happy path deletes files. It is +complete when the state machine remains exact through crashes and lost +responses. + +Required local and configured-RustFS evidence includes: + +- crash before and after attempt persistence; +- crash after successor sentinel but before manifest CAS; +- crash after manifest CAS during partial deletion; +- crash after receipt persistence with the caller losing the result; +- same ID and digest returning the same terminal result; +- same ID with another digest refusing; +- stale plans terminating only when effect-free; +- foreign sentinel, manifest movement, gaps, overflow, and collisions failing + closed; +- pending attempts blocking ordinary claims; +- ordinary claims preserving the replay cursor and complete tail; +- only a proved whole-cut reclaim advancing the cursor to its new sentinel; +- `current_generation` never resetting or being reused; +- referenced, retired, orphan, malformed, and unknown objects receiving exact + conservative classifications; +- partial deletion resuming idempotently; +- checkpoint-body and bootstrap-pointer crash recovery; +- old claim/reclaim IDs returning typed expiry results; +- stock RC.1 refusing the new details kind; +- the deleted-successor-sentinel regression returning fence/unknown rather + than success; and +- all existing OmniGraph Phase-B1 and RFC-022 recovery evidence remaining + green. + +## What this PR intentionally does not do + +This slice creates safe retention authority. It does not by itself activate +public streaming. + +The following remain separate work: + +- a Lance-owned reserve-first physical-growth ledger, bounded + materialization-attempt count, and emergency control headroom; +- OmniGraph's graph-global `GraphHistoryBudget` across every manifest writer; +- schema v9, stream-config v3, state v2, and recovery-v12 activation; +- compare-and-chain token state and trusted contributor attribution; +- persistent revisioned quiesce/resume/abort-drain and strict correction; +- SDK, HTTP, CLI, Cedar, OpenAPI, and shutdown/cancellation product parity; +- automatic SchemaApply, Optimize, Repair, Cleanup, or branch integration; +- multi-shard routing, fresh reads, or overlapping-process failover; and +- arbitrary per-generation WAL-prefix reclamation. + +Even a green reclaim primitive would not itself permit public admission: token, +attribution, lifecycle/correction, authorization, shutdown, cross-version, and +product-parity evidence remain independent gates. A retained-storage ledger or +graph-global history budget is required only for a later profile that promises +a hard storage bound. + +## Alternative routes + +| Route | What it gives us | Main problem | Disposition | +|---|---|---|---| +| Upstream PR, wait for merge/release | No temporary dependency divergence | Upstream scheduling could block OmniGraph for months | Correct eventually, but not our calendar | +| **Upstream-first PR plus temporary exact-commit pin** | We control implementation timing while ownership and review remain upstream-shaped | We temporarily carry an unreleased commit and must re-audit rebases | **Recommended** | +| Permanent OmniGraph Lance fork | Full control over implementation and releases | Continuous merge, security, compatibility, format, and review burden; easy architectural divergence | Reject | +| Vendor Lance crates or apply a private Cargo patch | Avoids a visible fork repository | This is still a fork, with worse provenance, review, and upgrade ergonomics | Reject | +| Implement raw `_mem_wal` GC in OmniGraph | Could appear faster initially | Reimplements Lance's private state machine; cannot prove ownership/fencing/lost results; violates the substrate boundary | Reject | +| Object-store lifecycle or age-based deletion | Operationally simple | Age and path shape do not prove liveness; may delete a live fence, WAL entry, upload, or generation | Reject | +| Use `cleanup_old_versions(delete_unverified=true)` | Reuses an existing cleanup call | Generic cleanup does not own `_mem_wal`; unverified age deletion has the wrong safety contract | Reject | +| Keep every MemWAL object forever | No deletion race; no Lance patch | Storage and control history grow without bound; provider exhaustion is not predictable from ordinary LIST | **Selected first profile**, with loud operational risk and no storage-bound claim | +| Block at a finite limit and require export/rebuild | Gives an explicit escape without deleting in place | RC.1 still lacks a source-derived enforced physical-growth reservation; crash-created randomized output and multipart state can exceed an advisory estimate | Possible emergency posture after stronger bounds, not a replacement for the PR | +| Periodically export and rebuild | Reclaims the old root after an operator-controlled cutover | Requires a proved sealed cut and preserves no indefinite in-place service; it does not recover pending reclaim attempts | Operational escape, not steady-state reclamation | +| Build a custom OmniGraph WAL | Avoids the missing MemWAL API | Rebuilds Lance's WAL, fencing, recovery, buffering, and LSM responsibilities and reopens the hardest distributed-systems problems | Reject | +| Use only the direct write path or add group commit | Preserves the current ownership model and may reduce constant cost | Does not provide the durable early-acknowledgement contract or isolate continuous small writes from graph publication | Useful interim optimization, not a streaming architecture | + +## Delivery and exit strategy if this optimization is scheduled + +1. Develop the patch in a clean Lance checkout following Lance's own format, + Rust, testing, and documentation rules. +2. Keep the commits narrowly scoped: fence correctness, durable claim + authority, opaque reclaim execution, and bounded checkpoint/inventory. +3. Open the Lance PR as ready for review, with its public contract and crash + matrix in the PR description. +4. Pin OmniGraph to the exact reviewed PR-head commit only after the local and + RustFS acceptance suite passes. +5. Re-audit and update the pin when the upstream branch rebases; never follow a + mutable branch name. +6. When upstream merges, move OmniGraph to the canonical commit or release and + remove all contributor-fork references. +7. If upstream requests compatible changes, adapt the patch while preserving + RFC-026's safety contract. If upstream rejects the underlying ownership or + recovery model, return to RFC review rather than accumulating a permanent + hidden fork. + +## Conclusion + +Unbounded retain-all lets OmniGraph proceed without this patch. If we later +choose managed reclamation, we still cannot honestly implement it only in +OmniGraph: the safe route is to contribute the missing MemWAL lifecycle to +Lance, temporarily consume an exact reviewed commit only if needed, and remove +that divergence when upstream catches up. Lance continues to own Lance storage. diff --git a/wal-options.md b/wal-options.md new file mode 100644 index 00000000..c4e329b8 --- /dev/null +++ b/wal-options.md @@ -0,0 +1,592 @@ +# WAL Options + +**Status:** decision note; RFC-026 records the selected profile + +**Updated:** 2026-07-21 + +**Surveyed substrate:** OmniGraph's pinned Lance 9.0.0-rc.1 plus the public +LanceDB MemWAL integration on Lance 9.1.0-beta.4 + +**Related:** [RFC-026](docs/rfcs/0026-memwal-streaming-ingest.md), +[WAL thinking](WAL-thinking.md), +[Lance MemWAL PR](lance-memwal-pr.md), and +[architectural invariants](docs/dev/invariants.md) + +## Executive summary + +OmniGraph has already chosen Lance MemWAL as its streaming substrate. The open +decision is not whether to build a different WAL. It is how much lifecycle +machinery must be complete before OmniGraph exposes durable stream admission as +a product feature. + +There are five relevant options: + +1. **Unbounded retain-all on stock Lance:** never delete MemWAL objects inside + a root and do not advertise a file-count or retained-byte limit. Storage + grows monotonically; provider exhaustion is an accepted, loud operational + risk for this first profile. +2. **A narrow local Lance patch with managed GC:** implement the missing + attempt, receipt, accounting, fencing, and reclamation primitives in Lance, + pin that reviewed revision now, and upstream it without depending on the + upstream review calendar. +3. **Whole-root rotation or rebuild:** seal and fold a root, perform an external + operator/service cutover to a fresh root, and retire the old root as an + operational unit rather than deleting live MemWAL internals piecemeal. +4. **A LanceDB-style approach that acknowledges WAL writes:** acknowledge after + the WAL write is durable, make graph visibility and compaction asynchronous, + and accept that the public contract does not yet include a hard retained- + storage bound or a complete online-GC proof. +5. **Keep the current direct-write path:** retain the existing graph-visible + commit acknowledgement and use batching/group commit as latency relief. + +The decision is **Option 1 now**. Preserve direct writes as the safe default, +run the first streaming profile on stock Lance with no OmniGraph MemWAL GC, +and keep the row, memory, recovery, sequencing, lifecycle, and terminal- +disposition contracts strict. The near-cap fold is now closed without lowering +the 8,192-row/32-MiB admission cap: fold scanning charges logical slices and +densifies selected rows before retention. The measured full-fold RSS delta was +284,934,144 bytes (about 272 MiB), guarded by a 384-MiB remeasurement tripwire. + +Managed reclamation, a hard retained-storage envelope, whole-root rotation, +and an upstream Lance patch are deferred optimizations. They can improve the +operating lifetime later, but they are no longer activation dependencies. +Option 4 remains useful precedent for the acknowledgement boundary; OmniGraph's +selected profile keeps stronger fencing, recovery, sequencing, and terminal- +disposition requirements than the surveyed LanceDB path. + +No option in this document authorizes public streaming today. + +## The contracts that must not be conflated + +There are three different success points: + +```text +durable stream admission + ↓ +fold into one or more Lance base tables + ↓ +one __manifest publication makes the graph change visible +``` + +- **WAL admission success** means the submitted data is durable and + recoverable after a crash. +- **Fold success** means the data has been resolved against graph state and + written into the appropriate base Lance dataset or datasets. +- **Graph commit success** means one `__manifest` publication made the complete + graph transition authoritative. + +An interactive `mutate` or `load` call acknowledges the third point. A stream +API may acknowledge the first point, but it must not call that acknowledgement +a committed or graph-visible write. + +The RFC-026 contract still requires every admitted write to have a provable +path to fold, correction or terminal disposition, quiescence, and `SEALED` +within the declared **row and memory** bounds. It deliberately does not promise +a retained-byte, file-count, or provider-quota bound for the selected profile. + +## Current facts + +The decision starts from these facts: + +- RFC-022's direct write path is correct and implemented. It is expensive for + many small writes because every request pays recovery, Lance table effects, + and graph publication. +- RFC-026 Phase A and the private B1 row/fold core are implemented for one + main-only, unsharded stream with one live writer process. +- A private B1 acknowledgement requires both Lance's durability watcher and a + same-writer post-durability epoch check. It is not a graph-commit + acknowledgement. +- `__manifest` remains the only graph-visibility point. A WAL, MemTable, or + flushed generation is pending physical state until a fold publishes it. +- Gate R0 found an all-shape closure bug. It is now fixed: fold accounting uses + logical Arrow slices and a dense `take`, the legal high-entropy near-cap + generation publishes successfully, and the 8,192-row/32-MiB admission bound + is unchanged. +- Stock Lance RC.1 creates a fresh randomized directory for each generation + materialization attempt, writes objects before the shard-manifest CAS, and + persists no cross-open materialization-attempt ID, cap, reservation, or + terminal receipt. +- Generic Lance version cleanup does not own `_mem_wal`. Deleting WAL objects + can weaken writer fencing, so OmniGraph does not delete raw `_mem_wal` paths. +- Ordinary object-store listing is not a complete retained-storage or billing + account. It may omit incomplete multipart uploads, provider-retained + versions, delete markers, locked objects, and local staging residue. + +## Comparison + +| Option | Early durable acknowledgement | `__manifest` remains graph visibility | Hard retained-storage bound | Online reuse of reclaimed space | Requires Lance change | Current disposition | +|---|---:|---:|---:|---:|---:|---| +| 1. Unbounded retain-all | Yes | Yes | No, deliberately | No | No | **Selected first profile** | +| 2. Local Lance patch + managed GC | Yes | Yes | Candidate | Yes | Yes | Deferred optional optimization | +| 3. Whole-root rotation/rebuild | Yes, when paired with 1 or 2 | Yes | Bounds the active root, not all retired roots by itself | Coarse-grained operator retirement | No Lance-internal GC change | Optional operational escape | +| 4. LanceDB-style WAL acknowledgement | Yes | Yes for committed reads; optional fresh reads would be a separate mode | No hard proof on the public OSS shape | No on the surveyed public OSS shape | No | Precedent; OmniGraph keeps stronger correctness contracts | +| 5. Direct-write path | No separate stream acknowledgement | Yes | Existing bounded request/recovery model | Ordinary Lance maintenance | No | Production default and safe fallback | + +## Option 1: unbounded retain-all on stock Lance — selected + +### Basic idea + +Delete nothing inside `_mem_wal`. OmniGraph uses stock Lance's durable writer, +replay, generation, and fold machinery, but never interprets raw paths as safe +garbage and never promises a maximum retained-object count or byte total. + +```text +open/claim writer and append WAL + ↓ +materialize and fold + ↓ +retain every MemWAL object indefinitely +``` + +Storage use is monotonic. If the provider runs out of capacity or refuses a +write, that failure is loud and typed through the existing write/recovery +boundary; the system must not silently drop acknowledged data or pretend the +failed write committed. Capacity monitoring is operational guidance, not a +correctness proof or admission watermark. + +### What must exist + +- The existing 8,192-row/32-MiB admission limit and one-resident-writer bound. +- A fold path that closes every admitted row shape within its separately + measured working-memory envelope. +- Durable watcher-plus-post-fence acknowledgement, typed ambiguity, recovery + barriers, compare-and-chain sequencing, attribution, lifecycle, and terminal + disposition before a public caller exists. +- Operator-visible backlog and observed storage, explicitly labelled as + advisory rather than a hard retained-storage or billing bound. +- No OmniGraph deletion of `_mem_wal`, including apparently orphaned random + generation roots or WAL fence sentinels. + +### Advantages + +- Uses stock Lance's MemWAL write machinery. +- Performs no unsafe raw-path deletion. +- Avoids making a missing stock-Lance accounting/receipt API a release-calendar + dependency. +- Is the shortest route to the selected durable-admission contract. + +### Limitations + +- Space never becomes reusable inside the root. +- Conservative row and Arrow-memory admission may fill one generation before + provider capacity is exhausted; this is not a retained-storage reservation. +- Provider quota may be exhausted without OmniGraph predicting the exact point. +- Repeated failures or ambiguous materialization attempts can retain additional + residue forever. +- A long-lived root may eventually require operator-provisioned capacity or a + whole-root rotation. That is an operational consequence, not an advertised + finite-root protocol. + +### Decision gate + +The storage-envelope/attempt-receipt questions remain recorded evidence, but +they are no longer activation gates because this profile makes no corresponding +bound. The closure gate at this layer is now green: every legal admitted +generation must continue to fold or reach a typed terminal/recovery state. The +near-cap closure cell remains a regression gate locally and on configured +RustFS. + +## Option 2: a narrow local Lance patch with managed GC + +### Basic idea + +Implement the missing generally useful MemWAL lifecycle primitives in Lance +now, pin OmniGraph to the exact reviewed patch revision, and submit the same +upstream-shaped change to Lance. OmniGraph does not wait for upstream review or +a release to use it, but it also does not move Lance's physical ownership into +an OmniGraph raw-path collector. + +The patch should expose an opaque Lance-owned protocol such as: + +```text +inspect exact reclaimable cut + ↓ +durably persist attempt and exact plan + ↓ +fence/advance writer epoch and successor sentinel + ↓ +delete the Lance-owned object set + ↓ +persist terminal receipt +``` + +### Required capabilities + +- Stable materialization-attempt identity or deterministic attempt-owned + output. +- Durable started/completed/ambiguous receipts across lost responses and cold + reopen. +- Source-derived maximum physical-growth reservations enforced before effects. +- Exact Lance-owned inventory or durable accounting, including multipart and + provider-retained state within the supported storage profile. +- Whole-cut eligibility proving the generation is in the base table, required + indexes have caught up, and retained history no longer needs it. +- Sentinel-first writer fencing and a post-success epoch check so deleting an + old WAL slot cannot let a stale writer receive clean success. +- Idempotent inspect/plan/execute recovery after partial deletion. +- Bounded claim, reclaim, receipt, and checkpoint history. + +### Advantages + +- Enables indefinitely running roots rather than finite retain-all lifetimes. +- Makes periodic GC a safe execution policy over a proved Lance operation. +- Keeps knowledge of Lance's private object layout and fencing rules in Lance. +- Can eventually replace conservative permanent charges with reusable + capacity. + +### Limitations + +- OmniGraph carries a temporary dependency delta until upstream accepts an + equivalent implementation. +- Every Lance upgrade must rebase, audit, and rerun the exact fault and storage + matrix. +- This is materially more work than the first retain-all milestone. +- If the patch becomes an indefinite OmniGraph-only series, it has become a + product fork regardless of what we call it. + +### Decision gate + +Option 2 is production-ready only after local and real object-store crash tests +cover plan creation, lost results, partial deletion, stale plans, stale +writers, unknown objects, cold reconstruction, hard admission-watermark +arithmetic, and bounded history. Passing the Lance patch tests alone does not +activate OmniGraph's public API; the common RFC-026 token, attribution, +lifecycle, correction, authorization, and graph-history contracts still apply. + +## Option 3: whole-root rotation or rebuild + +### Basic idea + +Avoid deleting individual objects inside a live MemWAL binding. Instead: + +1. stop admission; +2. resolve recovery and fold every acknowledged row that can be closed; +3. reach a durable `SEALED` state; +4. export/rebuild into a fresh graph root; +5. perform a reviewed authority cutover to the fresh state; and +6. retire the old root under an operator-controlled retention policy after all + processes that could write it are terminated. + +The new root/path is never reused, so a stale writer targeting the retired root +cannot alter the new authoritative root. + +This is a whole-graph operational cutover. It is not the same as rebinding one +table to a fresh physical lifetime inside the existing graph. Such an in-root +table rebind is inactive Phase-D work and would need its own recovery-owned +physical build plus one `__manifest` publication protocol before it could be +used here. + +### What it solves + +- Provides the escape hatch for Option 1 when permanent charges approach the + finite-root limit. +- Quarantines ambiguous or historically messy physical state rather than + teaching an application-level collector to interpret it. +- Reuses the project's strict export/import storage-format posture. + +### What it does not solve + +- Rotation does not prove that every row in an old root was folded. That proof + must happen before the authoritative cutover. +- It bounds the active root, not the aggregate bytes of all retained retired + roots. +- An old root is not safe to delete while a foreign/stale process can still + write it or while operator retention requires it. +- Frequent whole-root rebuilds may be too expensive for steady-state + operation. + +Option 3 is therefore an optional coarse operational escape, not part of the +first retain-all activation contract and not a replacement for a future safe +online-GC protocol. + +## Option 4: LanceDB-style acknowledgement of WAL writes + +### What LanceDB currently does + +The public LanceDB OSS path provides the clearest precedent for a narrower +contract: + +```text +restricted merge_insert upsert + ↓ +validate and collect the input for one shard + ↓ +reuse a cached ShardWriter + ↓ +ShardWriter::put + ↓ +put returns; with the default configuration the WAL is durable + ↓ +return version = 0 + ↓ +separate LSM-aware read (open PR) / external compactor +``` + +With Lance's default durable configuration, `put` waits for WAL persistence. +The result is intentionally not a normal base-table version: LanceDB returns +the submitted row count and `version = 0` because the insert/update split and +base-table effect are not known until later compaction. LanceDB permits +`durable_write = false`, and its adapter does not perform OmniGraph's separate +same-writer post-durability `check_fenced()`. Its clean return is therefore an +API precedent, not evidence for OmniGraph's acknowledgement guarantee. + +At the surveyed public revision, ordinary native scans do not merge this fresh +tier; the proposed LSM-aware read path remains a separate open change. The +public repository also contains no complete MemWAL fold worker, retained-byte +quota, materialization-attempt ledger, or MemWAL-specific GC. Remote/Cloud +lifecycle endpoints delegate to private server code and therefore do not +provide public evidence for those guarantees. + +This is not evidence that the missing lifecycle is impossible or that the +public adapter is an end-to-end production lifecycle. It shows only that +LanceDB exposes the earlier-admission API shape while the complete public OSS +read/fold/reclamation lifecycle remains separate or absent. + +### The corresponding OmniGraph contract + +An OmniGraph version of this option would acknowledge a **WAL write**, not a +graph commit: + +```text +acknowledged means: + ✓ the batch is durably present in MemWAL + ✓ OmniGraph can replay it after a supported crash + ✓ pre-admission shape and policy checks completed + +acknowledged does not mean: + ✗ the rows are visible to normal graph reads + ✗ graph-state-dependent validation completed + ✗ a graph commit or time-travel point exists + ✗ storage occupied by the WAL has a proved hard lifetime bound +``` + +An acknowledged row may therefore later reach a committed graph version, be +rejected by a state-dependent constraint with a durable terminal reason, or +remain blocked behind recovery/operator action. It may never disappear +silently. + +Normal committed reads would continue to use only `__manifest`-selected base +table versions. A separate explicitly weaker `Fresh` read could later merge +MemWAL data, but it would be a distinct read contract and must never silently +replace snapshot-isolated committed reads. + +### What this buys us + +- It removes fold and `__manifest` publication from the acknowledgement's + critical path; the real latency/throughput impact still requires the checked + local and object-store product instrument. +- Supports callers whose contract is durable buffering with delayed graph + visibility. +- A smaller first product surface: admission status, asynchronous fold, and + operator observation can precede online reclamation. +- Operational experience with real workloads before finalizing every long- + lifetime optimization. + +### What this gives up or defers + +- No advertised hard retained-storage bound on the current stock-Lance/public- + LanceDB shape. +- OmniGraph's explicit fold may converge acknowledged rows into the graph, but + the surveyed public LanceDB `optimize`/cleanup paths neither perform that + fold nor reclaim MemWAL. Retained MemWAL storage remains monotonic until a + whole-root retirement or a Lance-owned GC primitive exists. +- A long outage, repeated crash window, or stalled compactor can accumulate + physical residue until writes must be stopped or the root rebuilt. +- Without a complete attempt receipt, retrying an ambiguous call cannot be + presented as exactly once merely because the logical payload matches. +- Fresh reads, if added, have additional snapshot/deduplication semantics and + cannot provide graph-wide multi-dataset atomicity by reading table-local + MemWAL state directly. +- The retained-storage observations remain advisory. The separate near-cap + closure failure has been fixed and is no longer part of this trade-off. + +### Minimum safety floor even under this weaker option + +Choosing Option 4 must not mean copying every current LanceDB omission. At +minimum OmniGraph would still require: + +- `durable_write = true`; an in-memory-only acknowledgement is not allowed; +- successful per-put durability-watcher completion followed by the same + writer's post-durability `check_fenced()`; +- typed `AckUnknown` for every post-invocation ambiguous result; +- durable caller write IDs/tokens sufficient to prevent an unknown attempt + from being silently replayed as a newer direct write; +- durable per-write status that eventually records graph publication, a typed + terminal rejection/dead-letter disposition, or the exact recovery/operator + blocker; an acknowledged write may never be silently dropped; +- the common RFC-026 trusted-attribution and compare-and-chain token contracts, + so correction/retry cannot impersonate the acknowledged writer or reorder a + same-key history; +- a durable strict-block and correction/withdrawal/dead-letter protocol for + rows that fail deferred uniqueness, referential-integrity, cardinality, or + other graph-state-dependent validation; +- persistent quiesce/resume/abort lifecycle revisions and bounded management + receipts, so every acknowledged row has a proved publication or terminal + disposition before rebuild; +- the existing synchronous recovery barrier before another relevant writer + proceeds; +- one explicit API result whose vocabulary says `accepted` or `durable`, never + `committed` or `visible`; +- strict main-only/unsharded/single-live-writer-process refusals until broader + topology is separately proved; +- operator-visible fold backlog and observed storage, with monitoring and a + loud stop/rebuild threshold clearly labelled advisory: ordinary inventory + cannot prove billed bytes, so Option 4 explicitly accepts that a provider + quota may be exhausted before the observation threshold; and +- no raw `_mem_wal` deletion by OmniGraph. + +Fallback to the direct path is permitted only before the first possible WAL +effect. After `put` may have run, OmniGraph must resolve that attempt or return +`AckUnknown`/`RecoveryRequired`; it must never submit the same logical write +through the direct path merely because the WAL response was inconvenient. + +### Governance consequence + +RFC-026 now explicitly chooses the no-storage-bound part of this contract for +the retain-all profile. That decision does not import the surveyed LanceDB +adapter wholesale. The governing principle plus invariants 5, 6, 13, and 14 +still apply: + +- invariant 5 and the governing principle require acknowledged pending state + to remain durably recoverable until graph publication or a contract-defined + terminal disposition; Option 4 may not relax that rule; +- invariant 6 already distinguishes durable stream admission from a graph- + visible write, so early acknowledgement itself is compatible; +- invariant 13 still requires bounded, typed, observable memory, recovery, and + backpressure paths. Provider-capacity exhaustion is the explicit exception + in the selected storage profile and must fail loudly; it is not evidence that + the process can be left memory-unbounded; and +- invariant 14 requires an RFC and boundary-matched crash, compatibility, + refusal, recovery, and measured-performance evidence for this observable + guarantee change. + +The selected decision is that durable buffering before fold/publication is +worth monotonic-storage risk and delayed visibility. OmniGraph still promises +recoverability and contract-defined disposition for acknowledged data; it does +not promise indefinitely bounded retained storage. + +## Option 5: keep direct writes and improve batching + +### Basic idea + +Do not activate a public WAL admission contract yet. Continue to use the +implemented RFC-022 path: + +```text +validate and stage + ↓ +arm recovery + ↓ +commit every affected Lance table + ↓ +publish once through __manifest + ↓ +acknowledge graph-visible success +``` + +Reduce its fixed cost through caller batching, server-side group scheduling +that preserves request semantics, shared Lance sessions/caches, and measured +access-shape improvements. + +### Advantages + +- Preserves the strongest and already implemented success contract. +- Introduces no new retained MemWAL state or recovery surface. +- Remains the correct path for callers requiring immediate graph visibility. +- Provides a safe pre-effect fallback when stream admission cannot begin. + +### Limitations + +- Every logical batch still pays the graph commit protocol. +- It retains the complete graph-commit critical path for every acknowledged + request; the comparative latency/throughput effect must be measured rather + than inferred. +- Client-side buffering before submission is not durably protected by + OmniGraph. + +Direct writes remain necessary under every other option. Streaming is an +additional acknowledgement contract, not a replacement for transactions that +must be graph-visible on return. + +## Routes that are not valid options + +### Blind periodic garbage collection + +Running a timer does not establish that an object is garbage. A safe periodic +collector is the execution mechanism for Option 2 after Lance can prove an +exact eligible cut, fence writers, recover a partial deletion, and issue a +terminal receipt. “List everything unreferenced and old enough” may delete an +in-progress generation, history/index-required data, or a WAL sentinel needed +to fence a stale writer. + +### A separate OmniGraph WAL + +Building a new log, transaction manager, or LSM tree would duplicate Lance and +force OmniGraph to solve distributed fencing, replay, compaction, reclamation, +and backpressure twice. It remains denied by the architectural substrate rule. + +### Silent fallback after ambiguity + +No option may convert a possible durable WAL effect into a fresh direct write +without resolving the first attempt. That can duplicate, reorder, or overwrite +logical changes while returning an apparently clean result. + +## Selected staged route + +### Stage 1: preserve the baseline and keep closure green + +- Keep direct writes as the production default. +- Keep MemWAL admission crate-private and feature-gated. +- Keep the widest-shape fold closure cell green without lowering admission. +- Re-run the one-exclusive-fold RSS instrument when its 384-MiB CI + remeasurement tripwire is crossed; the tripwire is not a runtime allocator + limit. +- Preserve the post-durability epoch check and `AckUnknown` behavior. + +### Stage 2: implement the public-contract prerequisites privately + +- Implement compare-and-chain sequencing, trusted attribution, durable + lifecycle/correction, terminal management receipts, authorization, status, + and shutdown ownership on the existing manifest/recovery chassis. +- Keep the topology main-only, unsharded, and one live writer process. +- Make observed storage/backlog visible, but label it advisory and add no + artificial file or byte limit. +- Keep all public schema/SDK/HTTP/CLI/OpenAPI surfaces absent until this private + contract and its crash/cross-version evidence pass. + +### Stage 3: add product surfaces + +- Expose explicit durable-admission vocabulary, fold/block status, lifecycle + controls, Cedar, SDK, HTTP, CLI, and OpenAPI parity only after Stage 2 is + evidence-green. +- Document monotonic retained storage and provider exhaustion as the selected + operating posture. + +### Later, only if operations justify it + +- Add Option 3 when whole-root rotation has a concrete operator need. +- Implement Option 2 only when retained-storage cost justifies the Lance patch + and its crash/inventory/fencing matrix. +- Treat any hard byte/file admission bound or `GraphHistoryBudget` as a new + measured RFC decision, not prerequisite scaffolding for retain-all. + +## Decision statement + +OmniGraph chooses unbounded retain-all as the first streaming storage profile. +Lance MemWAL remains the substrate, `__manifest` remains the only graph- +visibility point, direct writes remain the production-safe default, and +OmniGraph performs no MemWAL GC. Row/memory/recovery/lifecycle/disposition +bounds remain strict; retained bytes and file count do not have an advertised +limit. A Lance-owned reclamation patch and whole-root rotation are optional +future improvements, not blockers for this profile. + +## References + +- [RFC-026: MemWAL streaming ingest](docs/rfcs/0026-memwal-streaming-ingest.md) +- [WAL thinking](WAL-thinking.md) +- [Lance MemWAL PR proposal](lance-memwal-pr.md) +- [OmniGraph write-path state of affairs](docs/dev/writing-path-state-of-affairs.md) +- [Lance MemWAL format](https://lance.org/format/table/mem_wal/) +- [LanceDB MemWAL write integration](https://github.com/lancedb/lancedb/blob/8450683b2aba033b12d8fe4c6e1601cc4b733b91/rust/lancedb/src/table/merge/lsm.rs) +- [LanceDB proposed LSM read path, PR #3489](https://github.com/lancedb/lancedb/pull/3489) +- [LanceDB remote lifecycle integration, PR #3501](https://github.com/lancedb/lancedb/pull/3501) From ac8c29aae7e85ee1d96ad7b8b5993e674f159a1a Mon Sep 17 00:00:00 2001 From: aaltshuler Date: Tue, 21 Jul 2026 15:09:20 +0300 Subject: [PATCH 2/2] Preserve Gate R0 failure diagnostics --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42e2c4a6..18dddb13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -211,7 +211,9 @@ jobs: # omnigraph/failpoints, so each line rebuilds only its crate. run: | cargo test --locked -p omnigraph-engine --features failpoints --test failpoints + set +e output=$(cargo test --locked -p omnigraph-engine --features failpoints --test memwal_stream_cost gate_r0_ -- --nocapture 2>&1); status=$? + set -e echo "$output" [ "$status" -eq 0 ] || exit "$status" echo "$output" | grep -Eq "test result: ok\. [1-9][0-9]* passed" \ @@ -244,7 +246,9 @@ jobs: # This is a remeasurement tripwire for the current single-exclusive- # fold shape, not a runtime allocator limit. Keep it outside the # gate_r0_ filter because the subprocess owner has a stable older name. + set +e output=$(cargo test --locked -p omnigraph-engine --features failpoints --test memwal_stream_cost widest_legal_generation_records_no_roll_estimates_and_peak_rss -- --exact --nocapture 2>&1); status=$? + set -e echo "$output" [ "$status" -eq 0 ] || exit "$status" echo "$output" | grep -Eq "test result: ok\. 1 passed" \