From 10075521ea759825ebf747396cddc165ad5a73a9 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 30 Jul 2026 10:14:07 -0700 Subject: [PATCH] feat(ledger): let an operator declare historical failed appends, so /verify can go green again The public verifier on the production ORB reports ok:false and cannot be made honest-green, because one of its two findings has no declaration mechanism. Measured there (2069 rows): - 83 content mismatches, all in seq 5..257, all in schemaVersion 2 and 3 -- every one of the 1,569 v5 rows verifies clean. Recomputed offline with the repo's own canonicalJson/contentDigest to confirm the count and range, then brute-forced every subset of null-valued keys per row (the obvious "stored as null, hashed as undefined" hypothesis): no subset reproduces the committed digest for any of the 83. Those preimages are genuinely unrecoverable, which LOOPOVER_LEDGER_CONTENT_WAIVER (#9850) already covers. - 231 unchained records -- decision_records rows no ledger row ever vouched for -- between 2026-07-04 and 2026-07-24, and NONE since. Historical residue from a durability bug that has stopped, against a structurally perfect chain. The content waiver deliberately cannot cover this (chain findings are never waivable), so there was no way to declare it. A permanently-red integrity endpoint is the worst possible signal: "red" stops distinguishing old damage from a real failed append tomorrow, which is the exact thing the check exists to catch. LOOPOVER_LEDGER_UNCHAINED_WAIVER declares it, mirroring #9850's four properties. Two differences, both forced and both argued in the parser's doc: - TIME-bounded, not seq-bounded, because an orphan has no ledger row and therefore no sequence number. #9850's rule against time bounds is about boundaries that MOVE; both bounds here are absolute instants, so the interval is exactly as fixed as a seq range. - A declared maxRecords closes what a seq range gets for free. A time interval could come to contain more orphans than were counted, so exceeding the declared count makes the WHOLE waiver stop applying -- a new failed append cannot hide behind an old declaration. A truncated TAIL stays unwaivable. Writing the test for that caught the first implementation excluding the window in SQL before distinguishing interior from tail, which silently waived the very attack signature the doc promised was never waivable; every waiver predicate now carries the interior test. --- apps/loopover-ui/public/openapi.json | 2 +- .../src/lib/selfhost-env-reference.ts | 5 + src/env.d.ts | 8 + src/openapi/spec.ts | 2 +- src/review/decision-record.ts | 126 +++++++++++++-- src/selfhost/preflight.ts | 21 ++- test/unit/decision-record.test.ts | 145 +++++++++++++++++- test/unit/selfhost-preflight.test.ts | 16 ++ 8 files changed, 308 insertions(+), 17 deletions(-) diff --git a/apps/loopover-ui/public/openapi.json b/apps/loopover-ui/public/openapi.json index 28e8d2aed6..7bba7c1a52 100644 --- a/apps/loopover-ui/public/openapi.json +++ b/apps/loopover-ui/public/openapi.json @@ -21052,7 +21052,7 @@ "summary": "Verify a window of the hash-chained decision ledger (resumable via afterSeq)", "responses": { "200": { - "description": "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, contentMismatches (#9850 -- the COUNT of rows whose preimage no longer matches the digest the chain committed to; a content mismatch no longer aborts the scan, because one unreconcilable row used to hide every row after it, so a structural break further along was unreachable. The verdict is unchanged: any mismatch is still ok:false and the first is still reported as `break`), waivedContentMismatches + contentWaiver (#9850 -- a DECLARED, seq-bounded exclusion from the content re-check for rows whose preimage is unrecoverable, set per-deployment via LOOPOVER_LEDGER_CONTENT_WAIVER. Both bounds and a reason are mandatory, the range and count are always published even on a clean chain, waived mismatches are counted separately and never folded into contentMismatches, and CHAIN checks are never waived -- a waived row that is mis-chained still fails), and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)." + "description": "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, contentMismatches (#9850 -- the COUNT of rows whose preimage no longer matches the digest the chain committed to; a content mismatch no longer aborts the scan, because one unreconcilable row used to hide every row after it, so a structural break further along was unreachable. The verdict is unchanged: any mismatch is still ok:false and the first is still reported as `break`), waivedContentMismatches + contentWaiver (#9850 -- a DECLARED, seq-bounded exclusion from the content re-check for rows whose preimage is unrecoverable, set per-deployment via LOOPOVER_LEDGER_CONTENT_WAIVER. Both bounds and a reason are mandatory, the range and count are always published even on a clean chain, waived mismatches are counted separately and never folded into contentMismatches, and CHAIN checks are never waived -- a waived row that is mis-chained still fails), waivedUnchainedRecords + unchainedWaiver (#9933 -- the sibling declaration for records no ledger row ever vouched for, which the CONTENT waiver deliberately cannot cover because chain findings are never waivable. Bounded by two ABSOLUTE instants plus a declared maximum count, so the window cannot drift and one more orphan inside it than declared makes the whole waiver stop applying; a truncated TAIL stays unwaivable), and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)." }, "409": { "description": "First break found: sequence_gap | predecessor_mismatch | row_hash_mismatch | missing_record | content_mismatch | short_tail (a record newer than the verified tip has no chain entry — the truncated-tail signature) | unchained_record (an INTERIOR record has no chain entry — the failed-append signature). Records younger than the 5-minute append grace window are not reported: the record insert and its chain append are two writes moments apart, and a verify landing between them is not evidence of tampering." diff --git a/apps/loopover-ui/src/lib/selfhost-env-reference.ts b/apps/loopover-ui/src/lib/selfhost-env-reference.ts index b37770fb62..45d9ef0c67 100644 --- a/apps/loopover-ui/src/lib/selfhost-env-reference.ts +++ b/apps/loopover-ui/src/lib/selfhost-env-reference.ts @@ -305,6 +305,10 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ name: "LOOPOVER_LEDGER_CONTENT_WAIVER", firstReference: "src/selfhost/preflight.ts", }, + { + name: "LOOPOVER_LEDGER_UNCHAINED_WAIVER", + firstReference: "src/selfhost/preflight.ts", + }, { name: "LOOPOVER_MCP_TOKEN", firstReference: "src/selfhost/preflight.ts", @@ -789,6 +793,7 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `LOOPOVER_LEDGER_ANCHOR_KEYS` | `src/selfhost/preflight.ts` |", "| `LOOPOVER_LEDGER_ANCHOR_PRIVATE_KEY` | `src/selfhost/preflight.ts` |", "| `LOOPOVER_LEDGER_CONTENT_WAIVER` | `src/selfhost/preflight.ts` |", + "| `LOOPOVER_LEDGER_UNCHAINED_WAIVER` | `src/selfhost/preflight.ts` |", "| `LOOPOVER_MCP_TOKEN` | `src/selfhost/preflight.ts` |", "| `LOOPOVER_METRICS_REPO_LABELS` | `src/server.ts` |", "| `LOOPOVER_PUBLIC_SCORE_TERMS_ALLOWED_REPOS` | `src/selfhost/inert-config.ts` |", diff --git a/src/env.d.ts b/src/env.d.ts index ac51f23e1c..5df8680ab8 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -640,6 +640,14 @@ declare global { * where the alternative is a permanently-failing public endpoint. Chain checks are never waived, both * bounds and a reason are mandatory, and the range plus its count are published by /verify. */ LOOPOVER_LEDGER_CONTENT_WAIVER?: string; + /** #9933: the sibling declaration for UNCHAINED records — decision_records rows no ledger row ever + * vouched for (the failed-append signature), which the content waiver deliberately cannot cover because + * chain findings are never waivable. Format `..||`: both bounds are + * ABSOLUTE instants (so the window cannot drift the way a relative one would), and the declared count is + * a fail-closed guard — one more orphan inside the window than declared and the whole waiver stops + * applying, so a new failed append can never hide behind an old declaration. A truncated TAIL stays + * unwaivable. See parseLedgerUnchainedWaiver for the full reasoning. */ + LOOPOVER_LEDGER_UNCHAINED_WAIVER?: string; LOOPOVER_LEDGER_ANCHOR_KEYS?: string; /** External ledger anchoring (#9270, epic #9267): the PRIVATE half of the anchor-signing keypair — an * ECDSA P-256 key in PKCS8 PEM (the `\n`-escaped single-line form is accepted, since that is how a key diff --git a/src/openapi/spec.ts b/src/openapi/spec.ts index a6edebc233..dc9a99b4cc 100644 --- a/src/openapi/spec.ts +++ b/src/openapi/spec.ts @@ -1882,7 +1882,7 @@ export function buildOpenApiSpec() { tags: ["Public"], summary: "Verify a window of the hash-chained decision ledger (resumable via afterSeq)", responses: { - 200: { description: "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, contentMismatches (#9850 -- the COUNT of rows whose preimage no longer matches the digest the chain committed to; a content mismatch no longer aborts the scan, because one unreconcilable row used to hide every row after it, so a structural break further along was unreachable. The verdict is unchanged: any mismatch is still ok:false and the first is still reported as `break`), waivedContentMismatches + contentWaiver (#9850 -- a DECLARED, seq-bounded exclusion from the content re-check for rows whose preimage is unrecoverable, set per-deployment via LOOPOVER_LEDGER_CONTENT_WAIVER. Both bounds and a reason are mandatory, the range and count are always published even on a clean chain, waived mismatches are counted separately and never folded into contentMismatches, and CHAIN checks are never waived -- a waived row that is mis-chained still fails), and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)." }, + 200: { description: "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, contentMismatches (#9850 -- the COUNT of rows whose preimage no longer matches the digest the chain committed to; a content mismatch no longer aborts the scan, because one unreconcilable row used to hide every row after it, so a structural break further along was unreachable. The verdict is unchanged: any mismatch is still ok:false and the first is still reported as `break`), waivedContentMismatches + contentWaiver (#9850 -- a DECLARED, seq-bounded exclusion from the content re-check for rows whose preimage is unrecoverable, set per-deployment via LOOPOVER_LEDGER_CONTENT_WAIVER. Both bounds and a reason are mandatory, the range and count are always published even on a clean chain, waived mismatches are counted separately and never folded into contentMismatches, and CHAIN checks are never waived -- a waived row that is mis-chained still fails), waivedUnchainedRecords + unchainedWaiver (#9933 -- the sibling declaration for records no ledger row ever vouched for, which the CONTENT waiver deliberately cannot cover because chain findings are never waivable. Bounded by two ABSOLUTE instants plus a declared maximum count, so the window cannot drift and one more orphan inside it than declared makes the whole waiver stop applying; a truncated TAIL stays unwaivable), and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)." }, 409: { description: "First break found: sequence_gap | predecessor_mismatch | row_hash_mismatch | missing_record | content_mismatch | short_tail (a record newer than the verified tip has no chain entry — the truncated-tail signature) | unchained_record (an INTERIOR record has no chain entry — the failed-append signature). Records younger than the 5-minute append grace window are not reported: the record insert and its chain append are two writes moments apart, and a verify landing between them is not evidence of tampering." }, }, }); diff --git a/src/review/decision-record.ts b/src/review/decision-record.ts index 4208351316..f694e4acab 100644 --- a/src/review/decision-record.ts +++ b/src/review/decision-record.ts @@ -620,6 +620,65 @@ export function parseLedgerContentWaiver(raw: string | undefined): LedgerContent return { fromSeq, toSeq, reason }; } +export type LedgerUnchainedWaiver = { fromIso: string; toIso: string; maxRecords: number; reason: string }; + +/** + * PURE. Parse `LOOPOVER_LEDGER_UNCHAINED_WAIVER`, format `..||`. + * + * WHY A SECOND WAIVER (#9933). `unchained_record` means a decision_records row exists that no ledger row ever + * vouched for -- the failed-append signature. The content waiver above deliberately cannot cover it (property + * 4: chain findings are never waivable), which is right for tampering but leaves a bounded, already-fixed + * historical failure permanently red. Measured on the production ORB: 231 orphans between 2026-07-04 and + * 2026-07-24 and NONE since, against an otherwise structurally perfect chain. A permanently-red integrity + * endpoint is the worst possible signal, because "red" stops distinguishing old damage from a real failed + * append tomorrow -- the precise thing this check exists to catch. + * + * WHY TIME-BOUNDED, given parseLedgerContentWaiver's property 1 says NOT to be. That rule is about boundaries + * that MOVE: a relative window ("the last 30 days") silently swallows new damage as the clock advances. Both + * bounds here are ABSOLUTE instants, so the interval is exactly as fixed as a seq range -- only an explicit + * edit widens it. Seq is not available as an alternative in any case: an orphan is defined by having no ledger + * row, so it has no sequence number to name. + * + * `maxRecords` closes the remaining gap that a seq range gets for free. A seq interval names a fixed number of + * rows; a time interval names a fixed span that could in principle come to contain MORE orphans than the + * operator counted. Declaring the count means a new failed append backdated into the window cannot hide behind + * the declaration: the moment the real count exceeds what was declared, the waiver stops applying entirely and + * the endpoint goes red. Fail-closed in the same direction as everything else here. + * + * The other three properties are unchanged and enforced below: both ends required, a reason is mandatory, and + * only the INTERIOR `unchained_record` kind is declarable -- `short_tail` (a record newer than the verified + * tip with no chain entry) stays unwaivable, because a truncated tail is exactly the attack this defends. + * + * Pipe-delimited rather than colon-delimited purely because ISO-8601 instants contain colons; a colon split + * could not tell the range from the reason. + * + * Returns null on anything malformed -- fail CLOSED, waiving nothing. preflight.ts surfaces the malformed + * value rather than leaving it silently inert. + */ +export function parseLedgerUnchainedWaiver(raw: string | undefined): LedgerUnchainedWaiver | null { + if (!raw) return null; + const parts = raw.split("|"); + if (parts.length < 3) return null; + const [rangePart, countPart, ...reasonParts] = parts; + const reason = reasonParts.join("|").trim(); + if (reason === "") return null; + // `parts.length >= 3` above already guarantees both indices, and the regex guarantees its own groups, so + // these are asserted rather than defaulted -- a `?? ""` fallback here would only add an arm no input can + // reach. Number()/Date.parse() of a genuinely bad value still lands on the NaN guard below. + const match = /^\s*(\S+)\s*\.\.\s*(\S+)\s*$/.exec(rangePart!); + if (!match) return null; + const fromMs = Date.parse(match[1]!); + const toMs = Date.parse(match[2]!); + // A descending or unparseable range is a mistake, not an intent. + if (Number.isNaN(fromMs) || Number.isNaN(toMs) || toMs < fromMs) return null; + // Must be a positive integer: "0" waives nothing while looking like a declaration, and a fractional or + // negative count is a typo. Number() rather than parseInt so "12abc" is rejected instead of read as 12. + const maxRecords = Number(countPart!.trim()); + if (!Number.isInteger(maxRecords) || maxRecords < 1) return null; + // Normalized to ISO so the published value is unambiguous regardless of how the operator wrote it. + return { fromIso: new Date(fromMs).toISOString(), toIso: new Date(toMs).toISOString(), maxRecords, reason }; +} + export async function verifyDecisionLedger( env: Env, afterSeq = 0, @@ -637,6 +696,11 @@ export async function verifyDecisionLedger( * contentMismatches, so "83 rows are excused" can never read as "83 rows are fine". */ waivedContentMismatches: number; contentWaiver: LedgerContentWaiver | null; + /** #9933: orphaned records inside the declared unchained waiver -- counted and published separately, never + * folded into a clean result's silence, so "231 records were never chained" can never read as "nothing to + * see here". 0 when no waiver is configured or none fell inside it. */ + waivedUnchainedRecords: number; + unchainedWaiver: LedgerUnchainedWaiver | null; break?: LedgerBreak; }> { const bounded = Math.max(1, Math.min(1000, limit)); @@ -662,6 +726,9 @@ export async function verifyDecisionLedger( let waivedContentMismatches = 0; let firstContentMismatch: LedgerBreak | null = null; const contentWaiver = parseLedgerContentWaiver(env.LOOPOVER_LEDGER_CONTENT_WAIVER); + // #9933: the sibling declaration for orphaned (never-appended) records -- see parseLedgerUnchainedWaiver. + const unchainedWaiver = parseLedgerUnchainedWaiver(env.LOOPOVER_LEDGER_UNCHAINED_WAIVER); + let waivedUnchainedRecords = 0; const decisionRecordsPruneCutoff = retentionCutoffIsoForTable("decision_records"); const [totalRow, globalTip, prior] = await Promise.all([ env.DB.prepare("SELECT COUNT(*) AS n FROM decision_ledger").first<{ n: number }>(), @@ -674,7 +741,7 @@ export async function verifyDecisionLedger( const tipSeq = globalTip?.seq ?? 0; const tipHash = globalTip?.rowHash ?? LEDGER_GENESIS_HASH; // `== null` deliberately: D1 drivers disagree on .first() returning null vs undefined for no-row. - if (afterSeq > 0 && prior == null) return { ok: false, checked: 0, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, waivedContentMismatches, contentWaiver, break: { kind: "sequence_gap", atSeq: afterSeq, expectedSeq: afterSeq } }; + if (afterSeq > 0 && prior == null) return { ok: false, checked: 0, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, waivedContentMismatches, contentWaiver, waivedUnchainedRecords, unchainedWaiver, break: { kind: "sequence_gap", atSeq: afterSeq, expectedSeq: afterSeq } }; let prevHash = prior?.rowHash ?? LEDGER_GENESIS_HASH; let expectedSeq = afterSeq + 1; const { results } = await env.DB.prepare( @@ -700,10 +767,10 @@ export async function verifyDecisionLedger( // from) so a call that finds ZERO new rows still has an anchor to reconcile against. let lastVerifiedCreatedAt = prior?.createdAt ?? null; for (const row of results) { - if (row.seq !== expectedSeq) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, waivedContentMismatches, contentWaiver, break: { kind: "sequence_gap", atSeq: row.seq, expectedSeq } }; - if (row.prevHash !== prevHash) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, waivedContentMismatches, contentWaiver, break: { kind: "predecessor_mismatch", atSeq: row.seq } }; + if (row.seq !== expectedSeq) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, waivedContentMismatches, contentWaiver, waivedUnchainedRecords, unchainedWaiver, break: { kind: "sequence_gap", atSeq: row.seq, expectedSeq } }; + if (row.prevHash !== prevHash) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, waivedContentMismatches, contentWaiver, waivedUnchainedRecords, unchainedWaiver, break: { kind: "predecessor_mismatch", atSeq: row.seq } }; const recomputed = await ledgerRowHash(prevHash, { seq: row.seq, recordId: row.recordId, recordDigest: row.recordDigest, createdAt: row.createdAt }); - if (recomputed !== row.rowHash) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, waivedContentMismatches, contentWaiver, break: { kind: "row_hash_mismatch", atSeq: row.seq } }; + if (recomputed !== row.rowHash) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, waivedContentMismatches, contentWaiver, waivedUnchainedRecords, unchainedWaiver, break: { kind: "row_hash_mismatch", atSeq: row.seq } }; // #9078: the promised record/ledger reconciliation — a row_hash chained cleanly can still commit to a // digest whose CONTENT has since been rewritten (or whose preimage is simply gone). Neither is visible to // the chain-only checks above, since those only ever compare ledger rows against each other. @@ -721,7 +788,7 @@ export async function verifyDecisionLedger( if (decisionRecordsPruneCutoff !== null && row.createdAt < decisionRecordsPruneCutoff) { prunedRecords += 1; } else { - return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, waivedContentMismatches, contentWaiver, break: { kind: "missing_record", atSeq: row.seq, recordId: row.recordId } }; + return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, waivedContentMismatches, contentWaiver, waivedUnchainedRecords, unchainedWaiver, break: { kind: "missing_record", atSeq: row.seq, recordId: row.recordId } }; } } else { let recomputedContentDigest: string | null = null; @@ -777,11 +844,39 @@ export async function verifyDecisionLedger( // the RECORD row, and this reconciliation only ever reports records that still exist. if (nextAfterSeq === null && lastVerifiedCreatedAt !== null) { const graceCutoffIso = new Date(Date.parse(nowIso()) - LEDGER_APPEND_GRACE_MS).toISOString(); - const orphan = await env.DB.prepare( - "SELECT id, created_at AS createdAt FROM decision_records r WHERE r.created_at <= ? AND NOT EXISTS (SELECT 1 FROM decision_ledger l WHERE l.record_id = r.id) ORDER BY r.created_at DESC LIMIT 1", - ) - .bind(graceCutoffIso) - .first<{ id: string; createdAt: string }>(); + // #9933: count what the declared waiver actually covers BEFORE deciding anything. The count is the guard + // that makes a time-bounded waiver as tight as a seq-bounded one: declaring 231 means a 232nd orphan + // appearing inside the same window -- a new failed append, or one backdated into it -- cannot hide behind + // the declaration. Over the declared maximum, the waiver stops applying in full and every orphan is + // reported again, rather than waiving the first N and quietly reporting the rest. + // + // Every waiver predicate carries `created_at <= lastVerifiedCreatedAt` -- the INTERIOR test. Excluding the + // window without it silently waived a short_tail sitting inside the window, i.e. exactly the truncated-tail + // attack the docs promise is unwaivable. Encoding "interior" in the SQL rather than checking it afterwards + // keeps the count and the break search agreeing on which rows the waiver covers. + if (unchainedWaiver !== null) { + const covered = await env.DB.prepare( + "SELECT COUNT(*) AS n FROM decision_records r WHERE r.created_at >= ? AND r.created_at <= ? AND r.created_at <= ? AND r.created_at <= ? AND NOT EXISTS (SELECT 1 FROM decision_ledger l WHERE l.record_id = r.id)", + ) + .bind(unchainedWaiver.fromIso, unchainedWaiver.toIso, lastVerifiedCreatedAt, graceCutoffIso) + .first<{ n: number }>(); + /* v8 ignore next -- defensive, mirroring the totalCount read above: a bare COUNT(*) always returns + * exactly one row (even {n: 0} against an empty table), so the fallback only satisfies .first()'s + * optional-by-signature TS return type. */ + const coveredCount = covered?.n ?? 0; + if (coveredCount <= unchainedWaiver.maxRecords) waivedUnchainedRecords = coveredCount; + } + // Orphans inside an APPLIED waiver are excluded from the break search; everything else is reported exactly + // as before. A waiver that failed its count guard applies to nothing, so the window is not excluded here. + const waiverApplies = unchainedWaiver !== null && waivedUnchainedRecords > 0; + const orphan = await (waiverApplies + ? env.DB.prepare( + "SELECT id, created_at AS createdAt FROM decision_records r WHERE r.created_at <= ? AND NOT (r.created_at >= ? AND r.created_at <= ? AND r.created_at <= ?) AND NOT EXISTS (SELECT 1 FROM decision_ledger l WHERE l.record_id = r.id) ORDER BY r.created_at DESC LIMIT 1", + ).bind(graceCutoffIso, unchainedWaiver.fromIso, unchainedWaiver.toIso, lastVerifiedCreatedAt) + : env.DB.prepare( + "SELECT id, created_at AS createdAt FROM decision_records r WHERE r.created_at <= ? AND NOT EXISTS (SELECT 1 FROM decision_ledger l WHERE l.record_id = r.id) ORDER BY r.created_at DESC LIMIT 1", + ).bind(graceCutoffIso) + ).first<{ id: string; createdAt: string }>(); // `== null` deliberately: D1 drivers disagree on .first() returning null vs undefined for no-row. if (orphan != null) { return { @@ -795,9 +890,14 @@ export async function verifyDecisionLedger( contentMismatches, waivedContentMismatches, contentWaiver, + waivedUnchainedRecords, + unchainedWaiver, break: orphan.createdAt > lastVerifiedCreatedAt - ? { kind: "short_tail", atSeq: expectedSeq - 1 } + ? // A truncated TAIL is never waivable -- see parseLedgerUnchainedWaiver. The waiver's window can + // only ever exclude interior orphans, and this arm is reached only for a record newer than the + // verified tip, which is the attack signature itself. + { kind: "short_tail", atSeq: expectedSeq - 1 } : { kind: "unchained_record", atSeq: expectedSeq - 1, recordId: orphan.id }, }; } @@ -805,9 +905,9 @@ export async function verifyDecisionLedger( // A content mismatch is still a FAILED verification -- the scan continuing does not soften the verdict, it // only stops one bad row from hiding every row after it. The first is reported as `break` exactly as before. if (firstContentMismatch !== null) { - return { ok: false, checked, nextAfterSeq, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, waivedContentMismatches, contentWaiver, break: firstContentMismatch }; + return { ok: false, checked, nextAfterSeq, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, waivedContentMismatches, contentWaiver, waivedUnchainedRecords, unchainedWaiver, break: firstContentMismatch }; } - return { ok: true, checked, nextAfterSeq, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, waivedContentMismatches, contentWaiver }; + return { ok: true, checked, nextAfterSeq, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, waivedContentMismatches, contentWaiver, waivedUnchainedRecords, unchainedWaiver }; } /** One ledger row, exactly as chained -- the shape `GET /v1/public/decision-ledger/row/:seq` returns. */ diff --git a/src/selfhost/preflight.ts b/src/selfhost/preflight.ts index 9f5812acdf..85400f1807 100644 --- a/src/selfhost/preflight.ts +++ b/src/selfhost/preflight.ts @@ -1,7 +1,7 @@ import { createPrivateKey } from "node:crypto"; import { CRON_INTERVAL_MIN_MS } from "./cron-alignment"; import { currentAnchorKey, parseAnchorPublicKeys } from "../review/ledger-anchor"; -import { parseLedgerContentWaiver } from "../review/decision-record"; +import { parseLedgerContentWaiver, parseLedgerUnchainedWaiver } from "../review/decision-record"; export type SelfHostPreflightProblem = { var: string; @@ -102,6 +102,24 @@ function checkLedgerContentWaiverConfig(problems: SelfHostPreflightProblem[], en } } +/** + * #9933: same reasoning as the content waiver above -- a malformed value waives nothing, silently, and the + * operator only discovers it when the public endpoint keeps reporting the orphans they thought they had + * declared. Note this check cannot validate the COUNT against reality (that needs the database); it only + * catches a value the parser rejects outright. + */ +function checkLedgerUnchainedWaiverConfig(problems: SelfHostPreflightProblem[], env: SelfHostPreflightEnv): void { + const raw = nonBlank(env["LOOPOVER_LEDGER_UNCHAINED_WAIVER"]); + if (!raw) return; + if (parseLedgerUnchainedWaiver(raw) === null) { + addProblem( + problems, + "LOOPOVER_LEDGER_UNCHAINED_WAIVER", + 'Set but unparseable, so NOTHING is waived and /v1/public/decision-ledger/verify will keep reporting the unchained records you meant to declare. Expected "..||" with two parseable timestamps, toIso >= fromIso, maxRecords a positive integer, and a non-empty reason.', + ); + } +} + function checkLedgerAnchorConfig(problems: SelfHostPreflightProblem[], env: SelfHostPreflightEnv): void { const rawKeys = nonBlank(env["LOOPOVER_LEDGER_ANCHOR_KEYS"]); const privateKey = nonBlank(env["LOOPOVER_LEDGER_ANCHOR_PRIVATE_KEY"]); @@ -366,6 +384,7 @@ export function preflightEnv(env: SelfHostPreflightEnv): SelfHostPreflightResult checkLedgerAnchorConfig(problems, env); checkLedgerContentWaiverConfig(problems, env); + checkLedgerUnchainedWaiverConfig(problems, env); return problems.length === 0 ? { ok: true, problems: [] } : { ok: false, problems }; } diff --git a/test/unit/decision-record.test.ts b/test/unit/decision-record.test.ts index f2dcde217c..bdd6be1d24 100644 --- a/test/unit/decision-record.test.ts +++ b/test/unit/decision-record.test.ts @@ -5,6 +5,7 @@ import { contentDigest, DECISION_RECORD_SCHEMA_VERSION, parseLedgerContentWaiver, + parseLedgerUnchainedWaiver, persistDecisionRecord, deriveReevaluationReason, REEVALUATION_REASON_BY_ORIGIN, @@ -396,7 +397,7 @@ describe("decision ledger (#8837)", () => { it("verifying a completely empty ledger returns ok:true with a zero tip, and skips the tail-truncation check (nothing to anchor against yet)", async () => { const env = createTestEnv(); const verified = await verifyDecisionLedger(env); - expect(verified).toEqual({ ok: true, checked: 0, nextAfterSeq: null, tipSeq: 0, tipHash: LEDGER_GENESIS_HASH, totalCount: 0, prunedRecords: 0, contentMismatches: 0, waivedContentMismatches: 0, contentWaiver: null }); + expect(verified).toEqual({ ok: true, checked: 0, nextAfterSeq: null, tipSeq: 0, tipHash: LEDGER_GENESIS_HASH, totalCount: 0, prunedRecords: 0, contentMismatches: 0, waivedContentMismatches: 0, contentWaiver: null, waivedUnchainedRecords: 0, unchainedWaiver: null }); }); it("TAIL TRUNCATION now breaks verify instead of passing clean (#9122): dropping the newest ledger rows leaves an orphaned decision_records tail", async () => { @@ -651,6 +652,98 @@ describe("verifier vs absence (#9474 pruned records, #9489 grace + interior orph expect(verified).toMatchObject({ ok: true, checked: 2, prunedRecords: 0 }); expect(verified.break).toBeUndefined(); }); + const orphanAt = async (env: Env, id: string, at: Date) => + env.DB.prepare( + "INSERT INTO decision_records (id, repo_full_name, pull_number, head_sha, action, reason_code, record_digest, record_json, created_at) SELECT ?, repo_full_name, 99, head_sha, action, reason_code, record_digest, record_json, ? FROM decision_records LIMIT 1", + ) + .bind(id, at.toISOString()) + .run(); + // A window that brackets daysAgo(2) but excludes daysAgo(1), so "inside" vs "outside" is unambiguous. + const window = (max: number) => `${daysAgo(3).toISOString()}..${daysAgo(1.5).toISOString()}|${max}|historical failed appends`; + + it("REGRESSION: a declared interior orphan no longer fails the chain, and is disclosed rather than hidden", async () => { + // Without this the production ORB sat permanently ok:false on 231 orphans from an already-fixed bug -- + // and a permanently-red integrity endpoint cannot report a NEW failed append, which is the whole point. + const env = createTestEnv({ LOOPOVER_LEDGER_UNCHAINED_WAIVER: window(5) }); + await persistAt(env, 1, daysAgo(3)); + await persistAt(env, 3, daysAgo(1)); + await orphanAt(env, "declared-orphan", daysAgo(2)); + + const verified = await verifyDecisionLedger(env); + + expect(verified.ok).toBe(true); + expect(verified.break).toBeUndefined(); + expect(verified.waivedUnchainedRecords).toBe(1); + expect(verified.unchainedWaiver).toMatchObject({ maxRecords: 5, reason: "historical failed appends" }); + }); + + it("INVARIANT: one more orphan than declared makes the WHOLE waiver stop applying -- new damage cannot hide behind an old declaration", async () => { + // The property that makes a time-bounded waiver as tight as a seq-bounded one. Declaring 1 and finding 2 + // must go red, not waive one and quietly report the other. + const env = createTestEnv({ LOOPOVER_LEDGER_UNCHAINED_WAIVER: window(1) }); + await persistAt(env, 1, daysAgo(3)); + await persistAt(env, 3, daysAgo(1)); + await orphanAt(env, "declared-orphan", daysAgo(2)); + await orphanAt(env, "undeclared-orphan", daysAgo(2.5)); + + const verified = await verifyDecisionLedger(env); + + expect(verified.ok).toBe(false); + expect(verified.waivedUnchainedRecords).toBe(0); + expect(verified.break).toMatchObject({ kind: "unchained_record" }); + }); + + it("INVARIANT: an orphan OUTSIDE the declared window is still reported", async () => { + const env = createTestEnv({ LOOPOVER_LEDGER_UNCHAINED_WAIVER: window(50) }); + await persistAt(env, 1, daysAgo(5)); + await persistAt(env, 3, daysAgo(0.5)); + await orphanAt(env, "outside-orphan", daysAgo(1)); + + const verified = await verifyDecisionLedger(env); + + expect(verified.ok).toBe(false); + expect(verified.break).toEqual({ kind: "unchained_record", atSeq: 2, recordId: "outside-orphan" }); + }); + + it("INVARIANT: a truncated TAIL is never waivable, even squarely inside the window", async () => { + // short_tail is the attack signature itself -- a record newer than the verified tip with no chain entry. + // The waiver may only ever excuse INTERIOR orphans. + const env = createTestEnv({ LOOPOVER_LEDGER_UNCHAINED_WAIVER: window(50) }); + await persistAt(env, 1, daysAgo(3)); + await orphanAt(env, "tail-orphan", daysAgo(2)); + + const verified = await verifyDecisionLedger(env); + + expect(verified.ok).toBe(false); + expect(verified.break).toMatchObject({ kind: "short_tail" }); + }); + + it("INVARIANT: no waiver configured is byte-identical to before -- the mechanism is inert unless declared", async () => { + const env = createTestEnv(); + await persistAt(env, 1, daysAgo(3)); + await persistAt(env, 3, daysAgo(1)); + await orphanAt(env, "interior-orphan", daysAgo(2)); + + const verified = await verifyDecisionLedger(env); + + expect(verified.ok).toBe(false); + expect(verified.waivedUnchainedRecords).toBe(0); + expect(verified.unchainedWaiver).toBeNull(); + expect(verified.break).toEqual({ kind: "unchained_record", atSeq: 2, recordId: "interior-orphan" }); + }); + + it("INVARIANT: a MALFORMED waiver waives nothing -- fail closed, never fail open", async () => { + const env = createTestEnv({ LOOPOVER_LEDGER_UNCHAINED_WAIVER: "garbage" }); + await persistAt(env, 1, daysAgo(3)); + await persistAt(env, 3, daysAgo(1)); + await orphanAt(env, "interior-orphan", daysAgo(2)); + + const verified = await verifyDecisionLedger(env); + + expect(verified.ok).toBe(false); + expect(verified.unchainedWaiver).toBeNull(); + }); + }); // #9850: a content mismatch used to ABORT verification, so one unreconcilable row was a denial-of- @@ -761,6 +854,56 @@ describe("parseLedgerContentWaiver (#9850)", () => { }); }); +describe("parseLedgerUnchainedWaiver (#9933)", () => { + const ok = "2026-07-04T00:00:00Z..2026-07-25T00:00:00Z|231|historical failed appends, fixed"; + + it("parses an absolute range, a declared count, and a reason", () => { + expect(parseLedgerUnchainedWaiver(ok)).toEqual({ + fromIso: "2026-07-04T00:00:00.000Z", + toIso: "2026-07-25T00:00:00.000Z", + maxRecords: 231, + reason: "historical failed appends, fixed", + }); + }); + + it("keeps a reason containing pipes intact -- only the first two fields are structural", () => { + expect(parseLedgerUnchainedWaiver("2026-07-04T00:00:00Z..2026-07-25T00:00:00Z|5|a|b|c")).toMatchObject({ reason: "a|b|c" }); + }); + + it("REQUIRES a reason -- you cannot waive silently", () => { + for (const bad of ["2026-07-04T00:00:00Z..2026-07-25T00:00:00Z|5|", "2026-07-04T00:00:00Z..2026-07-25T00:00:00Z|5| "]) { + expect(parseLedgerUnchainedWaiver(bad)).toBeNull(); + } + }); + + it("REQUIRES both bounds and a count -- an open-ended waiver is a blanket exemption wearing a range's clothes", () => { + for (const bad of [ + "2026-07-04T00:00:00Z|5|r", + "..2026-07-25T00:00:00Z|5|r", + "2026-07-04T00:00:00Z..|5|r", + "2026-07-04T00:00:00Z..2026-07-25T00:00:00Z|r", + ]) { + expect(parseLedgerUnchainedWaiver(bad)).toBeNull(); + } + }); + + it("rejects a descending range, which is a mistake rather than an intent", () => { + expect(parseLedgerUnchainedWaiver("2026-07-25T00:00:00Z..2026-07-04T00:00:00Z|5|r")).toBeNull(); + }); + + it("rejects a non-positive or non-integer count -- 0 waives nothing while looking like a declaration", () => { + for (const bad of ["...|0|r", "...|-3|r", "...|2.5|r", "...|12abc|r", "...|many|r"]) { + expect(parseLedgerUnchainedWaiver(bad.replace("...", "2026-07-04T00:00:00Z..2026-07-25T00:00:00Z"))).toBeNull(); + } + }); + + it("fails CLOSED on anything malformed, so a typo can never widen an exclusion", () => { + for (const bad of [undefined, "", " ", "all|5|r", "not-a-date..also-not|5|r", "2026-07-04T00:00:00Z-2026-07-25T00:00:00Z|5|r"]) { + expect(parseLedgerUnchainedWaiver(bad)).toBeNull(); + } + }); +}); + describe("content waiver applied by verifyDecisionLedger (#9850)", () => { const seedChained = async (env: Env, count: number) => { for (let i = 1; i <= count; i += 1) { diff --git a/test/unit/selfhost-preflight.test.ts b/test/unit/selfhost-preflight.test.ts index 176589ab63..d2460a3e5c 100644 --- a/test/unit/selfhost-preflight.test.ts +++ b/test/unit/selfhost-preflight.test.ts @@ -468,6 +468,22 @@ describe("ledger-anchor configuration preflight (#9769)", () => { expect(waiverProblems({ LOOPOVER_LEDGER_CONTENT_WAIVER: "5-257:pre-9123 record overwrite" })).toEqual([]); }); + // #9933: same class again for the sibling UNCHAINED waiver -- a value that parses to nothing is worse than + // an unset one, because the operator believes the orphans are declared. + const unchainedProblems = (env: Record) => + preflightEnv({ ...base, ...env }).problems.filter((p: SelfHostPreflightProblem) => p.var === "LOOPOVER_LEDGER_UNCHAINED_WAIVER"); + + it("flags a set-but-unparseable unchained waiver, which would otherwise waive nothing in silence", () => { + const problems = unchainedProblems({ LOOPOVER_LEDGER_UNCHAINED_WAIVER: "2026-07-04T00:00:00Z..2026-07-25T00:00:00Z|231" }); // no reason + expect(problems).toHaveLength(1); + expect(problems[0]!.message).toContain("NOTHING is waived"); + }); + + it("stays silent for an unset unchained waiver (opt-in) and for a well-formed one", () => { + expect(unchainedProblems({})).toEqual([]); + expect(unchainedProblems({ LOOPOVER_LEDGER_UNCHAINED_WAIVER: "2026-07-04T00:00:00Z..2026-07-25T00:00:00Z|231|historical failed appends" })).toEqual([]); + }); + it("INVARIANT: anchoring is opt-in — configuring none of it is never a problem", () => { expect(preflightEnv(base)).toEqual({ ok: true, problems: [] }); });