From c6423bde700f0813bf62e8b7c105f17678ea5902 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 30 Jul 2026 11:38:59 -0700 Subject: [PATCH 1/2] feat(fairness): one-command public verifier CLI Closes #9723. LoopOver publishes fairness numbers and says a stranger can check them, but checking them meant reimplementing canonical JSON, reimplementing ECDSA over it, and hand-diffing four endpoints -- so in practice nobody did, and a broken commitment could sit published indefinitely. npx -p @loopover/mcp loopover-verify Fetches the public surfaces, recomputes every commitment, prints a per-claim PASS/FAIL table and exits non-zero on any failure. Four claims: every recordDigest recomputes from its own record; every corpusChecksum rehashes to a downloadable corpus, with a scored record committing to the EMPTY corpus treated as a failure; the signed ledger checkpoint verifies offline against a published key, including that the displayed signingInput really is the preimage of the displayed payload; and the headline totals agree with the ledger-derived parity rollups. No credentials of any kind: every path is under /v1/public/ and no Authorization header is ever set, asserted by a test. To make that possible without a repo checkout, the digest primitives (canonicalJson/sha256Hex/contentDigest) and the VERIFYING half of ledger anchoring move into @loopover/contract, re-exported from their old homes so every call site is unchanged. Signing stays in the Worker, which is the only thing that may hold the private key. The anchor verifier's own comment used to say it was 'the function a third party reimplements' -- that reimplementation was the gap. A claim that could not be checked reports SKIP and is never counted as a pass. The api-schema generator now scans the bin directory rather than one hardcoded filename, so a future bin cannot silently opt out of response validation. Output is byte-identical today. --- packages/loopover-contract/package.json | 8 + .../loopover-contract/src/anchor-verify.ts | 102 +++++++ packages/loopover-contract/src/digest.ts | 54 ++++ packages/loopover-mcp/bin/loopover-verify.ts | 170 +++++++++++ .../loopover-mcp/lib/verify-public-claims.ts | 270 ++++++++++++++++++ packages/loopover-mcp/package.json | 3 +- scripts/gen-contract-api-schemas.ts | 26 +- src/review/decision-record.ts | 40 +-- src/review/ledger-anchor.ts | 115 ++------ test/unit/loopover-verify-cli.test.ts | 175 ++++++++++++ test/unit/verify-public-claims.test.ts | 237 +++++++++++++++ 11 files changed, 1075 insertions(+), 125 deletions(-) create mode 100644 packages/loopover-contract/src/anchor-verify.ts create mode 100644 packages/loopover-contract/src/digest.ts create mode 100644 packages/loopover-mcp/bin/loopover-verify.ts create mode 100644 packages/loopover-mcp/lib/verify-public-claims.ts create mode 100644 test/unit/loopover-verify-cli.test.ts create mode 100644 test/unit/verify-public-claims.test.ts diff --git a/packages/loopover-contract/package.json b/packages/loopover-contract/package.json index a567bbf05b..4e848abb6e 100644 --- a/packages/loopover-contract/package.json +++ b/packages/loopover-contract/package.json @@ -75,6 +75,14 @@ "./api-requests": { "types": "./dist/api-requests.d.ts", "default": "./dist/api-requests.js" + }, + "./digest": { + "types": "./dist/digest.d.ts", + "default": "./dist/digest.js" + }, + "./anchor-verify": { + "types": "./dist/anchor-verify.d.ts", + "default": "./dist/anchor-verify.js" } }, "files": [ diff --git a/packages/loopover-contract/src/anchor-verify.ts b/packages/loopover-contract/src/anchor-verify.ts new file mode 100644 index 0000000000..e74c766ec9 --- /dev/null +++ b/packages/loopover-contract/src/anchor-verify.ts @@ -0,0 +1,102 @@ +// The verifier half of LoopOver's ledger anchoring (#9723) — payload shape, signing input, key-id +// derivation, and signature verification. +// +// `src/review/ledger-anchor.ts` used to own all of this, and its own comment on the verify function said it +// "is the function a third party's own verifier reimplements". That was the bug #9723 exists to fix: a +// third party reimplementing ECDSA-over-canonical-JSON in another language is exactly where a verifier +// silently diverges from the signer and starts reporting green on anchors it never actually checked. +// +// So the VERIFYING half lives here, in the published leaf package, and an outsider imports it. The SIGNING +// half stays in the Worker, because it needs the operator's private key and nothing outside the Worker may +// hold it. That split is the point: everything a skeptic needs is published; nothing a skeptic must not +// have comes with it. +import { canonicalJson } from "./digest.js"; + +/** Bump when the payload's FIELD SET changes meaning. A verifier reads this FIRST and refuses shapes it does + * not understand, rather than silently misreading a future field set as the current one. */ +export const LEDGER_ANCHOR_PAYLOAD_VERSION = 1 as const; + +/** Identifies WHICH chain an anchor commits to. A fixed string today (one ledger), but present from v1 so a + * second anchored chain can never be confused for this one by a verifier holding both. */ +export const LEDGER_ANCHOR_LEDGER_ID = "loopover.decision_ledger"; + +/** The exact bytes an anchor commits to. `totalCount` is included alongside `seq` deliberately: a chain + * truncated and re-chained to the same length would still have to match BOTH, and the pair is what lets a + * verifier notice "the tip moved backwards" without fetching every row. */ +export type LedgerAnchorPayload = { + v: typeof LEDGER_ANCHOR_PAYLOAD_VERSION; + ledger: typeof LEDGER_ANCHOR_LEDGER_ID; + seq: number; + rowHash: string; + totalCount: number; + at: string; +}; + +/** A payload plus the operator signature over its canonical serialization, and the id of the key that signed + * it. `keyId` is REQUIRED: without it a verifier holding a rotation history cannot tell which published key + * a given anchor should be checked against, and would have to try them all -- turning a failed verification + * (a real signal) into an ambiguous one. */ +export type SignedLedgerAnchor = { + payload: LedgerAnchorPayload; + keyId: string; + /** base64 P-1363 (r||s) ECDSA signature over `canonicalJson(payload)`, as WebCrypto produces it. */ + signature: string; +}; + +/** One published anchor-signing public key and the window it was valid for. `notAfter: null` = still current. + * Rotation is why this is a LIST: an anchor signed in 2026 must stay verifiable after a 2027 rotation, so + * retired keys are published forever rather than replaced. */ +export type AnchorPublicKey = { + keyId: string; + /** base64 SPKI DER -- the same encoding Rekor's `verifier.publicKey.rawBytes` takes (#9272). */ + publicKeySpki: string; + notBefore: string; + notAfter: string | null; +}; + +export function base64ToBytes(base64: string): Uint8Array { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index); + return bytes; +} + +/** + * Derive a key's id FROM the key itself -- sha256(SPKI DER), first 16 hex chars. Deliberately not an operator- + * chosen label: a derived id cannot drift from the key it names, cannot be reused for a different key across + * a rotation, and lets a verifier confirm that the key they fetched is the key an anchor claims was used. + */ +export async function computeAnchorKeyId(publicKeySpkiBase64: string): Promise { + const digest = await crypto.subtle.digest("SHA-256", base64ToBytes(publicKeySpkiBase64) as Uint8Array); + return [...new Uint8Array(digest)] + .map((byte) => byte.toString(16).padStart(2, "0")) + .join("") + .slice(0, 16); +} + +/** The exact bytes signed and verified -- one definition both sides share, so a serialization change can + * never silently break verification while leaving signing "working". */ +export function anchorSigningInput(payload: LedgerAnchorPayload): string { + return canonicalJson(payload); +} + +/** + * Verify a signed anchor against a published public key. Returns a boolean rather than throwing for an + * ordinarily-invalid input (wrong key, tampered payload, malformed signature) -- those are all "not verified", + * a fact about the anchor, not a caller bug. + */ +export async function verifyLedgerAnchorSignature(signed: SignedLedgerAnchor, publicKeySpkiBase64: string): Promise { + try { + const key = await crypto.subtle.importKey("spki", base64ToBytes(publicKeySpkiBase64) as Uint8Array, { name: "ECDSA", namedCurve: "P-256" }, true, [ + "verify", + ]); + return await crypto.subtle.verify( + { name: "ECDSA", hash: "SHA-256" }, + key, + base64ToBytes(signed.signature) as Uint8Array, + new TextEncoder().encode(anchorSigningInput(signed.payload)), + ); + } catch { + return false; + } +} diff --git a/packages/loopover-contract/src/digest.ts b/packages/loopover-contract/src/digest.ts new file mode 100644 index 0000000000..83a126446f --- /dev/null +++ b/packages/loopover-contract/src/digest.ts @@ -0,0 +1,54 @@ +// The one serialization rule every LoopOver commitment is computed under (#9723). +// +// `canonicalJson` + `sha256Hex` define every published digest in this system: `recordDigest` on a decision +// record, `corpusChecksum` on an eval-score record, the eval-corpus checksum, and the anchor payload's own +// digest. They lived in `src/review/decision-record.ts` -- inside the Worker, which is not published -- +// so the third-party verifier #9723 asks for could not recompute a single one of those digests without a +// repo checkout, and the "no repo checkout required beyond npx" requirement was unsatisfiable by +// construction. +// +// They live HERE, in the leaf contract package, for the same reason every other shared vocabulary does: +// this is the one module both the Worker and the separately-published CLI can import. `decision-record.ts` +// re-exports them, so every existing call site is unchanged and there is exactly one definition. +// +// DO NOT "improve" the serialization. These bytes are the preimage of digests already published, anchored, +// and quoted back to us by outside readers; any change silently invalidates every commitment ever made. +// The rules, stated so a reimplementation in another language can match them exactly: +// +// • Object keys are sorted by JS string order (UTF-16 code unit), ascending. +// • Keys whose value is `undefined` are OMITTED. A top-level `undefined` serializes as `null`. +// • No whitespace anywhere; strings and numbers use `JSON.stringify`'s own encoding. +// • Arrays keep their order -- ordering an array is the caller's job, not the serializer's. + +/** + * Deterministic JSON: sorted keys, `undefined` properties dropped, no whitespace. + * + * Throws on values JSON cannot represent (functions, symbols, bigints) rather than coercing them -- + * a silent wrong digest is far worse than a loud failure, because it validates as "just a mismatch". + */ +export function canonicalJson(value: unknown): string { + if (value === null || typeof value === "number" || typeof value === "boolean") return JSON.stringify(value); + if (typeof value === "string") return JSON.stringify(value); + if (typeof value === "undefined") return "null"; + if (Array.isArray(value)) return `[${value.map((entry) => canonicalJson(entry)).join(",")}]`; + if (typeof value === "object") { + const record = value as Record; + const keys = Object.keys(record) + .filter((key) => record[key] !== undefined) + .sort(); + return `{${keys.map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(",")}}`; + } + // Functions/symbols/bigints have no JSON meaning; refusing loudly beats a silent wrong digest. + throw new Error(`canonicalJson: unsupported value type "${typeof value}"`); +} + +/** SHA-256 of `text` as lowercase hex, via Web Crypto -- present in both Workers and Node 22. */ +export async function sha256Hex(text: string): Promise { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text)); + return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +/** The digest of a value under the canonical serialization: `sha256Hex(canonicalJson(value))`. */ +export async function contentDigest(value: unknown): Promise { + return sha256Hex(canonicalJson(value)); +} diff --git a/packages/loopover-mcp/bin/loopover-verify.ts b/packages/loopover-mcp/bin/loopover-verify.ts new file mode 100644 index 0000000000..b2549eb243 --- /dev/null +++ b/packages/loopover-mcp/bin/loopover-verify.ts @@ -0,0 +1,170 @@ +#!/usr/bin/env node +// `loopover-verify` — the one-command public verifier (#9723, epic #9722). +// +// npx -p @loopover/mcp loopover-verify +// +// Fetches LoopOver's public fairness surfaces, RECOMPUTES every commitment they publish, and prints a +// per-claim PASS/FAIL table. Exits non-zero if any claim fails, so it is usable as a CI assertion by +// somebody who does not trust us -- which is the only kind of verification that means anything. +// +// ZERO CREDENTIALS, BY CONSTRUCTION. Every path fetched below is under `/v1/public/`, no Authorization +// header is ever set, and no token is read from the environment. A verifier that needed our credentials +// would be verifying nothing; the point is that a stranger on a clean machine gets the same answer we do. +// +// The checks themselves live in `../lib/verify-public-claims.js` and are pure. This file is only: parse +// argv, fetch, render, exit. +import { argsWantJson, describeCliError, reportCliFailure } from "../lib/cli-error.js"; +import { formatTable } from "../lib/format-table.js"; +import { + checkAnchorCheckpoint, + checkCorpusCommitments, + checkRecordDigests, + checkStatsParity, + exitCodeFor, + summarize, + type ClaimResult, + type VerifiableEvalRecord, +} from "../lib/verify-public-claims.js"; + +const defaultApiUrl = "https://api.loopover.ai"; + +/** Per-request timeout. A verifier that hangs forever on one unreachable surface is worse than one that + * reports that surface as unreachable and carries on with the rest. */ +const requestTimeoutMs = 15_000; + +type FetchOutcome = { ok: true; value: T } | { ok: false; error: string }; + +function usage(): string { + return [ + "loopover-verify — independently verify LoopOver's published fairness claims.", + "", + "Usage: npx -p @loopover/mcp loopover-verify [options]", + "", + "Options:", + " --base-url API base to verify (default: " + defaultApiUrl + ")", + " --json emit machine-readable results instead of a table", + " --help show this message", + "", + "Exits 0 when every checked claim passes, 1 when any claim fails, 2 on a usage error.", + "Requires no credentials of any kind.", + ].join("\n"); +} + +/** Read `--base-url`, normalising away a trailing slash so path joins cannot double up. */ +export function parseBaseUrl(args: readonly string[], fallback = defaultApiUrl): { ok: true; baseUrl: string } | { ok: false; error: string } { + const index = args.indexOf("--base-url"); + if (index === -1) return { ok: true, baseUrl: fallback.replace(/\/+$/, "") }; + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) return { ok: false, error: "--base-url requires a URL" }; + try { + // Constructed rather than regex-checked: `new URL` is the same parser the fetch below will use, so a + // value that passes here cannot fail differently later. + const parsed = new URL(value); + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return { ok: false, error: `--base-url must be http(s), got "${parsed.protocol}"` }; + } catch { + return { ok: false, error: `--base-url is not a valid URL: "${value}"` }; + } + return { ok: true, baseUrl: value.replace(/\/+$/, "") }; +} + +/** + * GET a public JSON endpoint. Never throws and never sends credentials: a failed fetch becomes a described + * outcome the caller turns into a `skip`, so one unavailable surface cannot abort the whole run and hide + * the claims that WOULD have been checkable. + */ +async function apiGet(baseUrl: string, path: string): Promise> { + try { + const response = await fetch(`${baseUrl}${path}`, { + headers: { accept: "application/json" }, + signal: AbortSignal.timeout(requestTimeoutMs), + }); + if (!response.ok) return { ok: false, error: `HTTP ${response.status}` }; + return { ok: true, value: (await response.json()) as T }; + } catch (error) { + return { ok: false, error: describeCliError(error) }; + } +} + +/** Render one claim's row. The status column is a bare word so the output greps and diffs cleanly. */ +function renderTable(results: readonly ClaimResult[]): string { + return formatTable({ + headers: [ + { key: "status", label: "RESULT" }, + { key: "claim", label: "CLAIM" }, + { key: "detail", label: "DETAIL" }, + ], + rows: results.map((result) => ({ status: result.status.toUpperCase(), claim: result.claim, detail: result.detail })), + }); +} + +export async function runVerify(args: readonly string[], baseUrlOverride?: string): Promise { + const wantsJson = argsWantJson(args); + if (args.includes("--help") || args.includes("-h")) { + process.stdout.write(`${usage()}\n`); + return 0; + } + const parsed = parseBaseUrl(args, baseUrlOverride ?? defaultApiUrl); + if (!parsed.ok) return reportCliFailure(wantsJson, parsed.error); + const { baseUrl } = parsed; + + // Fetched together: they are independent reads, and a verifier that serialises four round trips for no + // reason is a slower verifier with no compensating property. + const [scoresOutcome, statsOutcome, checkpointOutcome, keysOutcome] = await Promise.all([ + apiGet<{ records?: VerifiableEvalRecord[] }>(baseUrl, "/v1/public/eval-scores"), + apiGet<{ totals?: { handled?: unknown }; reviewParity?: { verdicts?: unknown } }>(baseUrl, "/v1/public/stats"), + apiGet<{ signed?: unknown; signingInput?: unknown }>(baseUrl, "/v1/public/decision-ledger/anchor-payload"), + apiGet<{ keys?: unknown[] }>(baseUrl, "/v1/public/decision-ledger/anchor-key"), + ]); + + const records = scoresOutcome.ok && Array.isArray(scoresOutcome.value.records) ? scoresOutcome.value.records : []; + + // One corpus fetch per rule a record actually commits to -- not a blind enumeration. A rule with no + // published corpus simply stays absent from the map, which `checkCorpusCommitments` reports honestly. + const ruleIds = [...new Set(records.map((record) => record.workUnit?.ruleId).filter((ruleId): ruleId is string => typeof ruleId === "string"))]; + const corpusEntries = await Promise.all( + ruleIds.map(async (ruleId) => { + const outcome = await apiGet<{ cases?: unknown; checksum?: unknown }>(baseUrl, `/v1/public/eval-corpus?rule_id=${encodeURIComponent(ruleId)}`); + return [ruleId, outcome.ok ? outcome.value : undefined] as const; + }), + ); + + const results: ClaimResult[] = []; + if (!scoresOutcome.ok) { + results.push({ id: "record-digests", claim: "Every eval-score record's recordDigest recomputes from its own contents", status: "skip", detail: `/v1/public/eval-scores unavailable (${scoresOutcome.error})` }); + results.push({ id: "corpus-commitments", claim: "Every corpusChecksum matches a downloadable corpus, and no scored record commits to an empty one", status: "skip", detail: `/v1/public/eval-scores unavailable (${scoresOutcome.error})` }); + } else { + results.push(await checkRecordDigests(records)); + results.push(await checkCorpusCommitments(records, new Map(corpusEntries))); + } + + // A 404 here is the DOCUMENTED "anchor signing not configured / ledger empty" response, not an outage, + // so an unavailable checkpoint is passed through as `undefined` and reported as a skip by the check + // itself rather than being special-cased into a second, subtly different skip message here. + const keys = keysOutcome.ok && Array.isArray(keysOutcome.value.keys) ? (keysOutcome.value.keys as Parameters[1]) : []; + results.push(await checkAnchorCheckpoint(checkpointOutcome.ok ? checkpointOutcome.value : undefined, keys)); + + results.push( + statsOutcome.ok + ? checkStatsParity(statsOutcome.value, statsOutcome.value.reviewParity) + : { id: "stats-parity", claim: "Published headline stats agree with the ledger-derived parity rollups", status: "skip", detail: `/v1/public/stats unavailable (${statsOutcome.error})` }, + ); + + if (wantsJson) { + process.stdout.write(`${JSON.stringify({ baseUrl, results, summary: summarize(results) }, null, 2)}\n`); + } else { + process.stdout.write(`Verifying ${baseUrl} (no credentials used)\n\n${renderTable(results)}\n\n${summarize(results)}\n`); + } + return exitCodeFor(results); +} + +// Only self-executes as a real CLI. Importing this file in a test must not run the verifier or call +// process.exit -- see the in-process CLI harness pattern the other bins' tests use. +if (process.argv[1] !== undefined && import.meta.url === `file://${process.argv[1]}`) { + runVerify(process.argv.slice(2)) + .then((code) => { + process.exitCode = code; + }) + .catch((error: unknown) => { + process.exitCode = reportCliFailure(argsWantJson(process.argv.slice(2)), describeCliError(error)); + }); +} diff --git a/packages/loopover-mcp/lib/verify-public-claims.ts b/packages/loopover-mcp/lib/verify-public-claims.ts new file mode 100644 index 0000000000..1e12a7d73c --- /dev/null +++ b/packages/loopover-mcp/lib/verify-public-claims.ts @@ -0,0 +1,270 @@ +// The public verifier's claim checks (#9723, epic #9722). +// +// LoopOver publishes fairness numbers and says a stranger can check them. Until now "checking them" meant +// reading `verify-this-review.mdx`, reimplementing canonical JSON, reimplementing ECDSA-over-that, and +// hand-diffing four endpoints -- so in practice nobody ever did, and a broken commitment could have sat +// published indefinitely. This module is the check itself, as code, runnable by anyone with `npx`. +// +// EVERY FUNCTION HERE IS PURE. They take already-fetched payloads and return verdicts; the bin does the +// network. That is not a testing convenience -- it is what makes the verifier auditable. A reader can see +// exactly what is being asserted without tracing HTTP, and every claim below is exercised in tests against +// hand-written payloads including the failure side, so "the verifier would have caught it" is a tested +// property rather than a hope. +// +// WHAT A FAILURE MEANS. `fail` is always a statement about the DEPLOYMENT, never about the verifier's own +// luck: a digest that does not recompute, a commitment to a corpus nobody can obtain, an anchor whose +// signature does not verify. Anything the verifier merely could not obtain -- an endpoint that is disabled, +// a corpus behind a rule that publishes none -- is `skip`, reported as such and NOT counted as a pass. +// Collapsing "could not check" into "checked, fine" is the single easiest way to build a verifier that +// always says green, so the two are kept structurally distinct all the way to the exit code. +import { contentDigest, canonicalJson, sha256Hex } from "@loopover/contract/digest"; +import { anchorSigningInput, verifyLedgerAnchorSignature, type AnchorPublicKey, type SignedLedgerAnchor } from "@loopover/contract/anchor-verify"; + +export type ClaimStatus = "pass" | "fail" | "skip"; + +/** One checked claim. `detail` is written to be read by someone who did not write this code: on a failure it + * names what was expected and what was found, because "FAIL" with no preimage is not evidence of anything. */ +export type ClaimResult = { + id: string; + claim: string; + status: ClaimStatus; + detail: string; +}; + +/** An eval-score record, narrowed to the fields the checks read. Deliberately structural rather than the + * contract's full inferred type: the verifier must be able to check a record published by a DEPLOYMENT + * RUNNING AN OLDER BUILD than the CLI, and a nominal type would reject it before a single digest was + * recomputed -- turning "their build is older" into a crash instead of a report. */ +export type VerifiableEvalRecord = { + recordDigest?: unknown; + score?: { decided?: unknown } | undefined; + commitments?: { corpusChecksum?: unknown } | undefined; + workUnit?: { ruleId?: unknown; kind?: unknown } | undefined; + subject?: { id?: unknown } | undefined; +}; + +/** The checksum of an EMPTY case list -- `sha256(canonicalJson([]))`, i.e. sha256("[]"). Computed, never + * hardcoded, so it cannot drift from the serialization it is supposed to represent. */ +export async function emptyCorpusChecksum(): Promise { + return sha256Hex(canonicalJson([])); +} + +/** A short, stable label for a record in failure text. Falls back through the identifying fields rather than + * indexing blindly, since a malformed record is exactly the case this text has to describe. */ +export function describeRecord(record: VerifiableEvalRecord, index: number): string { + const rule = typeof record.workUnit?.ruleId === "string" ? record.workUnit.ruleId : null; + const subject = typeof record.subject?.id === "string" ? record.subject.id : null; + return rule ?? subject ?? `record[${index}]`; +} + +/** + * CLAIM 1: every published `recordDigest` is the digest of the record that carries it. + * + * The preimage is the record MINUS `recordDigest` -- a digest cannot cover itself -- under the same + * canonical serialization the Worker used. Because `canonicalJson` sorts keys, the field order the endpoint + * happened to serialize in is irrelevant, which is what makes this checkable at all from parsed JSON. + * + * A record with no `recordDigest` at all is a FAIL, not a skip: an unsigned record among signed ones is the + * exact shape a silently-dropped commitment takes, and skipping it would hide it. + */ +export async function checkRecordDigests(records: readonly VerifiableEvalRecord[]): Promise { + const claim = "Every eval-score record's recordDigest recomputes from its own contents"; + if (records.length === 0) { + return { id: "record-digests", claim, status: "skip", detail: "no eval-score records published" }; + } + const failures: string[] = []; + for (const [index, record] of records.entries()) { + const label = describeRecord(record, index); + const published = record.recordDigest; + if (typeof published !== "string" || published === "") { + failures.push(`${label}: no recordDigest published`); + continue; + } + const { recordDigest: _omitted, ...preimage } = record as Record & { recordDigest?: unknown }; + let recomputed: string; + try { + recomputed = await contentDigest(preimage); + } catch (error) { + // canonicalJson refuses values JSON cannot represent. Reaching here means the payload held something + // no JSON endpoint can actually emit, so report it rather than letting the throw kill every later claim. + failures.push(`${label}: not serializable (${error instanceof Error ? error.message : String(error)})`); + continue; + } + if (recomputed !== published) failures.push(`${label}: published ${published.slice(0, 16)}…, recomputed ${recomputed.slice(0, 16)}…`); + } + return failures.length === 0 + ? { id: "record-digests", claim, status: "pass", detail: `${records.length} record(s) recomputed exactly` } + : { id: "record-digests", claim, status: "fail", detail: failures.join("; ") }; +} + +/** + * CLAIM 2: each record's `corpusChecksum` is the checksum of a corpus that can actually be downloaded. + * + * `corpusByRuleId` holds what `/v1/public/eval-corpus?rule_id=…` returned for each rule -- absent when that + * fetch found nothing. The checksum is recomputed from the CASES as served, which is the whole point: a + * commitment is only worth anything if it covers bytes the reader can obtain, so this deliberately does not + * trust the corpus payload's own `checksum` field and recomputes from `cases` instead. + * + * The empty-corpus rule is the sharp edge #9723 names. `sha256(canonicalJson([]))` is the SAME value for + * every rule and every deployment, so a record committing to it commits to nothing -- while looking exactly + * like a real commitment to anyone eyeballing a hex string. Paired with `decided > 0` ("we scored real work") + * that is a contradiction, and it is reported as a failure rather than a curiosity. + */ +export async function checkCorpusCommitments( + records: readonly VerifiableEvalRecord[], + corpusByRuleId: ReadonlyMap, +): Promise { + const claim = "Every corpusChecksum matches a downloadable corpus, and no scored record commits to an empty one"; + const committed = records.filter((record) => typeof record.commitments?.corpusChecksum === "string"); + if (committed.length === 0) { + return { id: "corpus-commitments", claim, status: "skip", detail: "no record published a corpusChecksum" }; + } + const emptyChecksum = await emptyCorpusChecksum(); + const failures: string[] = []; + const checked: string[] = []; + // Counted separately from `checked`, which also collects "corpus not published" notes. A claim that + // rehashed NOTHING has verified nothing, and reporting that as a pass -- with a detail line that openly + // says "unverified" -- is precisely the "could not check, therefore fine" collapse this module's header + // rules out. It is a skip. + let rehashed = 0; + for (const [index, record] of committed.entries()) { + const label = describeRecord(record, index); + const published = record.commitments?.corpusChecksum as string; + const decided = typeof record.score?.decided === "number" ? record.score.decided : 0; + + if (published === emptyChecksum && decided > 0) { + failures.push(`${label}: commits to the EMPTY corpus while reporting decided=${decided}`); + continue; + } + const ruleId = typeof record.workUnit?.ruleId === "string" ? record.workUnit.ruleId : null; + const corpus = ruleId === null ? undefined : corpusByRuleId.get(ruleId); + if (corpus === undefined || !Array.isArray(corpus.cases)) { + // Not a failure: #9805 deliberately omits a commitment for a truncated or empty corpus, and a rule can + // legitimately publish no corpus. What would be a failure -- committing to nothing while claiming to + // have decided something -- is already caught above. + checked.push(`${label}: corpus not published (unverified)`); + continue; + } + const recomputed = await sha256Hex(canonicalJson(corpus.cases)); + if (recomputed !== published) failures.push(`${label}: committed ${published.slice(0, 16)}…, corpus hashes to ${recomputed.slice(0, 16)}…`); + else { + rehashed += 1; + checked.push(`${label}: ${corpus.cases.length} case(s) rehashed exactly`); + } + } + if (failures.length > 0) return { id: "corpus-commitments", claim, status: "fail", detail: failures.join("; ") }; + if (rehashed === 0) { + return { id: "corpus-commitments", claim, status: "skip", detail: `${committed.length} commitment(s) published, but no corresponding corpus is downloadable -- nothing could be rehashed` }; + } + return { id: "corpus-commitments", claim, status: "pass", detail: checked.join("; ") }; +} + +/** + * CLAIM 3: the current signed ledger checkpoint verifies offline, against a published key. + * + * "Offline" is load-bearing: the signature is checked with the operator's PUBLISHED public key and nothing + * else. No call goes back to any LoopOver service that could simply assert success -- this recomputes the + * signing input and runs ECDSA locally, so a deployment cannot pass by claiming to have passed. + * + * TWO independent things are checked, and the second is the one that matters most: + * + * 1. The signature verifies over the payload, under the key the checkpoint itself names. The key is + * selected by `keyId`, never by trying them all -- see `SignedLedgerAnchor`'s own doc for why a scan + * turns a real failure into an ambiguous one. + * 2. The `signingInput` the endpoint SHOWS equals the canonical serialization of the payload it shows. + * Without this, a deployment could display one payload and sign a different set of bytes, and check 1 + * would still pass against the bytes actually signed. Recomputing the preimage locally is what closes + * that gap, and it is exactly the check a reader cannot perform by eye. + */ +export async function checkAnchorCheckpoint( + checkpoint: { signed?: unknown; signingInput?: unknown } | undefined, + keys: readonly AnchorPublicKey[], +): Promise { + const claim = "The current signed ledger checkpoint verifies offline against a published key"; + const id = "anchor-checkpoint"; + if (checkpoint === undefined || !isSignedAnchor(checkpoint.signed)) { + return { id, claim, status: "skip", detail: "no signed checkpoint published (anchor signing not configured, or the ledger is empty)" }; + } + const signed = checkpoint.signed; + if (keys.length === 0) { + // A checkpoint nothing can be checked against is a real gap, but it is a gap in what is OBTAINABLE, + // not proof of a bad signature -- so it is a skip whose text says exactly what is missing. + return { id, claim, status: "skip", detail: "a checkpoint is published, but no signing key is published to check it against" }; + } + + if (typeof checkpoint.signingInput === "string") { + const recomputed = anchorSigningInput(signed.payload); + if (recomputed !== checkpoint.signingInput) { + return { id, claim, status: "fail", detail: `the published signingInput is NOT the canonical serialization of the published payload (signed bytes differ from displayed bytes)` }; + } + } + + const key = keys.find((candidate) => candidate.keyId === signed.keyId); + if (key === undefined) { + return { id, claim, status: "fail", detail: `checkpoint at seq ${signed.payload.seq} names key "${signed.keyId}", which is not among the ${keys.length} published key(s)` }; + } + const verified = await verifyLedgerAnchorSignature(signed, key.publicKeySpki); + return verified + ? { id, claim, status: "pass", detail: `checkpoint at seq ${signed.payload.seq} verifies against published key "${key.keyId}"` } + : { id, claim, status: "fail", detail: `checkpoint at seq ${signed.payload.seq} does NOT verify against its own named key "${key.keyId}"` }; +} + +/** Structural guard for a checkable signed checkpoint. Selects rather than validates: a payload shape this + * build does not recognise is reported as "nothing to check", not as a forgery. */ +function isSignedAnchor(value: unknown): value is SignedLedgerAnchor { + if (typeof value !== "object" || value === null) return false; + const candidate = value as { payload?: unknown; keyId?: unknown; signature?: unknown }; + if (typeof candidate.keyId !== "string" || typeof candidate.signature !== "string") return false; + const payload = candidate.payload as { seq?: unknown; rowHash?: unknown } | undefined; + return typeof payload === "object" && payload !== null && typeof payload.seq === "number" && typeof payload.rowHash === "string"; +} + +/** + * CLAIM 4: the headline stats agree with the ledger-derived surfaces they are supposed to summarize. + * + * `/v1/public/stats` is the number a reader actually sees; the parity rollups are computed from + * `decision_records` directly. They are two paths to the same underlying count, so a disagreement means one + * of them is wrong -- which is precisely the class of bug no amount of internal testing catches, because + * each surface is individually self-consistent. + * + * Compared with a tolerance, not for equality: the two are read at different instants against a live ledger, + * so a verdict landing between the two fetches moves one and not the other. The tolerance is absolute and + * tiny -- large enough to absorb that race, far too small to absorb a real divergence. + */ +export function checkStatsParity(stats: { totals?: { handled?: unknown } | undefined } | undefined, parity: { verdicts?: unknown } | undefined, tolerance = 5): ClaimResult { + const claim = "Published headline stats agree with the ledger-derived parity rollups"; + const handled = typeof stats?.totals?.handled === "number" ? stats.totals.handled : null; + const verdicts = typeof parity?.verdicts === "number" ? parity.verdicts : null; + if (handled === null || verdicts === null) { + return { + id: "stats-parity", + claim, + status: "skip", + detail: handled === null ? "/v1/public/stats published no totals.handled count" : "review-parity rollups published no verdict count", + }; + } + // The parity rollup covers a WINDOW while totals are all-time, so parity can only ever be the smaller of + // the two. It exceeding the all-time count is the direction that indicates a real accounting error: the + // two are computed by different code over the same ledger, which is the divergence no amount of + // per-surface testing catches, because each surface is individually self-consistent. + // + // Compared with a tolerance, not for equality: the two are read at different instants against a live + // ledger, so a verdict landing between the two fetches moves one and not the other. The tolerance is + // absolute and tiny -- large enough to absorb that race, far too small to absorb a real divergence. + if (verdicts > handled + tolerance) { + return { id: "stats-parity", claim, status: "fail", detail: `parity rollups report ${verdicts} verdict(s) over their window, exceeding the all-time handled count of ${handled}` }; + } + return { id: "stats-parity", claim, status: "pass", detail: `all-time handled=${handled}, windowed parity verdicts=${verdicts} -- consistent` }; +} + +/** The process exit code for a set of results: non-zero if ANY claim failed. Skips do not fail the run -- + * a disabled surface is not a broken one -- but they are never silently counted as passes either. */ +export function exitCodeFor(results: readonly ClaimResult[]): number { + return results.some((result) => result.status === "fail") ? 1 : 0; +} + +/** One-line summary under the table, so a caller reading only the last line still gets the counts. */ +export function summarize(results: readonly ClaimResult[]): string { + const count = (status: ClaimStatus) => results.filter((result) => result.status === status).length; + return `${count("pass")} passed, ${count("fail")} failed, ${count("skip")} skipped`; +} diff --git a/packages/loopover-mcp/package.json b/packages/loopover-mcp/package.json index e218a673d8..fdf3cb4b93 100644 --- a/packages/loopover-mcp/package.json +++ b/packages/loopover-mcp/package.json @@ -26,7 +26,8 @@ "access": "public" }, "bin": { - "loopover-mcp": "dist/bin/loopover-mcp.js" + "loopover-mcp": "dist/bin/loopover-mcp.js", + "loopover-verify": "dist/bin/loopover-verify.js" }, "exports": { "./package.json": "./package.json", diff --git a/scripts/gen-contract-api-schemas.ts b/scripts/gen-contract-api-schemas.ts index 22562c5d39..c2490e2dcb 100644 --- a/scripts/gen-contract-api-schemas.ts +++ b/scripts/gen-contract-api-schemas.ts @@ -23,7 +23,21 @@ import { fileURLToPath } from "node:url"; const SOURCE = "src/openapi/schemas.ts"; const OUTPUT = "packages/loopover-contract/src/api-schemas.ts"; const OPENAPI_DOCUMENT = "apps/loopover-ui/public/openapi.json"; -const CLI_BIN = "packages/loopover-mcp/bin/loopover-mcp.ts"; +/** + * The CLI bin DIRECTORY, scanned -- not one named file (#9723). + * + * This was a single hardcoded path to `loopover-mcp.ts`, correct only while that was the package's only + * bin. A second one now ships (`loopover-verify.ts`), and a hardcoded filename means any future bin is + * invisible here and reads every response as `any` -- silently opting out of the validation this generator + * exists to enforce, which is the exact failure mode `cliApiPaths`' own doc warns about ("a new call site + * must not be able to opt out of validation by simply not being added here"). + * + * Scanning the directory closes that by construction. It is a LATENT fix, not a live one: today's output is + * byte-identical, because `loopover-verify.ts` deliberately does not use the shared path-first `apiGet` + * convention -- a public verifier has to keep checking a deployment running an OLDER build than itself, so + * validating those payloads against this build's schemas would turn version skew into a false failure. + */ +const CLI_BIN_DIR = "packages/loopover-mcp/bin"; /** * Every literal `/v1/...` path the CLI hands to its api helpers. Scanned, not listed: a hand-kept endpoint @@ -455,7 +469,15 @@ export function generate(deps: { readFile?: (path: string) => string; listDir?: const readFile = deps.readFile ?? ((path: string) => readFileSync(path, "utf8")); readModule = readFile; listModules = deps.listDir ?? ((dir: string) => readdirSync(dir)); - return renderApiSchemas(readFile(SOURCE), readFile(OPENAPI_DOCUMENT), readFile(CLI_BIN)); + // Every bin's sources concatenated: `cliApiPaths` collects into a Set, so a path two bins both call is + // emitted once, and the join cannot create a spurious match across the boundary (the regex is anchored on + // an `api*(` call and cannot span files). + const binSources = listModules(CLI_BIN_DIR) + .filter((entry) => entry.endsWith(".ts")) + .sort() + .map((entry) => readFile(`${CLI_BIN_DIR}/${entry}`)) + .join("\n"); + return renderApiSchemas(readFile(SOURCE), readFile(OPENAPI_DOCUMENT), binSources); } function main(): void { diff --git a/src/review/decision-record.ts b/src/review/decision-record.ts index 4208351316..4a2a8c6fa6 100644 --- a/src/review/decision-record.ts +++ b/src/review/decision-record.ts @@ -47,44 +47,18 @@ // decision's plan from a heuristic close to a hold — see that field's own doc comment. import { deliveryIdOrigin, type DeliveryIdOrigin } from "../queue/delivery-id"; import { errorMessage, nowIso } from "../utils/json"; +import { canonicalJson, contentDigest, sha256Hex } from "@loopover/contract/digest"; import { retentionCutoffIsoForTable } from "../db/retention"; /** Bump when the record's FIELD SET changes meaning — consumers compare records only within a version. */ export const DECISION_RECORD_SCHEMA_VERSION = "6"; // v6 (#9743): + findingsCount, so "findings raised per PR" is derivable from the LEDGER rather than from the AI review cache (which is keyed for reuse, not anchored, and so cannot back a reproducibility claim); v5 (#8834): + aiAgreement (inter-run agreement folded with the verbalized confidence); v4 (#9124/#9135): configDigest digests the resolved policy (+ settingsDigest split out), promptDigest digests the actual sent prompt, modelId -> modelIds (real identities), ciState populated, + divertedByHoldout; v3 (#8962): + salvageability {score, factors}; v2 (#8834): + aiConfidence, model/prompt commitments -/** - * Canonical JSON: recursively key-sorted, no insignificant whitespace — the ONE serialization every digest - * in this system is computed over. Identical logical inputs must always hash identically, so object key - * order (an artifact of construction, not meaning) can never influence a digest. Arrays keep their order - * (order IS meaning there). undefined object members are dropped (JSON has no undefined); undefined inside - * arrays follows JSON.stringify's own null coercion. PURE. - */ -export function canonicalJson(value: unknown): string { - if (value === null || typeof value === "number" || typeof value === "boolean") return JSON.stringify(value); - if (typeof value === "string") return JSON.stringify(value); - if (typeof value === "undefined") return "null"; - if (Array.isArray(value)) return `[${value.map((entry) => canonicalJson(entry)).join(",")}]`; - if (typeof value === "object") { - const record = value as Record; - const keys = Object.keys(record) - .filter((key) => record[key] !== undefined) - .sort(); - return `{${keys.map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(",")}}`; - } - // Functions/symbols/bigints have no JSON meaning; refusing loudly beats a silent wrong digest. - throw new Error(`canonicalJson: unsupported value type "${typeof value}"`); -} - -/** SHA-256 hex over UTF-8 text via Web Crypto (available in the Workers runtime AND Node ≥20). */ -export async function sha256Hex(text: string): Promise { - const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text)); - return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join(""); -} - -/** Digest of any JSON-shaped value via the canonical serialization above. */ -export async function contentDigest(value: unknown): Promise { - return sha256Hex(canonicalJson(value)); -} +// The canonical serialization and the digest pair every commitment here is computed under. Defined in +// @loopover/contract (digest.ts) rather than in this file, because the public verifier CLI (#9723) has to +// recompute these exact digests from a published package with no repo checkout -- and re-exported from +// here so every existing `from "./decision-record"` call site is unchanged and there is one definition, +// not two that can drift. See that module's header for the serialization rules themselves. +export { canonicalJson, sha256Hex, contentDigest }; /** The published, public-safe decision record. Counts/digests/enums only — no diffs, no private config * contents (their DIGEST is the commitment), no author identity beyond what the PR page already shows. */ diff --git a/src/review/ledger-anchor.ts b/src/review/ledger-anchor.ts index 54762d2ffc..25de480c94 100644 --- a/src/review/ledger-anchor.ts +++ b/src/review/ledger-anchor.ts @@ -16,57 +16,36 @@ // ECDSA P-256 / SHA-256 via WebCrypto: native to the Workers runtime, so this adds no dependency, and it is // exactly what Rekor's `hashedrekord` accepts as a self-managed verifier key (#9272) -- the same keypair // serves both the local signature and the transparency-log submission. +// +// The VERIFIER half of this module -- the payload shape, its signing input, key-id derivation and signature +// verification -- moved to @loopover/contract (anchor-verify.ts) for #9723, so the public verifier CLI checks +// anchors with this exact code instead of a reimplementation that could silently diverge. Signing stays here: +// it needs the operator's private key, which nothing outside the Worker may hold. Re-exported below so every +// existing `from "./ledger-anchor"` call site is unchanged. +import { + anchorSigningInput, + base64ToBytes, + computeAnchorKeyId, + LEDGER_ANCHOR_LEDGER_ID, + LEDGER_ANCHOR_PAYLOAD_VERSION, + verifyLedgerAnchorSignature, + type AnchorPublicKey, + type LedgerAnchorPayload, + type SignedLedgerAnchor, +} from "@loopover/contract/anchor-verify"; import { canonicalJson, sha256Hex } from "./decision-record"; -/** Bump when the payload's FIELD SET changes meaning. A verifier reads this FIRST and refuses shapes it does - * not understand, rather than silently misreading a future field set as the current one. */ -export const LEDGER_ANCHOR_PAYLOAD_VERSION = 1 as const; - -/** Identifies WHICH chain an anchor commits to. A fixed string today (one ledger), but present from v1 so a - * second anchored chain can never be confused for this one by a verifier holding both. */ -export const LEDGER_ANCHOR_LEDGER_ID = "loopover.decision_ledger"; - -/** The exact bytes an anchor commits to. `totalCount` is included alongside `seq` deliberately: a chain - * truncated and re-chained to the same length would still have to match BOTH, and the pair is what lets a - * verifier notice "the tip moved backwards" without fetching every row. */ -export type LedgerAnchorPayload = { - v: typeof LEDGER_ANCHOR_PAYLOAD_VERSION; - ledger: typeof LEDGER_ANCHOR_LEDGER_ID; - seq: number; - rowHash: string; - totalCount: number; - at: string; -}; - -/** A payload plus the operator signature over its canonical serialization, and the id of the key that signed - * it. `keyId` is REQUIRED: without it a verifier holding a rotation history cannot tell which published key - * a given anchor should be checked against, and would have to try them all -- turning a failed verification - * (a real signal) into an ambiguous one. */ -export type SignedLedgerAnchor = { - payload: LedgerAnchorPayload; - keyId: string; - /** base64 P-1363 (r||s) ECDSA signature over `canonicalJson(payload)`, as WebCrypto produces it. */ - signature: string; -}; - -/** One published anchor-signing public key and the window it was valid for. `notAfter: null` = still current. - * Rotation is why this is a LIST: an anchor signed in 2026 must stay verifiable after a 2027 rotation, so - * retired keys are published forever rather than replaced. */ -export type AnchorPublicKey = { - keyId: string; - /** base64 SPKI DER -- the same encoding Rekor's `verifier.publicKey.rawBytes` takes (#9272). */ - publicKeySpki: string; - notBefore: string; - notAfter: string | null; +export { + anchorSigningInput, + computeAnchorKeyId, + LEDGER_ANCHOR_LEDGER_ID, + LEDGER_ANCHOR_PAYLOAD_VERSION, + verifyLedgerAnchorSignature, + type AnchorPublicKey, + type LedgerAnchorPayload, + type SignedLedgerAnchor, }; -function base64ToBytes(base64: string): Uint8Array { - const binary = atob(base64); - const bytes = new Uint8Array(binary.length); - for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index); - return bytes; -} - function bytesToBase64(bytes: Uint8Array): string { let binary = ""; for (const byte of bytes) binary += String.fromCharCode(byte); @@ -84,16 +63,6 @@ function pemToBytes(pem: string): Uint8Array { return base64ToBytes(base64); } -/** - * Derive a key's id FROM the key itself -- sha256(SPKI DER), first 16 hex chars. Deliberately not an operator- - * chosen label: a derived id cannot drift from the key it names, cannot be reused for a different key across - * a rotation, and lets a verifier confirm that the key they fetched is the key an anchor claims was used. - */ -export async function computeAnchorKeyId(publicKeySpkiBase64: string): Promise { - const digest = await crypto.subtle.digest("SHA-256", base64ToBytes(publicKeySpkiBase64) as Uint8Array); - return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("").slice(0, 16); -} - /** * Build the versioned, self-describing payload for a tip. PURE and synchronous -- the caller supplies `at` * (or accepts the default) so a scheduled job's payload is reproducible in a test. @@ -109,12 +78,6 @@ export function buildLedgerAnchorPayload(tip: { seq: number; rowHash: string; to }; } -/** The exact bytes signed and verified -- one definition both sides share, so a serialization change can - * never silently break verification while leaving signing "working". */ -export function anchorSigningInput(payload: LedgerAnchorPayload): string { - return canonicalJson(payload); -} - /** * Sign an anchor payload with the operator's ECDSA P-256 private key (PKCS8 PEM, held as a Worker secret and * never in the repo). `keyId` is supplied by the caller -- it is `computeAnchorKeyId` of the PUBLISHED public @@ -142,32 +105,6 @@ export async function signLedgerAnchorPayload( return { payload, keyId, signature: bytesToBase64(new Uint8Array(signature)) }; } -/** - * Verify a signed anchor against a published public key. Returns a boolean rather than throwing for an - * ordinarily-invalid input (wrong key, tampered payload, malformed signature) -- those are all "not verified", - * a fact about the anchor, not a caller bug. This is the function a third party's own verifier reimplements; - * it is exported so our tests exercise the SAME path an outsider would, never a privileged shortcut. - */ -export async function verifyLedgerAnchorSignature(signed: SignedLedgerAnchor, publicKeySpkiBase64: string): Promise { - try { - const key = await crypto.subtle.importKey( - "spki", - base64ToBytes(publicKeySpkiBase64) as Uint8Array, - { name: "ECDSA", namedCurve: "P-256" }, - true, - ["verify"], - ); - return await crypto.subtle.verify( - { name: "ECDSA", hash: "SHA-256" }, - key, - base64ToBytes(signed.signature) as Uint8Array, - new TextEncoder().encode(anchorSigningInput(signed.payload)), - ); - } catch { - return false; - } -} - /** * Parse the published key list from its configured JSON. Never throws: malformed config yields an EMPTY list, * which fails closed at the route (no keys published => nothing claims to be verifiable) rather than a 500 on diff --git a/test/unit/loopover-verify-cli.test.ts b/test/unit/loopover-verify-cli.test.ts new file mode 100644 index 0000000000..26dcf5218d --- /dev/null +++ b/test/unit/loopover-verify-cli.test.ts @@ -0,0 +1,175 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { contentDigest } from "@loopover/contract/digest"; + +// The bin's own source, imported in-process (the format-table.test.ts pattern). It is import-safe by +// construction -- it self-executes only when it is argv[1] -- so this drives the real entry point: argv +// parsing, fetch orchestration, rendering and exit code, without spawning a subprocess whose execution +// would be invisible to instrumentation. +async function loadCli() { + return import("../../packages/loopover-mcp/bin/loopover-verify") as Promise<{ + parseBaseUrl: (args: readonly string[], fallback?: string) => { ok: true; baseUrl: string } | { ok: false; error: string }; + runVerify: (args: readonly string[], baseUrlOverride?: string) => Promise; + }>; +} + +/** Capture stdout for one run rather than letting the CLI print into the test output. */ +function captureStdout() { + const chunks: string[] = []; + const spy = vi.spyOn(process.stdout, "write").mockImplementation((chunk: unknown) => { + chunks.push(String(chunk)); + return true; + }); + return { chunks, restore: () => spy.mockRestore(), text: () => chunks.join("") }; +} + +/** Serve a fixed map of path -> body; anything unmapped 404s, which is what a real deployment does for a + * surface it has disabled. */ +function stubFetch(routes: Record) { + vi.stubGlobal("fetch", async (input: string | URL) => { + const url = new URL(String(input)); + const key = `${url.pathname}${url.search}`; + const body = routes[key] ?? routes[url.pathname]; + if (body === undefined) return new Response(JSON.stringify({ error: "not_found" }), { status: 404 }); + return new Response(JSON.stringify(body), { status: 200, headers: { "content-type": "application/json" } }); + }); +} + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe("parseBaseUrl", () => { + it("defaults, strips a trailing slash, and accepts an explicit http(s) base", async () => { + const { parseBaseUrl } = await loadCli(); + expect(parseBaseUrl([])).toEqual({ ok: true, baseUrl: "https://api.loopover.ai" }); + expect(parseBaseUrl(["--base-url", "https://example.test/"])).toEqual({ ok: true, baseUrl: "https://example.test" }); + expect(parseBaseUrl(["--base-url", "http://localhost:8787"])).toEqual({ ok: true, baseUrl: "http://localhost:8787" }); + }); + + it("rejects a missing value, a flag mistaken for a value, a non-URL, and a non-http scheme", async () => { + const { parseBaseUrl } = await loadCli(); + expect(parseBaseUrl(["--base-url"]).ok).toBe(false); + expect(parseBaseUrl(["--base-url", "--json"]).ok).toBe(false); + expect(parseBaseUrl(["--base-url", "not a url"]).ok).toBe(false); + const scheme = parseBaseUrl(["--base-url", "file:///etc/passwd"]); + expect(scheme.ok).toBe(false); + expect(scheme.ok === false && scheme.error).toContain("must be http(s)"); + }); +}); + +describe("runVerify", () => { + it("prints usage and exits 0 for --help without touching the network", async () => { + const { runVerify } = await loadCli(); + const fetchSpy = vi.fn(); + vi.stubGlobal("fetch", fetchSpy); + const out = captureStdout(); + const code = await runVerify(["--help"]); + out.restore(); + expect(code).toBe(0); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(out.text()).toContain("no credentials"); + }); + + it("exits 1 and marks the row FAIL when a published digest does not recompute", async () => { + const { runVerify } = await loadCli(); + stubFetch({ + "/v1/public/eval-scores": { records: [{ subject: { id: "loopover" }, workUnit: { ruleId: "rule_a" }, score: { decided: 3 }, recordDigest: "0".repeat(64) }] }, + }); + const out = captureStdout(); + const code = await runVerify([], "https://example.test"); + out.restore(); + expect(code).toBe(1); + expect(out.text()).toContain("FAIL"); + }); + + it("exits 0 and reports PASS on a wholly consistent deployment", async () => { + const { runVerify } = await loadCli(); + const preimage = { subject: { id: "loopover" }, workUnit: { ruleId: "rule_a" }, score: { decided: 3 }, commitments: {} }; + stubFetch({ + "/v1/public/eval-scores": { records: [{ ...preimage, recordDigest: await contentDigest(preimage) }] }, + "/v1/public/stats": { totals: { handled: 500 }, reviewParity: { verdicts: 12 } }, + }); + const out = captureStdout(); + const code = await runVerify([], "https://example.test"); + out.restore(); + expect(code).toBe(0); + const text = out.text(); + expect(text).toContain("PASS"); + expect(text).not.toContain("FAIL"); + }); + + it("degrades every unreachable surface to SKIP rather than failing or throwing", async () => { + // A deployment with public stats switched off is not a deployment failing verification, and one dead + // endpoint must not abort the claims that were still checkable. + const { runVerify } = await loadCli(); + stubFetch({}); + const out = captureStdout(); + const code = await runVerify([], "https://example.test"); + out.restore(); + expect(code).toBe(0); + expect(out.text()).toContain("0 passed, 0 failed, 4 skipped"); + }); + + it("survives a transport-level throw, reporting it instead of crashing", async () => { + const { runVerify } = await loadCli(); + vi.stubGlobal("fetch", async () => { + throw new Error("ECONNREFUSED"); + }); + const out = captureStdout(); + const code = await runVerify([], "https://example.test"); + out.restore(); + expect(code).toBe(0); + expect(out.text()).toContain("SKIP"); + }); + + it("emits machine-readable results under --json", async () => { + const { runVerify } = await loadCli(); + stubFetch({ "/v1/public/stats": { totals: { handled: 5 }, reviewParity: { verdicts: 1 } } }); + const out = captureStdout(); + const code = await runVerify(["--json"], "https://example.test"); + out.restore(); + expect(code).toBe(0); + const parsed = JSON.parse(out.text()) as { baseUrl: string; results: Array<{ id: string; status: string }>; summary: string }; + expect(parsed.baseUrl).toBe("https://example.test"); + expect(parsed.results.map((result) => result.id)).toEqual(["record-digests", "corpus-commitments", "anchor-checkpoint", "stats-parity"]); + expect(parsed.results.find((result) => result.id === "stats-parity")?.status).toBe("pass"); + }); + + it("sends no Authorization header on any request", async () => { + // The load-bearing property of the whole tool: a verifier that needs our credentials verifies nothing. + const { runVerify } = await loadCli(); + const seen: Array> = []; + vi.stubGlobal("fetch", async (_input: string | URL, init?: RequestInit) => { + seen.push(Object.fromEntries(new Headers(init?.headers).entries())); + return new Response(JSON.stringify({}), { status: 200, headers: { "content-type": "application/json" } }); + }); + const out = captureStdout(); + await runVerify([], "https://example.test"); + out.restore(); + expect(seen.length).toBeGreaterThan(0); + for (const headers of seen) expect(Object.keys(headers)).not.toContain("authorization"); + }); + + it("fetches one corpus per committed rule, deduplicating repeats", async () => { + const { runVerify } = await loadCli(); + const record = async (ruleId: string) => { + const preimage = { workUnit: { ruleId }, score: { decided: 1 }, commitments: { corpusChecksum: "abc" } }; + return { ...preimage, recordDigest: await contentDigest(preimage) }; + }; + const requested: string[] = []; + vi.stubGlobal("fetch", async (input: string | URL) => { + const url = new URL(String(input)); + if (url.pathname === "/v1/public/eval-corpus") requested.push(url.searchParams.get("rule_id") ?? ""); + if (url.pathname === "/v1/public/eval-scores") { + return new Response(JSON.stringify({ records: [await record("rule_a"), await record("rule_a"), await record("rule_b")] }), { status: 200 }); + } + return new Response(JSON.stringify({ error: "not_found" }), { status: 404 }); + }); + const out = captureStdout(); + await runVerify([], "https://example.test"); + out.restore(); + expect(requested.sort()).toEqual(["rule_a", "rule_b"]); + }); +}); diff --git a/test/unit/verify-public-claims.test.ts b/test/unit/verify-public-claims.test.ts new file mode 100644 index 0000000000..4525e86ebd --- /dev/null +++ b/test/unit/verify-public-claims.test.ts @@ -0,0 +1,237 @@ +import { describe, expect, it } from "vitest"; + +import { canonicalJson, contentDigest, sha256Hex } from "@loopover/contract/digest"; + +// The checks ship in the MCP package's lib/, same as format-table.test.ts: imported dynamically from the +// package path so the module under test is the exact one the published bin loads. +async function loadClaims() { + return import("../../packages/loopover-mcp/lib/verify-public-claims"); +} + +/** Build a record whose `recordDigest` is genuinely correct, the way the Worker builds one. */ +async function signedRecord(overrides: Record = {}) { + const preimage = { + schemaVersion: "1", + subject: { kind: "agent", id: "loopover" }, + workUnit: { kind: "outcome_confirmed_precision", ruleId: "rule_a" }, + score: { decided: 10, confirmed: 9 }, + commitments: { corpusChecksum: "deadbeef" }, + ...overrides, + }; + return { ...preimage, recordDigest: await contentDigest(preimage) }; +} + +describe("checkRecordDigests", () => { + it("passes when every published digest recomputes from its own record", async () => { + const { checkRecordDigests } = await loadClaims(); + const result = await checkRecordDigests([await signedRecord(), await signedRecord({ workUnit: { kind: "x", ruleId: "rule_b" } })]); + expect(result.status).toBe("pass"); + expect(result.detail).toContain("2 record(s)"); + }); + + it("is insensitive to the key ORDER the endpoint happened to serialize in", async () => { + // The entire reason a digest is checkable from parsed JSON: canonicalJson sorts keys, so a payload + // whose fields arrive in a different order must still verify. + const { checkRecordDigests } = await loadClaims(); + const record = await signedRecord(); + const reordered = Object.fromEntries(Object.entries(record).reverse()); + expect(Object.keys(reordered)).not.toEqual(Object.keys(record)); + expect((await checkRecordDigests([reordered as never])).status).toBe("pass"); + }); + + it("FAILS a record whose contents were altered after signing", async () => { + const { checkRecordDigests } = await loadClaims(); + const record = await signedRecord(); + const tampered = { ...record, score: { decided: 10_000, confirmed: 9_999 } }; + const result = await checkRecordDigests([tampered]); + expect(result.status).toBe("fail"); + expect(result.detail).toContain("recomputed"); + }); + + it("FAILS a record publishing no digest at all, rather than skipping it", async () => { + const { checkRecordDigests } = await loadClaims(); + const { recordDigest: _dropped, ...unsigned } = await signedRecord(); + expect((await checkRecordDigests([unsigned])).status).toBe("fail"); + // The empty-string arm is the same class of miss and must not read as "present". + expect((await checkRecordDigests([{ ...unsigned, recordDigest: "" }])).status).toBe("fail"); + }); + + it("reports an unserializable record instead of throwing and killing later claims", async () => { + const { checkRecordDigests } = await loadClaims(); + const result = await checkRecordDigests([{ recordDigest: "abc", subject: { id: "s" }, workUnit: { ruleId: undefined }, bad: () => 1 } as never]); + expect(result.status).toBe("fail"); + expect(result.detail).toContain("not serializable"); + }); + + it("skips, not passes, when nothing is published", async () => { + const { checkRecordDigests } = await loadClaims(); + expect((await checkRecordDigests([])).status).toBe("skip"); + }); +}); + +describe("checkCorpusCommitments", () => { + const corpusCases = [{ ruleId: "rule_a", outcome: "o", label: "confirmed", firedAt: "2026-01-01T00:00:00.000Z", decidedAt: "2026-01-02T00:00:00.000Z" }]; + + it("passes when a commitment rehashes to the downloadable corpus", async () => { + const { checkCorpusCommitments } = await loadClaims(); + const checksum = await sha256Hex(canonicalJson(corpusCases)); + const record = await signedRecord({ commitments: { corpusChecksum: checksum } }); + const result = await checkCorpusCommitments([record], new Map([["rule_a", { cases: corpusCases, checksum }]])); + expect(result.status).toBe("pass"); + expect(result.detail).toContain("1 case(s) rehashed exactly"); + }); + + it("recomputes from the CASES, not from the corpus's own checksum field", async () => { + // A deployment serving cases that do not hash to the checksum it advertises must be caught; trusting + // the advertised field would make the whole claim vacuous. + const { checkCorpusCommitments } = await loadClaims(); + const committed = await sha256Hex(canonicalJson(corpusCases)); + const record = await signedRecord({ commitments: { corpusChecksum: committed } }); + const tamperedCases = [{ ...corpusCases[0]!, label: "reversed" }]; + const result = await checkCorpusCommitments([record], new Map([["rule_a", { cases: tamperedCases, checksum: committed }]])); + expect(result.status).toBe("fail"); + expect(result.detail).toContain("hashes to"); + }); + + it("FAILS a record that commits to the EMPTY corpus while claiming decided work", async () => { + const { checkCorpusCommitments, emptyCorpusChecksum } = await loadClaims(); + const record = await signedRecord({ score: { decided: 42 }, commitments: { corpusChecksum: await emptyCorpusChecksum() } }); + const result = await checkCorpusCommitments([record], new Map()); + expect(result.status).toBe("fail"); + expect(result.detail).toContain("EMPTY corpus"); + expect(result.detail).toContain("decided=42"); + }); + + it("allows the empty-corpus commitment when nothing was decided", async () => { + // decided=0 with an empty corpus is coherent, not a contradiction -- the check must not fire on it. + const { checkCorpusCommitments, emptyCorpusChecksum } = await loadClaims(); + const record = await signedRecord({ score: { decided: 0 }, commitments: { corpusChecksum: await emptyCorpusChecksum() } }); + expect((await checkCorpusCommitments([record], new Map())).status).toBe("skip"); + }); + + it("SKIPS rather than passes when no committed corpus is downloadable", async () => { + // The regression this pins: reporting `pass` with a detail line that says "unverified" is the + // could-not-check/therefore-fine collapse the module header rules out. + const { checkCorpusCommitments } = await loadClaims(); + const record = await signedRecord({ commitments: { corpusChecksum: "0".repeat(64) } }); + const result = await checkCorpusCommitments([record], new Map()); + expect(result.status).toBe("skip"); + expect(result.detail).toContain("nothing could be rehashed"); + }); + + it("skips when no record published a commitment at all", async () => { + const { checkCorpusCommitments } = await loadClaims(); + const record = await signedRecord({ commitments: {} }); + expect((await checkCorpusCommitments([record], new Map())).status).toBe("skip"); + }); +}); + +describe("checkAnchorCheckpoint", () => { + /** A real ECDSA P-256 keypair + signature, produced exactly the way the Worker signs. */ + async function realCheckpoint() { + // Cast: generateKey's union return and exportKey's ArrayBuffer|JsonWebKey union are both resolved by + // the arguments used here (an asymmetric algorithm, and the "spki" format), which the lib types do not + // narrow on. + const pair = (await crypto.subtle.generateKey({ name: "ECDSA", namedCurve: "P-256" }, true, ["sign", "verify"])) as CryptoKeyPair; + const spki = new Uint8Array((await crypto.subtle.exportKey("spki", pair.publicKey)) as ArrayBuffer); + const publicKeySpki = btoa(String.fromCharCode(...spki)); + const { computeAnchorKeyId, anchorSigningInput } = await import("@loopover/contract/anchor-verify"); + const keyId = await computeAnchorKeyId(publicKeySpki); + const payload = { v: 1 as const, ledger: "loopover.decision_ledger" as const, seq: 7, rowHash: "abc123", totalCount: 7, at: "2026-07-30T00:00:00.000Z" }; + const signingInput = anchorSigningInput(payload); + const raw = new Uint8Array(await crypto.subtle.sign({ name: "ECDSA", hash: "SHA-256" }, pair.privateKey, new TextEncoder().encode(signingInput))); + const signed = { payload, keyId, signature: btoa(String.fromCharCode(...raw)) }; + return { signed, signingInput, keys: [{ keyId, publicKeySpki, notBefore: "2026-01-01T00:00:00.000Z", notAfter: null }] }; + } + + it("passes on a genuinely signed checkpoint verified against the published key", async () => { + const { checkAnchorCheckpoint } = await loadClaims(); + const { signed, signingInput, keys } = await realCheckpoint(); + const result = await checkAnchorCheckpoint({ signed, signingInput }, keys); + expect(result.status).toBe("pass"); + expect(result.detail).toContain("seq 7"); + }); + + it("FAILS when the payload was altered after signing", async () => { + const { checkAnchorCheckpoint } = await loadClaims(); + const { signed, keys } = await realCheckpoint(); + const tampered = { ...signed, payload: { ...signed.payload, seq: 9_999 } }; + expect((await checkAnchorCheckpoint({ signed: tampered }, keys)).status).toBe("fail"); + }); + + it("FAILS when the displayed signingInput is not the canonical serialization of the displayed payload", async () => { + // The check a reader cannot perform by eye: showing one payload while signing different bytes. + const { checkAnchorCheckpoint } = await loadClaims(); + const { signed, keys } = await realCheckpoint(); + const result = await checkAnchorCheckpoint({ signed, signingInput: '{"v":1,"seq":"something else"}' }, keys); + expect(result.status).toBe("fail"); + expect(result.detail).toContain("signed bytes differ from displayed bytes"); + }); + + it("FAILS when the checkpoint names a key that is not published", async () => { + const { checkAnchorCheckpoint } = await loadClaims(); + const { signed, signingInput, keys } = await realCheckpoint(); + const result = await checkAnchorCheckpoint({ signed: { ...signed, keyId: "not-a-published-key" }, signingInput }, keys); + expect(result.status).toBe("fail"); + expect(result.detail).toContain("not among"); + }); + + it("skips when no checkpoint is published, and when no key is published to check one against", async () => { + const { checkAnchorCheckpoint } = await loadClaims(); + const { signed } = await realCheckpoint(); + expect((await checkAnchorCheckpoint(undefined, [])).status).toBe("skip"); + expect((await checkAnchorCheckpoint({ signed: { nonsense: true } }, [])).status).toBe("skip"); + const noKeys = await checkAnchorCheckpoint({ signed }, []); + expect(noKeys.status).toBe("skip"); + expect(noKeys.detail).toContain("no signing key is published"); + }); +}); + +describe("checkStatsParity", () => { + it("passes when the windowed rollup fits inside the all-time total", async () => { + const { checkStatsParity } = await loadClaims(); + const result = checkStatsParity({ totals: { handled: 12_882 } }, { verdicts: 40 }); + expect(result.status).toBe("pass"); + expect(result.detail).toContain("12882"); + }); + + it("FAILS when the windowed rollup exceeds the all-time total beyond the race tolerance", async () => { + const { checkStatsParity } = await loadClaims(); + expect(checkStatsParity({ totals: { handled: 10 } }, { verdicts: 500 }).status).toBe("fail"); + }); + + it("absorbs a small in-flight difference rather than crying wolf", async () => { + // The two surfaces are read at different instants against a live ledger; a verdict landing between the + // fetches must not be reported as an accounting error. + const { checkStatsParity } = await loadClaims(); + expect(checkStatsParity({ totals: { handled: 100 } }, { verdicts: 103 }).status).toBe("pass"); + }); + + it("skips each side independently when its number is absent", async () => { + const { checkStatsParity } = await loadClaims(); + expect(checkStatsParity({ totals: {} }, { verdicts: 1 }).detail).toContain("totals.handled"); + expect(checkStatsParity({ totals: { handled: 1 } }, {}).detail).toContain("verdict count"); + expect(checkStatsParity(undefined, undefined).status).toBe("skip"); + }); +}); + +describe("exit code and summary", () => { + it("exits non-zero only when a claim FAILED -- a skip is not a failure", async () => { + const { exitCodeFor, summarize } = await loadClaims(); + const pass = { id: "a", claim: "a", status: "pass" as const, detail: "" }; + const skip = { id: "b", claim: "b", status: "skip" as const, detail: "" }; + const fail = { id: "c", claim: "c", status: "fail" as const, detail: "" }; + expect(exitCodeFor([pass, skip])).toBe(0); + expect(exitCodeFor([pass, skip, fail])).toBe(1); + expect(summarize([pass, skip, fail])).toBe("1 passed, 1 failed, 1 skipped"); + }); +}); + +describe("describeRecord", () => { + it("falls back through ruleId, subject id, then position", async () => { + const { describeRecord } = await loadClaims(); + expect(describeRecord({ workUnit: { ruleId: "rule_a" }, subject: { id: "s" } }, 0)).toBe("rule_a"); + expect(describeRecord({ subject: { id: "s" } }, 0)).toBe("s"); + expect(describeRecord({}, 3)).toBe("record[3]"); + }); +}); From ac3e597d11abed9c270caf452b1f58546ac50756 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 30 Jul 2026 11:45:28 -0700 Subject: [PATCH 2/2] docs(fairness): published methodology page Closes #9725. /fairness publishes precision, accuracy and parity figures whose method lived in code and issue history, so a reader could see the number but not the rule behind it. This is that page: the three disjoint data sources and which figures come from each, the outcome-confirmation rules, every window, precision and coverage definitions, sample floors, and a limitations section. Linked from /fairness (both the rule table and the parity block) and from the verify-this-review walkthrough, and kept next to the scoring code it describes so a definition change and a doc change land in the same review. Written against the code rather than from intent, which surfaced things worth stating plainly: the windows are flat with no time-decay anywhere; only the review-parity block is computed from the anchored ledger; coverage is computed but not displayed here; and the only interval on the page is the headline accuracy CI, in the single-instance branch -- per-rule precision, the per-repository column, both weekly trends and every parity figure are bare point estimates. Also fixes a mislabel the write-up exposed: the headline read "lifetime" while its accuracy pairing has always been bounded by audit-log retention, deliberately so (a lifetime denominator drifts the ratio toward 100%). totals now publish accuracyWindowDays, derived from the retention policy rather than restated as a literal, and the page renders the real window. --- .../content/docs/fairness-methodology.mdx | 187 ++++++++++++++++++ .../content/docs/verify-this-review.mdx | 5 + apps/loopover-ui/public/openapi.json | 5 + .../src/components/site/command-palette.tsx | 1 + .../src/components/site/docs-nav.tsx | 1 + .../site/fairness-report-page.test.tsx | 5 +- .../components/site/fairness-report-page.tsx | 30 ++- .../site/proof-of-power-stats.test.tsx | 1 + apps/loopover-ui/src/routes/docs.index.tsx | 1 + packages/loopover-contract/src/public-api.ts | 4 + src/db/retention.ts | 8 + src/review/public-stats.ts | 8 +- test/unit/public-stats.test.ts | 5 + 13 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 apps/loopover-ui/content/docs/fairness-methodology.mdx diff --git a/apps/loopover-ui/content/docs/fairness-methodology.mdx b/apps/loopover-ui/content/docs/fairness-methodology.mdx new file mode 100644 index 0000000000..3b87a98f3c --- /dev/null +++ b/apps/loopover-ui/content/docs/fairness-methodology.mdx @@ -0,0 +1,187 @@ +--- +title: Fairness methodology +description: How every number on the fairness report is computed — data sources, outcome-confirmation rules, windows, sample floors, and the limits of what each figure can support. +eyebrow: Core concepts +--- + +## Why this page exists + +The [fairness report](/fairness) publishes accuracy, per-rule precision, and review-parity figures. +Each is a claim, and a claim you cannot check is just an assertion. [Verify this +review](/docs/verify-this-review) shows you how to re-run the corpus yourself; this page tells you +what the numbers *mean* — which rows are counted, which are excluded, over what window, and where +each figure stops being able to support a conclusion. + +Nothing here is a summary of intent. Every rule below is the rule the code implements. + + + **Want the machine-checkable version?** `npx -p @loopover/mcp loopover-verify` recomputes the + published commitments from the public endpoints and prints a PASS/FAIL table. It needs no + credentials. See [Verify this review](/docs/verify-this-review). + + +## The three data sources are not interchangeable + +This is the single most important thing to understand before reading any number: the fairness report +draws on **three disjoint sources**, and figures from different sources cannot be compared or combined. + +| Source | What it is | Which figures come from it | +| --- | --- | --- | +| **Decision ledger** (`decision_records`) | The hash-chained, anchored record of every verdict | The **review-parity** block only: re-evaluation rate, re-evaluations by reason, per-author-class parity | +| **Signal history** (human-override audit events) | Human verdicts recorded against individual rule firings | The **measured accuracy per rule** table, and the reproducibility freeze point | +| **Outcome history** (auto-actions + reversal events) | What the gate did, and what humans later did about it | **Decision accuracy**, the per-repository table, and both weekly trends | + +A consequence worth stating plainly: **only the review-parity block is computed from the anchored +ledger.** Those figures are the ones an outsider holding the ledger export can recompute exactly. The +rule-precision and accuracy figures are computed from audit history, which is subject to a 90-day +retention bound and is not itself anchored. + +## Outcome confirmation: what makes a case confirmed or reversed + +### Per-rule precision + +A rule's precision counts **human verdicts recorded against that rule's firings**. A verdict is +written as `reversed` when a human undid what the gate did, and `confirmed` when a human let it stand. + +**Reversal is recorded when:** + +- A contributor **reopens** a pull request the gate had closed. +- A merged PR's body says it **reverts** an earlier PR — and only if that earlier PR's merge was + actually recorded. An uncorroborated "Reverts #N" records nothing. +- A merge **supersedes** a PR the gate had closed. +- The repository owner reopens a closed PR **and merges it within six hours**. A bare owner reopen is + deliberately *not* a reversal on its own — owners reopen for many reasons — so this is a case where + a genuine "the gate was wrong" event can go uncounted. + +**Confirmation is recorded when:** + +- The repository **owner** closes a PR without merging, agreeing with the gate's close. A + contributor closing their own PR records nothing. +- For the AI-judgment rules specifically, an explicit hold confirmation is recorded. +- For every other rule, a terminal disposition with no reversal records an **implicit** confirmation. + + + **Implicit and explicit confirmations are pooled.** Implicit confirmations are tagged in the + underlying data so a consumer *could* weight them separately — but the published precision does + not. A rule whose confirmations are largely implicit ("nobody undid it") is presented identically + to one whose confirmations are explicit ("a human affirmed it"). Those are different strengths of + evidence, and the published number does not distinguish them. + + +### Excluded from the precision denominator + +- **Unrecognized verdicts** — rows whose verdict is neither `confirmed` nor `reversed`. Counted + separately, and never allowed to inflate precision or help a rule clear its sample floor. +- **Non-attributable backfill** — one historical backfill took its labels verbatim from another + rule's human verdicts on the same pull requests. Including it printed one rule's number twice, so + it is excluded from both the precision aggregate and the published corpus. +- **Firings with no human verdict at all.** Most firings never receive one. The absence of an + override is not evidence in either direction, so it is not counted as either. + +## Windows + +**Every window is flat and unweighted. There is no time-decay and no recency weighting anywhere.** A +verdict from day 89 counts exactly as much as one from yesterday. + +| Figure | Window | +| --- | --- | +| Per-rule precision, and the published eval corpus | **90 days** | +| Review-parity block (re-evaluation rate, author-class parity) | **7 days** | +| Weekly trends | **8 weeks** | +| Fleet decision accuracy | **90 days** | +| Underlying audit-event retention | **90 days** — a hard bound on everything above it | + +The review-parity window being 7 days while the rule table is 90 is a real difference, not a +rounding of the same thing. The page prints the parity window's actual start and end dates. + +The weekly trend buckets a reversal by the **original action's** date, not the reversal's, so a past +week's figure never shifts retroactively as new reversals arrive. + +## Precision, accuracy, and coverage + +**Per-rule precision** is `confirmed / (confirmed + reversed)`, rounded to three decimals and +published as a percentage to one decimal. + +**Decision accuracy** is reversal-grounded: the share of the gate's merge/close actions that were not +later reversed. Its denominator is auto-actions in the retention window; dry-run actions are excluded. + +**Coverage is computed but is not displayed on the fairness report.** Where it appears elsewhere it +means `verdicts / (verdicts + holds)` — the share of decidable pull requests the gate actually +decided rather than handing to a person. Policy enforcement actions sit outside both the numerator +and the denominator. The per-rule table publishes a `decided` **count** and no fired-to-decided +ratio, so it does not tell you what share of a rule's firings were ever adjudicated. + +## Confidence intervals + +Where an interval is published it is a **Wilson score interval at z = 1.96** (95%). Wilson rather +than Wald deliberately: the Wald interval degenerates as a proportion approaches 0 or 1 and would +claim near-impossible certainty at, say, 59 of 60 confirmed. + +**What actually carries an interval on the fairness report: the headline decision-accuracy figure, +and only when the deployment is reporting as a single instance.** Everything else is a bare point +estimate: + +- Per-rule precision has **no** interval. +- The per-repository accuracy column has **no** interval. +- Neither weekly trend has an interval. +- No review-parity figure — re-evaluation rate, close rate, hold rate, findings per PR — has an interval. + +Read those as point estimates with unstated uncertainty, and lean on the accompanying counts (`decided`, +`verdicts`, `n=`) to judge how much weight any of them can carry. + +## Sample floors and suppression + +A figure computed from too few observations is suppressed rather than published, because a small-sample +percentage is worse than no percentage at all. + +| Figure | Floor | What happens below it | +| --- | --- | --- | +| Per-rule precision | **10** decided | The row still appears with its count; the percentage reads "insufficient data" | +| Weekly accuracy trend | **3** samples | The whole week is blanked, volume columns included | +| Fleet weekly trend | **5** verdicts | Both the verdict count and the accuracy are blanked | +| An instance counting toward the fleet | **5** decided | The instance is excluded entirely | +| Anti-gaming comparison | **3** eligible instances | Flags caught reads as unknown, not zero | + +Where a deployment is reported alongside a small number of others, some counts are withheld entirely: +at two instances, publishing a total would let a reader recover the other instance's volume by subtraction. + +## Known limitations + +These are the places where the published numbers are weaker than they look. They are listed here +because a methodology page that omitted them would be marketing. + +- **An accuracy of "—" means not measurable, not 100%.** When a deployment took no auto-actions in a + period, there is no action a reversal could have been recorded against. Accuracy is unknown, and is + published as unknown. +- **Reversals are undercounted by construction.** A bare owner reopen with no merge inside six hours, + and an uncorroborated revert claim, both record nothing. Real "the gate was wrong" events can be missed. +- **The published corpus is capped and can be a prefix.** The downloadable corpus is bounded at 2,000 + cases, while the precision figures are computed with an unbounded count. When a rule's corpus is + truncated or empty, its checksum commitment is **omitted** rather than published — a commitment + covering a prefix of the cases a number was computed over would mislead a reader re-deriving it. +- **The corpus and the precision aggregate are built by different code paths.** A corpus case requires + a firing paired to an override; the precision aggregate counts override rows alone. An override with + no matching firing counts toward published precision but has no corpus case. Expect the two to be + close, not identical. +- **Backtest scoring is oriented the other way.** The scorer used for calibration treats `reversed` as + its positive class, so its precision is not comparable to the published `confirmed / decided`. +- **Reads fail soft.** A failed database read yields an empty result rather than an error, so a + transient failure under-counts a figure rather than surfacing itself. Compare against the counts and + the `updatedAt` timestamp before drawing conclusions from a single load. +- **Parity counts evaluations, not pull requests.** A re-evaluated head SHA is its own row, so + `verdicts` exceeds the number of distinct PRs whenever anything was re-evaluated. +- **The `unknown` author class is reported, never folded.** A PR whose author association was never + recorded gets its own row rather than being bucketed into maintainer or contributor — bucketing it + would bias exactly the comparison the block exists to make. + +## Where this lives in the code + +Kept adjacent to the scoring code it describes, so a change to a definition and a change to this page +show up in the same review: + +- Per-rule precision and the provenance exclusions — `src/review/public-rule-precision.ts` +- Outcome-confirmation writers (what makes a case confirmed or reversed) — `src/review/outcomes-wire.ts` +- Accuracy, totals, and the fleet block — `src/review/public-stats.ts` +- Wilson intervals and the fleet fold — `src/orb/analytics.ts` +- Review-parity rollups — `src/review/review-parity-rollups.ts` +- The published corpus and its checksum — `src/review/public-eval-corpus.ts` diff --git a/apps/loopover-ui/content/docs/verify-this-review.mdx b/apps/loopover-ui/content/docs/verify-this-review.mdx index d9a2296831..a5e0c56d96 100644 --- a/apps/loopover-ui/content/docs/verify-this-review.mdx +++ b/apps/loopover-ui/content/docs/verify-this-review.mdx @@ -11,6 +11,11 @@ website only build trust if a skeptic can check them without asking anyone's per page is the end-to-end walkthrough: export the same corpus snapshot the numbers come from, verify its checksum, replay the same scorer over it, and compare what you get against what is published. +For what each published number *means* — which rows are counted, over what window, and where a +figure stops supporting a conclusion — see the [fairness methodology](/docs/fairness-methodology). +For a single command that recomputes the published commitments and exits non-zero if any fail, +run `npx -p @loopover/mcp loopover-verify`. + Everything below runs read-only against a corpus export and pure functions from `@loopover/engine`. Nothing posts anywhere, and nothing needs a LoopOver API key. diff --git a/apps/loopover-ui/public/openapi.json b/apps/loopover-ui/public/openapi.json index 28e8d2aed6..57fb20e052 100644 --- a/apps/loopover-ui/public/openapi.json +++ b/apps/loopover-ui/public/openapi.json @@ -402,6 +402,10 @@ }, "minutesSaved": { "type": "number" + }, + "accuracyWindowDays": { + "type": "number", + "nullable": true } }, "required": [ @@ -416,6 +420,7 @@ "reversed", "filteredPct", "accuracyPct", + "accuracyWindowDays", "minutesSaved" ] }, diff --git a/apps/loopover-ui/src/components/site/command-palette.tsx b/apps/loopover-ui/src/components/site/command-palette.tsx index 6d6791a616..e7ae0f85b2 100644 --- a/apps/loopover-ui/src/components/site/command-palette.tsx +++ b/apps/loopover-ui/src/components/site/command-palette.tsx @@ -63,6 +63,7 @@ const DEFAULT_ITEMS: PaletteItem[] = [ { label: "AI summaries policy", to: "/docs/ai-summaries", group: "Docs" }, { label: "Backtest & calibration", to: "/docs/backtest-calibration", group: "Docs" }, { label: "Verify this review", to: "/docs/verify-this-review", group: "Docs" }, + { label: "Fairness methodology", to: "/docs/fairness-methodology", group: "Docs" }, { label: "Privacy & security", to: "/docs/privacy-security", group: "Docs" }, { label: "Troubleshooting", to: "/docs/troubleshooting", group: "Docs" }, { label: "API reference", to: "/api", group: "Reference" }, diff --git a/apps/loopover-ui/src/components/site/docs-nav.tsx b/apps/loopover-ui/src/components/site/docs-nav.tsx index 7fc8074f04..bd35961083 100644 --- a/apps/loopover-ui/src/components/site/docs-nav.tsx +++ b/apps/loopover-ui/src/components/site/docs-nav.tsx @@ -108,6 +108,7 @@ export const docsNav: DocsGroup[] = [ { to: "/docs/upstream-drift", label: "Upstream drift" }, { to: "/docs/backtest-calibration", label: "Backtest & calibration" }, { to: "/docs/verify-this-review", label: "Verify this review" }, + { to: "/docs/fairness-methodology", label: "Fairness methodology" }, { to: "/docs/what-you-can-verify", label: "What you can verify" }, ], }, diff --git a/apps/loopover-ui/src/components/site/fairness-report-page.test.tsx b/apps/loopover-ui/src/components/site/fairness-report-page.test.tsx index 0b7e744f56..b90e7e6f0b 100644 --- a/apps/loopover-ui/src/components/site/fairness-report-page.test.tsx +++ b/apps/loopover-ui/src/components/site/fairness-report-page.test.tsx @@ -70,6 +70,7 @@ const FIXTURE: PublicStats = { reversed: 2, filteredPct: 40, accuracyPct: 97.8, + accuracyWindowDays: 90, minutesSaved: 2000, }, weekly: { reviewed: 10, merged: 6 }, @@ -230,7 +231,9 @@ describe("FairnessReportPage (#fairness-analytics)", () => { await waitFor(() => expect(screen.getByText("Decision accuracy")).toBeTruthy()); const accuracyCard = screen.getByText("Decision accuracy").closest("div")!.parentElement!; expect(accuracyCard.textContent).toContain("97.8%"); - expect(screen.getByText("2 human-reversed, lifetime")).toBeTruthy(); + // The window comes from the payload: the headline used to say "lifetime" while the accuracy pairing + // behind it has always been bounded by audit-log retention. + expect(screen.getByText("2 human-reversed, last 90 days")).toBeTruthy(); }); it("explains a withheld accuracy instead of letting it read as a dash-shaped mystery", async () => { diff --git a/apps/loopover-ui/src/components/site/fairness-report-page.tsx b/apps/loopover-ui/src/components/site/fairness-report-page.tsx index d025c6cd74..7fac49da52 100644 --- a/apps/loopover-ui/src/components/site/fairness-report-page.tsx +++ b/apps/loopover-ui/src/components/site/fairness-report-page.tsx @@ -30,6 +30,16 @@ function UnmeasurableAccuracyNote() { const intFmt = new Intl.NumberFormat("en"); +/** How to describe the span `accuracyPct` actually covers. + * + * This said "lifetime", which it has never been: the accuracy pairing is deliberately bounded by the audit + * log's retention window (see public-stats.ts's own note on why a lifetime denominator drifts the ratio + * toward 100%), so the headline claimed a wider basis than the number had. The window comes from the + * payload rather than a literal here, so it tracks the retention policy instead of drifting from it. */ +function accuracyWindowLabel(windowDays: number | null): string { + return windowDays === null ? "reversal-grounded" : `last ${windowDays} days`; +} + async function fetchPublicStats(): Promise { const result = await apiFetch(`${getApiOrigin()}/v1/public/stats`, { label: "LoopOver fairness report", @@ -133,8 +143,8 @@ export function FairnessReportPage() { ? `across ${intFmt.format(data.fleetAccuracy.instanceCount)} self-hosted instance${data.fleetAccuracy.instanceCount === 1 ? "" : "s"}, last ${data.fleetAccuracy.windowDays} days` : `self-reported by one self-hosted instance, not corroborated across operators, last ${data.fleetAccuracy.windowDays} days` : data.totals.reversed > 0 - ? `${intFmt.format(data.totals.reversed)} human-reversed, lifetime` - : "reversal-grounded, lifetime"} + ? `${intFmt.format(data.totals.reversed)} human-reversed, ${accuracyWindowLabel(data.totals.accuracyWindowDays)}` + : `reversal-grounded, ${accuracyWindowLabel(data.totals.accuracyWindowDays)}`}

{/* #9168 computes `basis` precisely so this number is not read as corroborated-across-operators when it is one operator's own disclosed outcomes; the page used to drop the field entirely. */} @@ -374,6 +384,14 @@ export function FairnessReportPage() { > verify this review + , or read how every number here is computed in the{" "} + + fairness methodology + .

@@ -444,6 +462,14 @@ export function FairnessReportPage() { > verify this review + . The definitions behind each column are in the{" "} + + fairness methodology + .

diff --git a/apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx b/apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx index 2c763ca733..2f076b0a6a 100644 --- a/apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx +++ b/apps/loopover-ui/src/components/site/proof-of-power-stats.test.tsx @@ -70,6 +70,7 @@ const PAYLOAD: PublicStats = { reversed: 33, filteredPct: 48.6, accuracyPct: 98.4, + accuracyWindowDays: 90, minutesSaved: 54160, }, weekly: { reviewed: 2000, merged: 900 }, diff --git a/apps/loopover-ui/src/routes/docs.index.tsx b/apps/loopover-ui/src/routes/docs.index.tsx index 534f45e2b5..58e06afa71 100644 --- a/apps/loopover-ui/src/routes/docs.index.tsx +++ b/apps/loopover-ui/src/routes/docs.index.tsx @@ -96,6 +96,7 @@ const AUDIENCES: Audience[] = [ { to: "/docs/ai-summaries", label: "AI summaries policy" }, { to: "/docs/backtest-calibration", label: "Backtest & calibration" }, { to: "/docs/verify-this-review", label: "Verify this review" }, + { to: "/docs/fairness-methodology", label: "Fairness methodology" }, ], }, { diff --git a/packages/loopover-contract/src/public-api.ts b/packages/loopover-contract/src/public-api.ts index 8be0164a08..b1275588cf 100644 --- a/packages/loopover-contract/src/public-api.ts +++ b/packages/loopover-contract/src/public-api.ts @@ -89,6 +89,10 @@ export const PublicStatsSchema = z.object({ reversed: z.number(), filteredPct: z.number().nullable(), accuracyPct: z.number().nullable(), + /** The window `accuracyPct` and `reversed` are bounded by, in days -- null if unbounded. Both are + * pruned with the audit log while `merged`/`closed` beside them are lifetime and fleet-folded, so the + * bound is published rather than left for a reader to assume. */ + accuracyWindowDays: z.number().nullable(), minutesSaved: z.number(), }), weekly: z.object({ reviewed: z.number(), merged: z.number() }), diff --git a/src/db/retention.ts b/src/db/retention.ts index 3d70a332ed..cc5acd5350 100644 --- a/src/db/retention.ts +++ b/src/db/retention.ts @@ -195,6 +195,14 @@ export const RETENTION_COMPOSITE_PK_TABLES: ReadonlySet = new Set([ * hash-chained created_at (which cannot be backdated without breaking the chain), so an operator cannot use * the tolerance to hide a fresh deletion. */ +/** The retention window, in days, a table's rows survive -- null when the table has no rule. Exported so a + * surface that is BOUNDED by retention can publish that bound instead of restating it as a literal: the + * fairness report described its accuracy figure as "lifetime" while the pairing behind it is pruned with + * `audit_events`, and a hand-copied "90" would drift the first time the policy changed. */ +export function retentionDaysForTable(table: string): number | null { + return RETENTION_POLICY.find((candidate) => candidate.table === table)?.days ?? null; +} + export function retentionCutoffIsoForTable(table: string, nowMs: number = Date.parse(nowIso())): string | null { const rule = RETENTION_POLICY.find((candidate) => candidate.table === table); return rule ? cutoffIso(rule.days, nowMs) : null; diff --git a/src/review/public-stats.ts b/src/review/public-stats.ts index c866744034..c97cda7a12 100644 --- a/src/review/public-stats.ts +++ b/src/review/public-stats.ts @@ -55,7 +55,7 @@ import { validateCalibrationPayload } from "./risk-control"; import { loadRepoFocusManifest } from "../signals/focus-manifest-loader"; import { resolveLoopOverSelfRepoFullName } from "../config/loopover-repo-focus-manifest"; import { errorMessage } from "../utils/json"; -import { retentionCutoffIsoForTable } from "../db/retention"; +import { retentionCutoffIsoForTable, retentionDaysForTable } from "../db/retention"; /** FALLBACK estimate of maintainer review/triage time saved per reviewed PR, used ONLY when the real per-PR * average (`estimateReviewEffort`'s minutes, persisted at publish time — see `reviewEffortMinutes` in the @@ -231,6 +231,11 @@ export interface PublicStatsPayload { reversed: number; filteredPct: number | null; accuracyPct: number | null; + /** The window `accuracyPct` and `reversed` are actually bounded by, in days -- null if unbounded. + * Published rather than left implicit because both are pruned with `audit_events` while `merged`/ + * `closed` beside them are lifetime and fleet-folded; the page described the figure as "lifetime", + * which the pairing has never been. Derived from the retention policy so it cannot drift from it. */ + accuracyWindowDays: number | null; minutesSaved: number; }; /** Trailing-7-day additions (by review time), for the "+N this week" hero delta. */ @@ -631,6 +636,7 @@ export async function getPublicStats( // (own-ledger `reversed`) and the denominator are drawn from the same population AND the same // retention window. See the windowed disposition query's own comment for both halves of that pairing. accuracyPct: accuracyPct(windowedMerged, windowedClosed, totals.reversed), + accuracyWindowDays: retentionDaysForTable("audit_events"), minutesSaved, }, weekly: { reviewed: w.reviewed ?? 0, merged: w.merged ?? 0 }, diff --git a/test/unit/public-stats.test.ts b/test/unit/public-stats.test.ts index 217c052636..f39a3d6af8 100644 --- a/test/unit/public-stats.test.ts +++ b/test/unit/public-stats.test.ts @@ -1,3 +1,4 @@ +import { retentionDaysForTable } from "../../src/db/retention"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createTestEnv } from "../helpers/d1"; import { upsertRepoFocusManifest } from "../../src/signals/focus-manifest-loader"; @@ -367,6 +368,10 @@ describe("getPublicStats — live aggregate over the review ledger", () => { expect(out.totals.accuracyPct).not.toBe(99); // Per-project accuracy is already same-scope and stays unchanged: 1 - 10/100 = 90. expect(out.byProject[0]!.accuracyPct).toBe(90); + // #9725: the bound that makes the above true is PUBLISHED, not left for a reader to infer. The fairness + // page described this figure as "lifetime" while both halves of the ratio are pruned with audit_events. + expect(out.totals.accuracyWindowDays).toBe(retentionDaysForTable("audit_events")); + expect(out.totals.accuracyWindowDays).toBe(90); }); it("keeps own-ledger per-PR effort sum separate from Orb fleet flat credit", async () => {