diff --git a/apps/loopover-ui/content/docs/verify-this-review.mdx b/apps/loopover-ui/content/docs/verify-this-review.mdx index 36a3c6df5f..c9eaa32ff0 100644 --- a/apps/loopover-ui/content/docs/verify-this-review.mdx +++ b/apps/loopover-ui/content/docs/verify-this-review.mdx @@ -98,4 +98,19 @@ verified. [#8136](https://github.com/JSONbored/loopover/issues/8136) and [#8137](https://github.com/JSONbored/loopover/issues/8137). This walkthrough is honest about stopping at the reproducibility line rather than implying the stronger guarantee. + + **Live decision execution is permanently out of scope for attestation, by decision.** The + attested-evaluation epic ([#8534](https://github.com/JSONbored/loopover/issues/8534)) attests the + *offline* backtest replay only. Attesting the *live* gate-decision path (the merge/close calls + that actually act on a PR) was evaluated and declined — + [#9141](https://github.com/JSONbored/loopover/issues/9141) — because a network-reachable + attestation service in the live request path trades a real availability guarantee for a proof the + deterministic rule stage already gives more cheaply through replay + ([#8832](https://github.com/JSONbored/loopover/issues/8832), + [#8838](https://github.com/JSONbored/loopover/issues/8838)), and because attestation proves + *computation*, not the decision record's own honesty — the actual gap a skeptical verifier cares + about, closed instead by an externally anchored, complete, replayable decision record. What "not + proved" above means in practice: every live decision's provenance rests on the decision-record + + replay + ledger-anchoring trust surface, never on attested execution — and that is not a temporary + gap awaiting hardware, it is the standing design. diff --git a/packages/loopover-engine/src/calibration/attestation-envelope.ts b/packages/loopover-engine/src/calibration/attestation-envelope.ts index cf4546a80f..d0065a550f 100644 --- a/packages/loopover-engine/src/calibration/attestation-envelope.ts +++ b/packages/loopover-engine/src/calibration/attestation-envelope.ts @@ -9,6 +9,18 @@ // separate maintainer work in the parent epic -- doing any of it here would be unreviewable scope and would // bake a verification policy into what is meant to be a transport shape. Same purity contract as the rest of // this module family: no IO, no randomness, no wall-clock reads. +// +// #9140 (schemaVersion 1, in place before any envelope is persisted -- #8537 has not shipped yet, so this is +// a pre-launch correction, not a breaking migration): two gaps fixed together. +// 1. `reportData` was a bare 32-byte SHA-256 hex digest, but SEV-SNP's `REPORT_DATA` field and TDX's +// `REPORTDATA` are both 64 BYTES (128 hex chars) of guest-supplied data -- the digest has to be PLACED +// into a field twice its width, and nothing specified how. See {@link buildAttestationReportData}'s +// doc comment for the finalized layout + worked test vectors. +// 2. The binding (`corpusChecksum:headSha:baseSha`) is deterministic across runs -- the same corpus at the +// same SHAs always produces the same `reportData`, so one genuine attested run's report could be +// presented for any later run with an identical binding. The second half of the now-64-byte field is a +// freshness component (a nonce or monotonic run id) tying the report to exactly one run; the envelope's +// new `runId` field carries the plaintext value a verifier checks it against. import { createHash } from "node:crypto"; @@ -30,8 +42,15 @@ export type AttestationEnvelope = { runtimeClass: string; /** Launch measurement, lowercase hex, 32-128 hex chars (widths differ per TEE technology). */ measurement: string; - /** sha256, lowercase hex, exactly 64 chars -- see {@link buildAttestationReportData} for the binding. */ + /** The 64-byte REPORT_DATA/REPORTDATA payload, lowercase hex, exactly 128 hex chars -- see + * {@link buildAttestationReportData} for the binding + freshness layout. */ reportData: string; + /** The freshness component bound into `reportData`'s second half (as `sha256(runId)`) -- a nonce or + * monotonic run id, plaintext here so a verifier can recompute `sha256(runId)` and check it against + * `reportData.slice(64)` AND against whatever run identifier they expect (e.g. the backtest run row this + * evidence claims to belong to), which is what stops a genuinely-signed but STALE report from being + * presented for a different run. Lowercase hex, 1-128 chars -- same shape family as `measurement`. */ + runId: string; /** The raw attestation report, base64, non-empty and <= 65536 chars. Never parsed here. */ attestationReport: string; verification: AttestationVerification; @@ -41,17 +60,25 @@ const TEE_TECHNOLOGIES: readonly string[] = ["sev-snp", "tdx"]; const RUNTIME_CLASS_MAX = 128; const MEASUREMENT_MIN_HEX = 32; const MEASUREMENT_MAX_HEX = 128; -const REPORT_DATA_HEX = 64; +/** #9140: 128 hex chars = 64 bytes, matching SEV-SNP `REPORT_DATA` / TDX `REPORTDATA` exactly (was 64/32, + * half the required width -- see this module's header comment). */ +const REPORT_DATA_HEX = 128; +const RUN_ID_MIN_HEX = 1; +const RUN_ID_MAX_HEX = 128; const ATTESTATION_REPORT_MAX = 65536; const LOWERCASE_HEX = /^[0-9a-f]+$/; const ISO_DATETIME = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})$/; -const BASE64 = /^[A-Za-z0-9+/]+={0,2}$/; +// #9140: the old `/^[A-Za-z0-9+/]+={0,2}$/` accepted a string whose length was not a multiple of 4 (e.g. 5 +// base64 characters), which is not valid base64 under any padding rule. This form requires the body to be +// whole groups of 4 characters, with at most one trailing padded group (2 chars + `==`, or 3 chars + `=`). +const BASE64 = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/; const ENVELOPE_KEYS: readonly string[] = [ "schemaVersion", "teeTechnology", "runtimeClass", "measurement", "reportData", + "runId", "attestationReport", "verification", ]; @@ -61,14 +88,68 @@ const VERIFICATION_KEYS: Record RUN_ID_MAX_HEX + ) { + errors.push(`runId: expected ${RUN_ID_MIN_HEX}-${RUN_ID_MAX_HEX} lowercase hex characters`); + } + const attestationReport = record["attestationReport"]; if ( !nonEmptyString(attestationReport) || diff --git a/packages/loopover-engine/test/attestation-envelope.test.ts b/packages/loopover-engine/test/attestation-envelope.test.ts index fee6d1b17c..1d319e19d6 100644 --- a/packages/loopover-engine/test/attestation-envelope.test.ts +++ b/packages/loopover-engine/test/attestation-envelope.test.ts @@ -8,21 +8,68 @@ const BASE = { teeTechnology: "sev-snp", runtimeClass: "loopover-backtest-runner", measurement: "a".repeat(64), - reportData: "b".repeat(64), + reportData: "b".repeat(128), + runId: "d4".repeat(16), attestationReport: "QUJD", verification: { status: "unverified" }, }; +// #9140 worked example, asserted verbatim (also published in buildAttestationReportData's own doc comment as +// the spec a non-JS verifier reimplements against). +const CORPUS_CHECKSUM = "a1".repeat(32); // 64 hex +const HEAD_SHA = "b2".repeat(20); // 40 hex +const BASE_SHA = "c3".repeat(20); // 40 hex +const RUN_ID = "d4".repeat(16); // 32 hex +const EXPECTED_REPORT_DATA = + "3309f8c4eadab7422c8b5ba378a12d52b5676f601faa4dcbac213bae93f5ae7e" + + "1d88ffa7d3cf1f07e5cf64b62016f3e688ad473286f2f613886b6ac02d00541d"; + test("barrel: the public entrypoint re-exports the attestation-envelope primitives (#8541)", () => { assert.equal(typeof buildAttestationReportData, "function"); assert.equal(typeof validateAttestationEnvelope, "function"); }); -test("buildAttestationReportData pins the corpusChecksum:headSha:baseSha binding (#8541)", () => { - assert.equal( - buildAttestationReportData({ corpusChecksum: "abc123", headSha: "head456", baseSha: "base789" }), - "fcc875115df49bf143b18fc8a8071e9a946858407fabf61e59ef1607d1cfb140", - ); +test("buildAttestationReportData: the published test vector (#9140) — sha256(binding) || sha256(runId), 128 hex chars", () => { + const reportData = buildAttestationReportData({ corpusChecksum: CORPUS_CHECKSUM, headSha: HEAD_SHA, baseSha: BASE_SHA, runId: RUN_ID }); + assert.equal(reportData, EXPECTED_REPORT_DATA); + assert.equal(reportData.length, 128); +}); + +test("buildAttestationReportData: injective over the (corpusChecksum, headSha, baseSha, runId) quadruple", () => { + const base = { corpusChecksum: CORPUS_CHECKSUM, headSha: HEAD_SHA, baseSha: BASE_SHA, runId: RUN_ID }; + const variants = [ + base, + { ...base, corpusChecksum: "a2".repeat(32) }, + { ...base, headSha: "b3".repeat(20) }, + { ...base, baseSha: "c4".repeat(20) }, + { ...base, runId: "d5".repeat(16) }, + // A different git-sha WIDTH (64-hex SHA-256 git object ids) must still be a distinct commitment. + { ...base, headSha: "b2".repeat(32) }, + ]; + const digests = variants.map((binding) => buildAttestationReportData(binding)); + assert.equal(new Set(digests).size, digests.length, "every varied input must produce a distinct reportData"); +}); + +test("buildAttestationReportData: the freshness half is independently addressable — only runId moves the second 64 hex chars", () => { + const base = { corpusChecksum: CORPUS_CHECKSUM, headSha: HEAD_SHA, baseSha: BASE_SHA, runId: RUN_ID }; + const baseReportData = buildAttestationReportData(base); + const freshRun = buildAttestationReportData({ ...base, runId: "d5".repeat(16) }); + assert.equal(freshRun.slice(0, 64), baseReportData.slice(0, 64), "the binding half is unaffected by a run-id change"); + assert.notEqual(freshRun.slice(64), baseReportData.slice(64), "the freshness half MUST change so a stale report cannot be replayed for a new run"); + + const rebound = buildAttestationReportData({ ...base, corpusChecksum: "a2".repeat(32) }); + assert.notEqual(rebound.slice(0, 64), baseReportData.slice(0, 64), "the binding half moves when the corpus/shas change"); + assert.equal(rebound.slice(64), baseReportData.slice(64), "the freshness half is unaffected by a binding-only change"); +}); + +test("buildAttestationReportData: throws (never silently mis-commits) on a malformed input shape", () => { + const base = { corpusChecksum: CORPUS_CHECKSUM, headSha: HEAD_SHA, baseSha: BASE_SHA, runId: RUN_ID }; + assert.throws(() => buildAttestationReportData({ ...base, corpusChecksum: "abc123" }), /corpusChecksum/); + assert.throws(() => buildAttestationReportData({ ...base, corpusChecksum: `${CORPUS_CHECKSUM.slice(0, 63)}:` }), /corpusChecksum/); + assert.throws(() => buildAttestationReportData({ ...base, headSha: "not-hex-and-wrong-length" }), /headSha/); + assert.throws(() => buildAttestationReportData({ ...base, baseSha: "" }), /baseSha/); + assert.throws(() => buildAttestationReportData({ ...base, runId: "UPPER" }), /runId/); + assert.throws(() => buildAttestationReportData({ ...base, runId: "" }), /runId/); }); test("validateAttestationEnvelope accepts a well-formed envelope and each verification variant (#8541)", () => { @@ -41,9 +88,39 @@ test("validateAttestationEnvelope accepts a well-formed envelope and each verifi }); test("validateAttestationEnvelope rejects structurally invalid input without throwing (#8541)", () => { - for (const bad of [null, undefined, 42, "envelope", [], { ...BASE, schemaVersion: 2 }, { ...BASE, reportData: "b".repeat(63) }, { ...BASE, rogue: 1 }]) { + for (const bad of [ + null, + undefined, + 42, + "envelope", + [], + { ...BASE, schemaVersion: 2 }, + { ...BASE, reportData: "b".repeat(63) }, // #9140: pre-fix width (32 bytes) is now rejected + { ...BASE, reportData: "b".repeat(127) }, + { ...BASE, rogue: 1 }, + ]) { const result = validateAttestationEnvelope(bad); assert.equal(result.valid, false); assert.ok(Array.isArray(result.errors) && result.errors.length > 0); } }); + +// #9140: the freshness envelope field — must be present, hex-shaped, and bounded like `measurement`. +test("validateAttestationEnvelope: runId is required, hex, and bounded 1-128 chars", () => { + assert.equal(validateAttestationEnvelope({ ...BASE, runId: undefined }).valid, false); + const missing = { schemaVersion: BASE.schemaVersion, teeTechnology: BASE.teeTechnology, runtimeClass: BASE.runtimeClass, measurement: BASE.measurement, reportData: BASE.reportData, attestationReport: BASE.attestationReport, verification: BASE.verification }; + assert.equal(validateAttestationEnvelope(missing).valid, false); + assert.equal(validateAttestationEnvelope({ ...BASE, runId: "NOTHEX" }).valid, false); + assert.equal(validateAttestationEnvelope({ ...BASE, runId: "" }).valid, false); + assert.equal(validateAttestationEnvelope({ ...BASE, runId: "a".repeat(129) }).valid, false); + assert.equal(validateAttestationEnvelope({ ...BASE, runId: "a" }).valid, true); +}); + +// #9140: the old regex accepted any run of base64-alphabet characters regardless of length, including widths +// that are not valid base64 under any padding rule (e.g. 5 characters can never be a whole base64 payload). +test("validateAttestationEnvelope: attestationReport must be valid base64 — length a multiple of 4", () => { + assert.equal(validateAttestationEnvelope({ ...BASE, attestationReport: "QUJDQ" }).valid, false); // 5 chars + assert.equal(validateAttestationEnvelope({ ...BASE, attestationReport: "QUJDQQ" }).valid, false); // 6 chars, no padding + assert.equal(validateAttestationEnvelope({ ...BASE, attestationReport: "QUJDQQ==" }).valid, true); // properly padded + assert.equal(validateAttestationEnvelope({ ...BASE, attestationReport: "QUJD" }).valid, true); // exact multiple of 4 +}); diff --git a/scripts/backfill-decision-labels.ts b/scripts/backfill-decision-labels.ts index b7a4d335e3..b352a7cc2b 100644 --- a/scripts/backfill-decision-labels.ts +++ b/scripts/backfill-decision-labels.ts @@ -67,7 +67,7 @@ export async function buildBundle(rows: CandidateRow[], stagedAt: string): Promi action: target.stratum === "close_arm" ? "close" : "hold", reasonCode: target.reasonCode, configDigest: "backfill:unavailable", - modelId: null, + modelIds: null, promptDigest: null, aiConfidence: target.aiConfidence, decidedAt: target.decidedAt, diff --git a/scripts/replay-decision.ts b/scripts/replay-decision.ts index a649ce8f46..aebc552a26 100644 --- a/scripts/replay-decision.ts +++ b/scripts/replay-decision.ts @@ -7,7 +7,14 @@ // // The bundle is one JSON object: { record: {...decision_records row}, replayInput: {...replay_json} }. // EXTRACT (operator, against the instance DB): -// SELECT json_build_object('record', to_jsonb(dr), 'replayInput', dri.replay_json::jsonb) +// SELECT json_build_object( +// -- #9135: `|| dr.record_json::jsonb` overlays every field the PUBLIC record carries (including +// -- camelCase fields that live only inside record_json, e.g. `divertedByHoldout`) on top of the +// -- raw snake_case row columns, so a field added to the record schema needs no matching change +// -- here to reach the CLI. +// 'record', to_jsonb(dr) || dr.record_json::jsonb, +// 'replayInput', dri.replay_json::jsonb +// ) // FROM decision_records dr JOIN decision_replay_inputs dri ON dri.record_id = dr.id // WHERE dr.id = 'record:#@'; // @@ -33,6 +40,8 @@ export function runReplayBundle(raw: string): { outcome: ReturnType AI_JUDGMENT_BLOCKER_CODES.has(blocker.code)); // #8962: recomputed here (two cheap reads, only when an AI judgment shaped the decision) rather than // threaded from the plan site — the record must carry the boundary evidence even when the knob is off. const salvageability = aiJudgment !== undefined ? await computeSalvageabilityForTarget(env, repoFullName, pr.number, pr.authorLogin, gate) : null; + const resolvedPolicy = gate.replay?.policy ?? settings; const { record, recordDigest } = await buildDecisionRecord({ repoFullName, pullNumber: pr.number, @@ -3494,21 +3506,25 @@ async function runAgentMaintenancePlanAndExecute( // #8838: the shared derivation — replayDecision recomputes reasonCode through this SAME function, so // the live mapping and the replay mapping can never drift apart. reasonCode: deriveDecisionReasonCode(disposition.blockerClass, policyCloseKind ?? null, gate.conclusion), - configDigest: await contentDigest(settings), + configDigest: await contentDigest({ policy: resolvedPolicy, untrustworthyRuleCodes: [...untrustworthyRuleCodes].sort() }), + settingsDigest: await contentDigest(settings), gatePack: settings.gatePack, - ciState: null, - modelId: null, - promptDigest: aiJudgment !== undefined ? await contentDigest({ version: REVIEW_PROMPT_VERSION, template: REVIEW_SYSTEM_PROMPT }) : null, + ciState: ciAggregate.ciState, + modelIds: aiJudgment?.modelIds ?? null, + promptDigest: aiJudgment?.promptDigest ?? null, aiConfidence: aiJudgment?.confidence ?? null, salvageability, + // #9135: legible on the record's own face — see maybeApplyCloseAuditHoldout's doc comment. + divertedByHoldout: closeAuditHoldout?.diverted ?? false, }); const recordId = await persistDecisionRecord(env, record, recordDigest); // #8838: persist the evaluation's own exact inputs beside the record (PRIVATE sibling, migration 0181) // so the replay harness can re-derive this decision bit-exactly. Best-effort, like the record itself; // the no-replay no-op (synthetic content-lane/bridge evaluations) lives inside the helper. Keyed to the // id persistDecisionRecord actually wrote (#9123: a supersession at the same head gets a revisioned id, - // not the base one this used to always recompute independently). - if (recordId !== null) await persistDecisionReplayInputForGate(env, recordId, gate, policyCloseKind ?? null); + // not the base one this used to always recompute independently). #9135: the holdout outcome rides along + // so `holdout_consistency` has something to check the public record against. + if (recordId !== null) await persistDecisionReplayInputForGate(env, recordId, gate, policyCloseKind ?? null, closeAuditHoldout ?? null); } // #2349 (PR 1): additive per-contributor calibration data, gated identically to recordNativeGateDecision // above -- see src/review/contributor-calibration.ts's doc comment. Currently write-only; nothing reads diff --git a/src/review/close-audit-holdout.ts b/src/review/close-audit-holdout.ts index 24d4027bd5..8c2ed9b0fd 100644 --- a/src/review/close-audit-holdout.ts +++ b/src/review/close-audit-holdout.ts @@ -9,19 +9,94 @@ // // PLACEMENT CONTRACT: the draw consumes the FINAL post-breaker plan — it runs after every gate/breaker // decision is made and can therefore never influence the decision itself, only whether the already-decided -// close executes or is diverted to a human. ε = 0 (the default) returns the plan untouched with zero I/O — -// byte-identical to today. +// close executes or is diverted to a human. ε = 0 (the default, and any repo without the +// gate.closeAuditHoldoutPct manifest knob) returns the plan untouched with zero I/O — byte-identical to +// today. // -// SCOPE: heuristic closes only. Policy closes (contributor cap, blacklist, copycat, review-nag, -// linked-issue hard rule) are enforcement, not quality predictions (#8827) — holding one for an accuracy -// audit would suspend enforcement for no measurement gain. Staged (auto_with_approval) closes already get a -// human; only autonomy-auto closes need the instrument. +// SCOPE: heuristic closes only. Policy closes (contributor cap, blacklist, copycat, review-nag, linked-issue +// hard rule) are enforcement, not quality predictions (#8827) — holding one for an accuracy audit would +// suspend enforcement for no measurement gain. Staged (auto_with_approval) closes already get a human; only +// autonomy-auto closes need the instrument. +// +// #9135: the draw itself used to be `Math.random()`, recorded nowhere — two decisions with identical +// findings/policy could carry `action: "close"` and `action: "hold"` with no trace of why. Fixed by deriving +// the draw from `HMAC(instance secret, seed)` (see `computeHoldoutDraw` below) instead of raw entropy, and by +// returning the decision-time holdout outcome (`{epsilonPct, draw, diverted}`) alongside the plan so the +// caller can persist it — onto `DecisionRecord.divertedByHoldout` (public) and +// `DecisionReplayInput.holdout` (private, `src/review/decision-replay.ts`) — instead of dropping it on the +// floor. The HMAC seed is the decision's own eventual record id (`record:#@`), so the +// draw is BOTH reproducible (an operator holding the instance secret can recompute it straight from the +// published record's own identity fields) AND unpredictable to a contributor without that secret trying to +// time a PR to dodge the holdout. import { DECISION_AUDIT_RUBRIC_VERSION } from "./decision-audit"; import { recordAuditEvent } from "../db/repositories"; import { incr } from "../selfhost/metrics"; import { resolveAgentDispositionLabels, type AgentDispositionLabelSettings, type PlannedAgentAction } from "../settings/agent-actions"; +import type { DecisionReplayHoldout } from "./decision-replay"; +import { hmacHex } from "../utils/crypto"; import { errorMessage, nowIso } from "../utils/json"; +/** system_flags key for this instrument's own dedicated HMAC secret (#9135). */ +const CLOSE_AUDIT_HOLDOUT_SECRET_FLAG = "close_audit_holdout:secret"; + +/** 256-bit random secret, lowercase hex. Web Crypto (worker + node) -- no external dependency needed for a + * one-shot key generation. */ +function generateHoldoutSecret(): string { + const bytes = crypto.getRandomValues(new Uint8Array(32)); + return [...bytes].map((byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +/** + * The instance's dedicated secret for deriving the close-audit holdout draw (#9135) — 256-bit, generated + * once and persisted in `system_flags`, race-safe across instances sharing a Postgres DB (INSERT OR IGNORE + + * re-read, mirroring `src/selfhost/orb-collector.ts`'s `getOrCreateAnonSecret`). SINGLE-PURPOSE: never the + * telemetry anonymization secret, the GitHub App private key, or any webhook-verification secret (key + * separation, the same discipline those other secrets already follow) — reusing one of them here would let + * this instrument's own published draw values leak information about a differently-scoped credential, and + * mixing this secret into telemetry hashing would do the reverse. + */ +async function getOrCreateCloseAuditHoldoutSecret(env: Env): Promise { + const read = async (): Promise => { + const row = await env.DB.prepare("SELECT value FROM system_flags WHERE key = ?") + .bind(CLOSE_AUDIT_HOLDOUT_SECRET_FLAG) + .first<{ value: string }>(); + return row?.value; + }; + const existing = await read(); + if (existing) return existing; + const generated = generateHoldoutSecret(); + await env.DB.prepare("INSERT OR IGNORE INTO system_flags (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)") + .bind(CLOSE_AUDIT_HOLDOUT_SECRET_FLAG, generated) + .run(); + /* v8 ignore next -- a row always exists after INSERT OR IGNORE, so the ?? fallback is unreachable */ + return (await read()) ?? generated; +} + +/** The deterministic seed a decision's draw is derived from — the SAME string `buildDecisionRecord`'s caller + * assembles as the record id (`record:#@`), so a verifier holding the instance secret + * can recompute the draw directly from the published record's own identity fields, with no side channel + * beyond the secret itself. Every re-evaluation of the same (repo, PR, head sha) reproduces the identical + * seed, and therefore the identical draw — the holdout does not re-flip a coin on every webhook retry. */ +function holdoutSeed(repoFullName: string, pullNumber: number, headSha: string | null | undefined): string { + return `record:${repoFullName}#${pullNumber}@${headSha ?? "unknown"}`; +} + +/** Map an HMAC-SHA256 hex digest to a [0,1) float from its first 4 bytes (32 bits) — the same width + * `Math.random()` promises, uniform enough for a fairness-audit draw (this is a reproducible SUBSTITUTE for + * `Math.random()`, not a cryptographic use of the resulting number). Exported so tests can assert the exact + * mapping against a known HMAC output. */ +export function hmacHexToUnitFloat(hex: string): number { + return Number.parseInt(hex.slice(0, 8), 16) / 0x100000000; +} + +/** The production draw (#9135): `HMAC(instance secret, seed)` rather than `Math.random()`, so it is + * reproducible from the record without storing raw entropy, and unpredictable to a contributor who does not + * hold the instance secret trying to time a PR to dodge the holdout. */ +async function computeHoldoutDraw(env: Env, seed: string): Promise { + const secret = await getOrCreateCloseAuditHoldoutSecret(env); + return hmacHexToUnitFloat(await hmacHex(secret, seed)); +} + /** A planned close the holdout may divert: heuristic (quality) closes executing WITHOUT a human in the loop. */ export function holdoutEligibleClose(planned: PlannedAgentAction[]): PlannedAgentAction | undefined { return planned.find((action) => action.actionClass === "close" && action.closeKind === "heuristic" && action.requiresApproval !== true); @@ -50,17 +125,26 @@ export function applyCloseAuditHoldout(planned: PlannedAgentAction[], labelSetti /** * The full holdout step: eligibility → draw → (on fire) divert the plan, persist the propensity record and - * the pending adjudication label row. Returns the (possibly diverted) plan. + * the pending adjudication label row. Returns the (possibly diverted) plan AND the decision-time holdout + * outcome (#9135) so the caller can persist it onto the decision record and its replay input — `null` when + * the holdout never evaluated at all (ε absent/0, close autonomy not auto, or no eligible close — the + * overwhelmingly common, zero-I/O path). * * Best-effort persistence with a HARD ordering rule: the plan is only diverted when the propensity record * WROTE — a hold whose draw was never logged is invisible to every downstream estimator and would silently * bias coverage, so on a write failure the close proceeds unheld (the instrument, not the gate, degrades). + * `holdout.diverted` always reflects what ACTUALLY happened to the plan (false on a write-failure + * degradation too), never merely "the draw fell under ε." */ export async function maybeApplyCloseAuditHoldout( env: Env, input: { repoFullName: string; pullNumber: number; + /** #9135: this decision's head sha — part of the deterministic draw seed (see `holdoutSeed`) and + * reproduced verbatim in the eventual decision record id. Absent only for a caller that genuinely has + * no head sha yet (mirrors `buildDecisionRecord`'s own `headSha ?? "unknown"` fallback). */ + headSha?: string | null | undefined; planned: PlannedAgentAction[]; /** gate.closeAuditHoldoutPct — percent 0-20; absent/0/invalid disables. */ epsilonPct: number | null | undefined; @@ -69,14 +153,14 @@ export async function maybeApplyCloseAuditHoldout( labelSettings?: AgentDispositionLabelSettings; rng?: () => number; }, -): Promise { +): Promise<{ planned: PlannedAgentAction[]; holdout: DecisionReplayHoldout | null }> { const epsilonPct = input.epsilonPct ?? 0; - if (epsilonPct <= 0 || !input.closeAutonomyIsAuto) return input.planned; + if (epsilonPct <= 0 || !input.closeAutonomyIsAuto) return { planned: input.planned, holdout: null }; const eligible = holdoutEligibleClose(input.planned); - if (eligible === undefined) return input.planned; + if (eligible === undefined) return { planned: input.planned, holdout: null }; - const draw = (input.rng ?? Math.random)(); - if (draw >= epsilonPct / 100) return input.planned; + const draw = input.rng ? input.rng() : await computeHoldoutDraw(env, holdoutSeed(input.repoFullName, input.pullNumber, input.headSha)); + if (draw >= epsilonPct / 100) return { planned: input.planned, holdout: { epsilonPct, draw, diverted: false } }; const targetId = `${input.repoFullName}#${input.pullNumber}`; try { @@ -99,8 +183,8 @@ export async function maybeApplyCloseAuditHoldout( .run(); } catch (error) { console.warn(JSON.stringify({ event: "close_audit_holdout_record_error", target: targetId, message: errorMessage(error).slice(0, 160) })); - return input.planned; // unlogged hold = biased instrument; the decided close proceeds instead + return { planned: input.planned, holdout: { epsilonPct, draw, diverted: false } }; // unlogged hold = biased instrument; the decided close proceeds instead } incr("loopover_close_audit_holdouts_total"); - return applyCloseAuditHoldout(input.planned, input.labelSettings); + return { planned: applyCloseAuditHoldout(input.planned, input.labelSettings), holdout: { epsilonPct, draw, diverted: true } }; } diff --git a/src/review/decision-record.ts b/src/review/decision-record.ts index 88c1dcfa50..f408f12e4c 100644 --- a/src/review/decision-record.ts +++ b/src/review/decision-record.ts @@ -20,10 +20,26 @@ // tracked follow-up once tenants exist, not yet built. That gap does not reduce the value against every OTHER // actor (a maintainer quietly deleting one disputed decision, or an unprivileged bug), or against accidental // corruption — both of which the chain below still catches deterministically. +// +// #9124 (v4): three of the four commitments this record makes did not commit to what actually decided the +// PR — fixed together, since all three are computed at the same call site (processors.ts): +// - `configDigest` now digests the RESOLVED `gateCheckPolicy(...)` object (the thing `evaluateGateCheck` +// actually ran against — including the live-calibrated AI close-confidence floor and the cron-refreshed +// untrustworthy-rule-code set) instead of raw `settings`. The raw settings digest survives as the new, +// separate `settingsDigest` field. +// - `promptDigest` now digests the ACTUAL `buildSystemPrompt(...)` output for this call (base template plus +// whichever of its suffixes resolved: grounding/enrichment/profile/security-focus/path-instructions/ +// `review.instructions`/screenshot-evidence/inline/category/improvement-signal), not the base constant +// alone — a changed `review.instructions` now moves this digest. +// - `modelIds` (renamed from the always-null `modelId`) carries the REAL parsed-reviewer identities from +// `reviewDiagnostics`, the full set when more than one model produced a usable opinion. +// - `ciState` is populated from the live CI aggregate already in scope at the call site (was hardcoded null). +// #9135 (also v4): `divertedByHoldout` records when the randomized close-audit holdout (#8831) converted this +// decision's plan from a heuristic close to a hold — see that field's own doc comment. import { errorMessage, nowIso } from "../utils/json"; /** Bump when the record's FIELD SET changes meaning — consumers compare records only within a version. */ -export const DECISION_RECORD_SCHEMA_VERSION = "3"; // v3 (#8962): + salvageability {score, factors}; v2 (#8834): + aiConfidence, model/prompt commitments +export const DECISION_RECORD_SCHEMA_VERSION = "4"; // v4 (#9124/#9135): configDigest digests the resolved policy (+ settingsDigest split out), promptDigest digests the actual sent prompt, modelId -> modelIds (real identities), ciState populated, + divertedByHoldout; v3 (#8962): + salvageability {score, factors}; v2 (#8834): + aiConfidence, model/prompt commitments /** * Canonical JSON: recursively key-sorted, no insignificant whitespace — the ONE serialization every digest @@ -71,15 +87,32 @@ export type DecisionRecord = { action: string; /** The clause that decided it: a blocker class, `policy_close:`, or the gate conclusion. */ reasonCode: string; - /** Digest of the RESOLVED effective settings (canonical JSON) — commits the operator to the exact config - * that judged this PR, including private overlays, without publishing their contents. */ + /** #9124: digest of the RESOLVED `gateCheckPolicy(...)` object (canonical JSON) — the thing + * `evaluateGateCheck` actually ran against, including the live-calibrated AI close-confidence floor + * (`readCalibratedThreshold`/`system_flags`, resolved OUTSIDE `settings`) and the cron-refreshed + * untrustworthy-rule-code set (`readUntrustworthyRuleCodes`) the precision breaker consults. Two decisions + * computed under a different calibrated floor or a different untrustworthy-code set now publish different + * digests even when raw `settings` is unchanged — see `settingsDigest` below for the raw-config commitment + * this field used to make instead. */ configDigest: string; + /** #9124: digest of the RAW resolved effective settings (`.loopover.yml` > DB > defaults, canonical JSON) — + * what `configDigest` digested before v4. Kept as its own field because it is independently useful (an + * operator can diff two decisions' settings without reconstructing the full resolved-policy shape); null + * for a caller that has not computed it. */ + settingsDigest: string | null; /** The gate policy pack in force (public enum, safe to publish alongside the digest). */ gatePack: string | null; /** CI aggregate consumed by the decision, when one was read. */ ciState: string | null; - /** Model + prompt commitments when an AI review contributed; null for rule-only decisions. */ - modelId: string | null; + /** #9124: the distinct provider/model identities (`AiReviewDiagnostic.model`, deduped + sorted) whose + * output actually shaped this decision — the FULL set when more than one model produced a parsed opinion, + * never a representative one. Renamed from the always-null `modelId` (v3 and earlier never threaded the + * real identities through). null for rule-only decisions. */ + modelIds: string[] | null; + /** #9124: sha256 of the ACTUAL system prompt sent (`buildSystemPrompt`'s real output — the base template + * plus whichever of its up-to-ten suffixes resolved for this call), not a digest of the base constant + * alone. A changed `review.instructions` (or any other suffix input) now moves this digest. null for + * rule-only decisions. */ promptDigest: string | null; /** #8834: the calibrated confidence of the AI-judgment finding that shaped this decision (consensus * defect / split), null when no AI judgment contributed. Persisted so every decision joins the @@ -89,19 +122,28 @@ export type DecisionRecord = { * decision — the second-axis evidence for auditing the close/hold boundary. null for rule-only decisions * (and for reconstructed/backfilled records predating v3). */ salvageability: { score: number; factors: string[] } | null; + /** #9135: true when the randomized close-audit holdout (#8831) diverted this decision's plan — the + * `action` recorded above is a HOLD that the deterministic pipeline would otherwise have executed as a + * heuristic CLOSE. Makes that divergence legible on the record's own face instead of requiring a + * cross-reference against `decision_audit_holdout` audit rows to notice two byte-identical-looking + * records disagree on outcome. false for every decision the holdout never touched, including every + * decision recorded before #9135 shipped (via `buildDecisionRecord`'s normalization below). */ + divertedByHoldout: boolean; decidedAt: string; }; /** Assemble the record and its own content digest. PURE given pre-computed digests. Normalizes the * optional-shaped caller fields (undefined -> null) HERE so call sites carry no fallback arms of their own. */ export async function buildDecisionRecord( - input: Omit & { + input: Omit & { decidedAt?: string; gatePack?: string | null | undefined; ciState?: string | null | undefined; baseSha?: string | null | undefined; aiConfidence?: number | null | undefined; salvageability?: { score: number; factors: string[] } | null | undefined; + settingsDigest?: string | null | undefined; + divertedByHoldout?: boolean | undefined; }, ): Promise<{ record: DecisionRecord; recordDigest: string }> { const record: DecisionRecord = { @@ -113,6 +155,8 @@ export async function buildDecisionRecord( baseSha: input.baseSha ?? null, aiConfidence: input.aiConfidence ?? null, salvageability: input.salvageability ?? null, + settingsDigest: input.settingsDigest ?? null, + divertedByHoldout: input.divertedByHoldout ?? false, }; return { record, recordDigest: await contentDigest(record) }; } @@ -173,12 +217,22 @@ export async function persistDecisionRecord(env: Env, record: DecisionRecord, re * not a digest commitment; the full value is the record's own `headSha` field). Returned WITHOUT a details * wrapper — the unified-comment bridge renders the collapsible chrome itself (UnifiedCollapsible). */ export function renderDecisionRecordSection(record: DecisionRecord, recordDigest: string): string { + // Defensive against an OLDER persisted record (a smaller schemaVersion, loaded straight back out of + // decision_records.record_json — see loadDecisionRecordCollapsible) whose JSON simply predates a field + // this schema version introduced: `?? ` here, not only a `!== null` check, so a genuinely ABSENT key + // (`undefined`, not `null`) degrades the same honest way an explicit null does, instead of throwing on + // `.length`/`.join` of undefined. + const modelIds = record.modelIds ?? null; + const divertedByHoldout = record.divertedByHoldout ?? false; const lines = [ `- **action**: ${record.action} · **clause**: \`${record.reasonCode}\``, `- **config**: \`${record.configDigest}\`${record.gatePack ? ` · **pack**: ${record.gatePack}` : ""}${record.ciState ? ` · **ci**: ${record.ciState}` : ""}`, - ...(record.modelId !== null || record.promptDigest !== null - ? [`- **model**: ${record.modelId ?? "n/a"}${record.promptDigest !== null ? ` · **prompt**: \`${record.promptDigest}\`` : ""}${record.aiConfidence !== null ? ` · **confidence**: ${record.aiConfidence}` : ""}`] + ...(modelIds !== null || record.promptDigest !== null + ? [`- **model**: ${modelIds !== null && modelIds.length > 0 ? modelIds.join("+") : "n/a"}${record.promptDigest !== null ? ` · **prompt**: \`${record.promptDigest}\`` : ""}${record.aiConfidence !== null ? ` · **confidence**: ${record.aiConfidence}` : ""}`] : []), + // #9135: a hold that is really a diverted close must be legible ON THE FACE of the public comment, not + // only inferable by cross-referencing a private audit row. + ...(divertedByHoldout ? ["- **note**: diverted by the randomized close-audit holdout (#8831) — the deterministic pipeline would otherwise have closed this PR"] : []), `- **record**: \`${recordDigest}\` (schema v${record.schemaVersion}, head \`${record.headSha.slice(0, 7)}\`)`, ]; return lines.join("\n"); diff --git a/src/review/decision-replay.ts b/src/review/decision-replay.ts index c3d10b83cd..859e6b43c5 100644 --- a/src/review/decision-replay.ts +++ b/src/review/decision-replay.ts @@ -12,9 +12,22 @@ // 2. `blocker_codes` — ordered blocker code list (order IS meaning: blockerClass is the first code). // 3. `reason_code` — re-derived exactly as the finalize site derives it (blockerClass → // policy_close: → conclusion) vs the PUBLIC record's reason_code. +// 4. `holdout_consistency` — #9135: the PUBLIC record's own `divertedByHoldout` claim vs the PRIVATE +// replay input's recorded `holdout.diverted` outcome. These are written by the same +// call (processors.ts) from the same holdout result, so they can only disagree if +// one was updated without the other — a real bug, not a re-derivation gap. // `action` is reported as PINNED, never re-derived: it depends on plan state (autonomy, holds, approvals) // outside the recorded pipeline — documented v1 scope on #8838. // +// #9135 SCOPE (honest, narrower claim over the broader one #8838 made): this replay input additionally +// records the close-audit holdout draw (the ONE unrecorded nondeterministic input #9135 found), but still +// does NOT capture every decision-time input the live plan consults — namely the precision-breaker flag +// state (`isHoldOnly`/`isCloseHoldOnly`, `system_flags`), the live CI aggregate, and the global +// `isGlobalAgentPause`/`isGlobalAgentFrozen` env switches. Each of those can also change which plan a given +// (findings, policy) pair produces, and none of them are in `DecisionReplayInput` as of this stage. Recording +// them is left to a follow-up widening this same input shape — documented here so the replay contract states +// a narrower, honestly-achievable claim rather than an unqualified broad one. +// // A divergence is a bug BY DEFINITION (the pipeline is supposed to be deterministic): callers exit non-zero // and file it; there is no "close enough" outcome. import { evaluateGateCheck, type GateCheckPolicy } from "../rules/advisory"; @@ -22,6 +35,18 @@ import { neutralHoldReasonCode } from "./parity-wire"; import type { Advisory, AdvisoryFinding } from "../types"; import { errorMessage, nowIso } from "../utils/json"; +/** The close-audit holdout's decision-time outcome (#9135), persisted beside the pipeline's own inputs so a + * hold that was really a diverted close is provable from the replay input, not just claimed by the public + * record. `draw` is the [0,1) value compared against `epsilonPct/100`; it is DERIVED (`HMAC(instance + * secret, seed)`, see close-audit-holdout.ts), never raw entropy, so persisting it here does not weaken the + * "unpredictable to a contributor trying to dodge the holdout" property — recomputing it still requires the + * instance secret, which never leaves the operator's environment. */ +export type DecisionReplayHoldout = { + epsilonPct: number; + draw: number; + diverted: boolean; +}; + /** What decision_replay_inputs.replay_json holds — the pipeline's exact inputs + the decision-time snapshot. */ export type DecisionReplayInput = { findings: AdvisoryFinding[]; @@ -30,6 +55,11 @@ export type DecisionReplayInput = { policyCloseKind?: string | null | undefined; /** Decision-time evaluation snapshot: what the live pipeline produced from these same inputs. */ evaluated: { conclusion: string; blockerCodes: string[] }; + /** #9135: the close-audit holdout's decision-time outcome, when the holdout mechanism actually drew (an + * eligible heuristic close under ε>0 auto autonomy). undefined/null when the holdout never evaluated for + * this decision (ε absent/0, close autonomy not auto, or no eligible close) — the overwhelmingly common + * case, and the SAME zero-I/O common path `maybeApplyCloseAuditHoldout` already guarantees. */ + holdout?: DecisionReplayHoldout | null | undefined; }; /** The slice of the PUBLIC decision record replay verifies against. */ @@ -37,6 +67,10 @@ export type ReplayableRecord = { id: string; reasonCode: string; action: string; + /** #9135: the PUBLIC record's own claim that the close-audit holdout diverted this decision — checked + * against `DecisionReplayInput.holdout.diverted` at the `holdout_consistency` stage. Defaults false for a + * pre-#9135 record (mirrors `DecisionRecord.divertedByHoldout`'s own normalization). */ + divertedByHoldout?: boolean; }; export type ReplayOutcome = @@ -45,7 +79,7 @@ export type ReplayOutcome = verdict: "divergence"; recordId: string; /** The FIRST divergent stage — later stages are downstream of it and not reported. */ - stage: "conclusion" | "blocker_codes" | "reason_code"; + stage: "conclusion" | "blocker_codes" | "reason_code" | "holdout_consistency"; expected: string; actual: string; }; @@ -89,6 +123,21 @@ export function replayDecision(record: ReplayableRecord, input: DecisionReplayIn if (reasonCode !== record.reasonCode) { return { verdict: "divergence", recordId: record.id, stage: "reason_code", expected: record.reasonCode, actual: reasonCode }; } + // #9135 stage 4: the PUBLIC record's `divertedByHoldout` claim and the PRIVATE replay input's own + // `holdout.diverted` outcome are written from the SAME holdout result at the SAME call site — they can + // only disagree if one was updated without the other, which is a real bug (a hold silently mislabeled as + // an ordinary decision, or vice versa), not a re-derivation gap the pipeline could ever legitimately close. + const recordDivertedByHoldout = record.divertedByHoldout ?? false; + const inputDivertedByHoldout = input.holdout?.diverted ?? false; + if (recordDivertedByHoldout !== inputDivertedByHoldout) { + return { + verdict: "divergence", + recordId: record.id, + stage: "holdout_consistency", + expected: String(recordDivertedByHoldout), + actual: String(inputDivertedByHoldout), + }; + } return { verdict: "match", recordId: record.id, conclusion: evaluation.conclusion, blockerCodes, reasonCode, pinnedAction: record.action }; } @@ -102,6 +151,9 @@ export async function persistDecisionReplayInputForGate( recordId: string, gate: { replay?: { findings: AdvisoryFinding[]; policy: GateCheckPolicy } | undefined; conclusion: string; blockers: Array<{ code: string }> }, policyCloseKind: string | null, + // #9135: the close-audit holdout's decision-time outcome for this same decision, when it drew — threaded + // straight through to the persisted replay input so `holdout_consistency` has something to check against. + holdout?: DecisionReplayHoldout | null, ): Promise { if (!gate.replay) return; await persistDecisionReplayInput(env, recordId, { @@ -109,6 +161,7 @@ export async function persistDecisionReplayInputForGate( policy: gate.replay.policy, policyCloseKind, evaluated: { conclusion: gate.conclusion, blockerCodes: gate.blockers.map((blocker) => blocker.code) }, + holdout: holdout ?? null, }); } diff --git a/src/services/agent-action-executor.ts b/src/services/agent-action-executor.ts index 18e5dc4b5c..82e99d1331 100644 --- a/src/services/agent-action-executor.ts +++ b/src/services/agent-action-executor.ts @@ -259,7 +259,10 @@ export type DecisionRecordContext = gatePack?: string | null | undefined; ciState?: string | null | undefined; baseSha?: string | null | undefined; - modelId?: string | null | undefined; + // #9124: renamed from the singular `modelId` -- these call sites (contributor-cap close, review-nag + // close, update_branch) have no AI judgment behind them today, so this is always omitted in practice, + // but the shape matches DecisionRecord.modelIds (the FULL parsed-reviewer set) rather than a single id. + modelIds?: string[] | null | undefined; promptDigest?: string | null | undefined; aiConfidence?: number | null | undefined; salvageability?: { score: number; factors: string[] } | null | undefined; @@ -311,9 +314,13 @@ async function recordCompletedDecision(env: Env, ctx: AgentActionExecutionContex action: action.actionClass, reasonCode: dr.reasonCode ?? defaultDecisionRecordReasonCode(action), configDigest: dr.configDigest, + // #9124: for every call site that reaches this generic path, configDigest already IS the raw resolved- + // settings digest (there is no separate gateCheckPolicy resolution behind a policy close or update_branch) + // -- so settingsDigest is honestly the same value, not a guess. + settingsDigest: dr.configDigest, gatePack: dr.gatePack ?? null, ciState: dr.ciState ?? null, - modelId: dr.modelId ?? null, + modelIds: dr.modelIds ?? null, promptDigest: dr.promptDigest ?? null, aiConfidence: dr.aiConfidence ?? null, salvageability: dr.salvageability ?? null, diff --git a/src/services/ai-review.ts b/src/services/ai-review.ts index f456a40382..d4422989e3 100644 --- a/src/services/ai-review.ts +++ b/src/services/ai-review.ts @@ -31,6 +31,7 @@ import { convergedFeatureActive } from "../review/feature-activation"; import { labelSelfHostReviewerModels, labelSelfHostReviewerNames, resolveConfiguredProviderNames } from "../selfhost/ai-config"; import { incr } from "../selfhost/metrics"; import { errorMessage } from "../utils/json"; +import { sha256Hex } from "../utils/crypto"; import type { ReviewProfile } from "../signals/focus-manifest"; import { isCodeFile } from "../signals/local-branch"; import { isTestPath } from "../signals/test-evidence"; @@ -455,6 +456,12 @@ export type LoopOverAiReviewResult = * review's two opinions combine into one. ADVISORY ONLY, never a gate input. */ valueAssessment: { magnitude: ImprovementMagnitude; rationale: string } | null; reviewDiagnostics?: AiReviewDiagnostic[] | undefined; + /** #9124: sha256 of the ACTUAL `buildSystemPrompt(promptInput)` output for this call — the base + * template plus whichever suffixes actually resolved (grounding/enrichment/profile/security-focus/ + * path-instructions/repo-instructions/screenshot-evidence/inline/category/improvement-signal). Lets a + * caller commit to the real prompt sent instead of the base constant alone (a changed + * `review.instructions` moves this digest). Always present on an "ok" result. */ + systemPromptDigest: string; }; /** A line-anchored review finding the model can emit for quiet inline PR comments (#inline-comments). `line` is @@ -554,6 +561,15 @@ export function formatReviewDiagnosticsForCapture( ); } +/** #9124: the distinct model identities that actually PRODUCED a usable opinion (`status: "parsed"`) — the + * reviewer(s) whose output shaped a consensus-defect/split verdict, as opposed to every model attempted + * (which can include a fallback that never fired, or a provider error). Sorted for a stable, order- + * independent commitment. Empty when nothing parsed (an inconclusive/failed run never reaches a finding that + * would read this). */ +export function parsedReviewModelIds(diagnostics: readonly AiReviewDiagnostic[]): string[] { + return [...new Set(diagnostics.filter((diagnostic) => diagnostic.status === "parsed").map((diagnostic) => diagnostic.model))].sort(); +} + type ReviewerOpinionOutcome = { review: ModelReview | null; fallbackNote?: string | undefined; @@ -2595,6 +2611,9 @@ export async function runLoopOverAiReview( // reviewers are told to verify claims against the attached CI/files; otherwise this is REVIEW_SYSTEM_PROMPT // unchanged (byte-identical). Computed from `promptInput` so it travels with the (possibly defanged) input. const system = buildSystemPrompt(promptInput); + // #9124: commit to the prompt actually sent, not the base constant — see `systemPromptDigest`'s own doc + // comment on `LoopOverAiReviewResult`. + const systemPromptDigest = await sha256Hex(system); const repoInstructionsSystemAppend = buildRepoInstructionsSystemAppend(promptInput.repoInstructions); // The daily neuron budget governs FREE/default-reviewer spend only. BYOK advisory calls bill the maintainer's // own provider account, so they are not counted here (and a BYOK advisory still runs when the free @@ -2948,6 +2967,7 @@ export async function runLoopOverAiReview( inlineFindings, valueAssessment, reviewDiagnostics, + systemPromptDigest, }; } diff --git a/src/types.ts b/src/types.ts index 77ccfea46c..2537051684 100644 --- a/src/types.ts +++ b/src/types.ts @@ -596,6 +596,18 @@ export type AdvisoryFinding = { * no model confidence); an absent/unparseable reviewer confidence degrades to 1.0 upstream, so omitting it here * behaves exactly like an at-or-above-floor confidence. */ confidence?: number; + /** #9124: the distinct provider/model identities (`AiReviewDiagnostic.model`, deduped + sorted) whose + * diagnostic reached `status: "parsed"` for an `ai_consensus_defect` / `ai_review_split` finding — the + * reviewer(s) that actually produced the opinion the finding reports. Carried on the finding (mirroring + * `confidence` immediately above) so the decision-record call site can thread the real model identity into + * `DecisionRecord.modelIds` instead of hardcoding `null`. Absent for every deterministic finding. */ + modelIds?: string[]; + /** #9124: sha256 of the ACTUAL system prompt sent for this AI judgment (`buildSystemPrompt`'s real output — + * base template plus whichever of its up-to-ten suffixes were resolved for this call: grounding, + * enrichment, profile/tone, security-focus, path instructions, `review.instructions`, screenshot evidence, + * inline, category, improvement-signal), NOT a digest of the base constant alone. Two repos with different + * `review.instructions` therefore publish different values here. Absent for every deterministic finding. */ + promptDigest?: string; /** Public-safe screenshot evidence for a `visual_regression_finding` / `visual_unrelated_issue_finding` * (`review.visual.bugAnalysis`) — the SAME shot URLs already rendered in the "Visual preview" collapsible, * carried alongside the finding (not just referenced by route path) so a later consumer — the PR-closed diff --git a/src/utils/crypto.ts b/src/utils/crypto.ts index fbf7c7d004..fb6163b12f 100644 --- a/src/utils/crypto.ts +++ b/src/utils/crypto.ts @@ -3,6 +3,16 @@ export async function sha256Hex(input: string): Promise { return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join(""); } +/** HMAC-SHA256 `value` with `secret`, returned as lowercase hex. Web Crypto (worker + node) — the same + * primitive `src/orb/relay.ts`'s `relaySignature` and `verifyGitHubSignature` below each inline their own + * copy of; a generic export here is for a NEW single-purpose HMAC (e.g. the close-audit holdout draw, + * #9135) that has no reason to take on either of those modules' unrelated concerns. */ +export async function hmacHex(secret: string, value: string): Promise { + const key = await crypto.subtle.importKey("raw", new TextEncoder().encode(secret), { name: "HMAC", hash: "SHA-256" }, false, ["sign"]); + const signature = await crypto.subtle.sign("HMAC", key, new TextEncoder().encode(value)); + return [...new Uint8Array(signature)].map((byte) => byte.toString(16).padStart(2, "0")).join(""); +} + export async function verifyGitHubSignature(rawBody: string, signatureHeader: string | null, secret: string | undefined): Promise { if (!signatureHeader?.startsWith("sha256=")) return false; if (!secret) return false; diff --git a/test/integration/public-decision-ledger-routes.test.ts b/test/integration/public-decision-ledger-routes.test.ts index ffce3e0802..8ad410ed66 100644 --- a/test/integration/public-decision-ledger-routes.test.ts +++ b/test/integration/public-decision-ledger-routes.test.ts @@ -37,7 +37,7 @@ describe("public decision-ledger/decision-records routes answer WITHOUT credenti configDigest: await contentDigest({ gatePack: "oss-anti-slop" }), gatePack: "oss-anti-slop", ciState: null, - modelId: null, + modelIds: null, promptDigest: null, aiConfidence: null, salvageability: null, diff --git a/test/unit/ai-review.test.ts b/test/unit/ai-review.test.ts index 4cac2b7f98..b95b0de8b4 100644 --- a/test/unit/ai-review.test.ts +++ b/test/unit/ai-review.test.ts @@ -548,6 +548,42 @@ describe("review.profile shapes the reviewer system prompt (#review-profile)", ( ); }); + // #9124: `systemPromptDigest` must commit to the ACTUAL system prompt sent (base template + whichever + // suffixes resolved), not the base constant alone — this is what lets the decision record's `promptDigest` + // move when a repo's `review.instructions` changes, instead of publishing the same digest for two + // materially different judges. + it("systemPromptDigest (#9124) is sha256 of the real sent prompt, and moves when repoInstructions changes", async () => { + const runWith = async (repoInstructions: string | undefined) => { + const run = vi.fn(async () => ({ response: reviewJson() })); + const env = createTestEnv({ + AI: { run } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + const result = await runLoopOverAiReview(env, { ...baseInput, repoInstructions }); + const calls = (run as ReturnType).mock.calls; + const system = (calls[0]?.[1] as { messages?: Array<{ content?: string }> })?.messages?.[0]?.content ?? ""; + return { result, system }; + }; + const { sha256Hex } = await import("../../src/utils/crypto"); + + const none = await runWith(undefined); + if (none.result.status !== "ok") throw new Error("expected ok"); + expect(none.result.systemPromptDigest).toBe(await sha256Hex(none.system)); + + const withInstructions = await runWith("Close anything touching src/billing."); + if (withInstructions.result.status !== "ok") throw new Error("expected ok"); + expect(withInstructions.result.systemPromptDigest).toBe(await sha256Hex(withInstructions.system)); + // Two repos with different review.instructions must NOT publish the same commitment. + expect(withInstructions.result.systemPromptDigest).not.toBe(none.result.systemPromptDigest); + + // Same input twice: byte-identical digest (deterministic, not a fresh hash of something incidental). + const again = await runWith("Close anything touching src/billing."); + if (again.result.status !== "ok") throw new Error("expected ok"); + expect(again.result.systemPromptDigest).toBe(withInstructions.result.systemPromptDigest); + }); + it("repoInstructions are passed as systemAppend only for self-host CLI reviewers (#1471)", async () => { const optionsFor = async (model: string, repoInstructions: string | undefined) => { const run = vi.fn(async () => ({ response: reviewJson() })); diff --git a/test/unit/attestation-envelope-engine.test.ts b/test/unit/attestation-envelope-engine.test.ts index b5f1fa51a9..4d6f8445a1 100644 --- a/test/unit/attestation-envelope-engine.test.ts +++ b/test/unit/attestation-envelope-engine.test.ts @@ -12,7 +12,13 @@ import { } from "../../packages/loopover-engine/src/calibration/attestation-envelope.js"; const MEASUREMENT = "a".repeat(64); -const REPORT_DATA = "b".repeat(64); +// #9140: reportData is now 128 hex chars (64 bytes -- SEV-SNP/TDX's REPORT_DATA/REPORTDATA field width), +// with a `runId` freshness field alongside it -- see this module's own doc comment for the finalized layout. +const REPORT_DATA = "b".repeat(128); +const RUN_ID = "d4".repeat(16); +const CORPUS_CHECKSUM = "a1".repeat(32); // 64 hex +const HEAD_SHA = "b2".repeat(20); // 40 hex +const BASE_SHA = "c3".repeat(20); // 40 hex function envelope(overrides: Record = {}): Record { return { @@ -21,6 +27,7 @@ function envelope(overrides: Record = {}): Record { - it("is the lowercase-hex sha256 of corpusChecksum:headSha:baseSha (pinned vector)", () => { - // Precomputed: sha256("abc123:head456:base789"). Pinned so a change to the binding format -- which would - // silently invalidate every previously-attested run -- fails here instead of shipping. - expect(buildAttestationReportData({ corpusChecksum: "abc123", headSha: "head456", baseSha: "base789" })).toBe( - "fcc875115df49bf143b18fc8a8071e9a946858407fabf61e59ef1607d1cfb140", +describe("buildAttestationReportData (#8541, #9140)", () => { + it("is sha256(corpusChecksum:headSha:baseSha) || sha256(runId), 128 lowercase hex chars (pinned vector)", () => { + // Precomputed. Pinned so a change to the binding/layout -- which would silently invalidate every + // previously-attested run -- fails here instead of shipping. Also published verbatim in + // buildAttestationReportData's own doc comment as the spec a non-JS verifier reimplements against. + expect(buildAttestationReportData({ corpusChecksum: CORPUS_CHECKSUM, headSha: HEAD_SHA, baseSha: BASE_SHA, runId: RUN_ID })).toBe( + "3309f8c4eadab7422c8b5ba378a12d52b5676f601faa4dcbac213bae93f5ae7e" + + "1d88ffa7d3cf1f07e5cf64b62016f3e688ad473286f2f613886b6ac02d00541d", ); }); - it("produces exactly 64 lowercase hex chars and is deterministic and field-order sensitive", () => { - const data = buildAttestationReportData({ corpusChecksum: "c", headSha: "h", baseSha: "b" }); - expect(data).toMatch(/^[0-9a-f]{64}$/); - expect(data).toBe(buildAttestationReportData({ corpusChecksum: "c", headSha: "h", baseSha: "b" })); + it("produces exactly 128 lowercase hex chars and is deterministic and field-order sensitive", () => { + const base = { corpusChecksum: CORPUS_CHECKSUM, headSha: HEAD_SHA, baseSha: BASE_SHA, runId: RUN_ID }; + const data = buildAttestationReportData(base); + expect(data).toMatch(/^[0-9a-f]{128}$/); + expect(data).toBe(buildAttestationReportData(base)); // Swapping which value lands in which position must change the digest (the fields are not interchangeable). - expect(data).not.toBe(buildAttestationReportData({ corpusChecksum: "h", headSha: "c", baseSha: "b" })); + expect(data).not.toBe(buildAttestationReportData({ ...base, headSha: BASE_SHA, baseSha: HEAD_SHA })); + }); + + it("is injective over the (corpusChecksum, headSha, baseSha, runId) quadruple", () => { + const base = { corpusChecksum: CORPUS_CHECKSUM, headSha: HEAD_SHA, baseSha: BASE_SHA, runId: RUN_ID }; + const variants = [ + base, + { ...base, corpusChecksum: "a2".repeat(32) }, + { ...base, headSha: "b3".repeat(20) }, + { ...base, baseSha: "c4".repeat(20) }, + { ...base, runId: "d5".repeat(16) }, + { ...base, headSha: "b2".repeat(32) }, // a different valid width (64-hex SHA-256 git object id) + ]; + const digests = variants.map((binding) => buildAttestationReportData(binding)); + expect(new Set(digests).size).toBe(digests.length); + }); + + it("only the second 64 hex chars move when runId changes; only the first 64 move for a binding-only change", () => { + const base = { corpusChecksum: CORPUS_CHECKSUM, headSha: HEAD_SHA, baseSha: BASE_SHA, runId: RUN_ID }; + const baseReportData = buildAttestationReportData(base); + const freshRun = buildAttestationReportData({ ...base, runId: "d5".repeat(16) }); + expect(freshRun.slice(0, 64)).toBe(baseReportData.slice(0, 64)); + expect(freshRun.slice(64)).not.toBe(baseReportData.slice(64)); + + const rebound = buildAttestationReportData({ ...base, corpusChecksum: "a2".repeat(32) }); + expect(rebound.slice(0, 64)).not.toBe(baseReportData.slice(0, 64)); + expect(rebound.slice(64)).toBe(baseReportData.slice(64)); + }); + + it("throws on a malformed input shape instead of silently mis-committing", () => { + const base = { corpusChecksum: CORPUS_CHECKSUM, headSha: HEAD_SHA, baseSha: BASE_SHA, runId: RUN_ID }; + // A colon injected into corpusChecksum can never pass the hex-shape check, closing the ambiguity gap. + expect(() => buildAttestationReportData({ ...base, corpusChecksum: `${CORPUS_CHECKSUM.slice(0, 63)}:` })).toThrow(/corpusChecksum/); + expect(() => buildAttestationReportData({ ...base, corpusChecksum: "abc123" })).toThrow(/corpusChecksum/); + expect(() => buildAttestationReportData({ ...base, headSha: "not-hex-and-wrong-length" })).toThrow(/headSha/); + expect(() => buildAttestationReportData({ ...base, baseSha: "" })).toThrow(/baseSha/); + expect(() => buildAttestationReportData({ ...base, runId: "UPPER" })).toThrow(/runId/); + expect(() => buildAttestationReportData({ ...base, runId: "" })).toThrow(/runId/); }); it("emits output usable as an envelope's reportData", () => { - const reportData = buildAttestationReportData({ corpusChecksum: "c", headSha: "h", baseSha: "b" }); - expect(validateAttestationEnvelope(envelope({ reportData })).valid).toBe(true); + const reportData = buildAttestationReportData({ corpusChecksum: CORPUS_CHECKSUM, headSha: HEAD_SHA, baseSha: BASE_SHA, runId: RUN_ID }); + expect(validateAttestationEnvelope(envelope({ reportData, runId: RUN_ID })).valid).toBe(true); }); }); -describe("validateAttestationEnvelope (#8541)", () => { +describe("validateAttestationEnvelope (#8541, #9140)", () => { it("accepts a well-formed envelope and returns it narrowed", () => { const result = validateAttestationEnvelope(envelope()); expect(result.valid).toBe(true); @@ -110,16 +157,27 @@ describe("validateAttestationEnvelope (#8541)", () => { expectRejected(envelope({ measurement: 64 }), "measurement"); }); - it("requires reportData to be exactly 64 lowercase hex (63 and 65 both rejected)", () => { - expect(validateAttestationEnvelope(envelope({ reportData: "b".repeat(64) })).valid).toBe(true); - expectRejected(envelope({ reportData: "b".repeat(63) }), "reportData"); - expectRejected(envelope({ reportData: "b".repeat(65) }), "reportData"); - expectRejected(envelope({ reportData: "B".repeat(64) }), "reportData"); // uppercase - expectRejected(envelope({ reportData: "z".repeat(64) }), "reportData"); // non-hex + it("requires reportData to be exactly 128 lowercase hex (127 and 129 both rejected)", () => { + expect(validateAttestationEnvelope(envelope({ reportData: "b".repeat(128) })).valid).toBe(true); + expectRejected(envelope({ reportData: "b".repeat(127) }), "reportData"); + expectRejected(envelope({ reportData: "b".repeat(129) }), "reportData"); + expectRejected(envelope({ reportData: "b".repeat(64) }), "reportData"); // #9140: the pre-fix (32-byte) width + expectRejected(envelope({ reportData: "B".repeat(128) }), "reportData"); // uppercase + expectRejected(envelope({ reportData: "z".repeat(128) }), "reportData"); // non-hex expectRejected(envelope({ reportData: null }), "reportData"); }); - it("requires attestationReport to be non-empty base64 within the size cap", () => { + it("#9140: requires runId to be 1..128 lowercase hex", () => { + expect(validateAttestationEnvelope(envelope({ runId: "a" })).valid).toBe(true); + expect(validateAttestationEnvelope(envelope({ runId: "a".repeat(128) })).valid).toBe(true); + expectRejected(envelope({ runId: "a".repeat(129) }), "runId"); + expectRejected(envelope({ runId: "" }), "runId"); + expectRejected(envelope({ runId: "NOTHEX" }), "runId"); + expectRejected(envelope({ runId: undefined }), "runId"); + expectRejected(envelope({ runId: 1 }), "runId"); + }); + + it("requires attestationReport to be non-empty, length-valid base64 within the size cap", () => { expect(validateAttestationEnvelope(envelope({ attestationReport: "QUJD" })).valid).toBe(true); expect(validateAttestationEnvelope(envelope({ attestationReport: "QQ==" })).valid).toBe(true); expect(validateAttestationEnvelope(envelope({ attestationReport: "A".repeat(65536) })).valid).toBe(true); @@ -127,6 +185,11 @@ describe("validateAttestationEnvelope (#8541)", () => { expectRejected(envelope({ attestationReport: "" }), "attestationReport"); expectRejected(envelope({ attestationReport: "not base64!" }), "attestationReport"); expectRejected(envelope({ attestationReport: 1 }), "attestationReport"); + // #9140: a run of valid base64-alphabet characters whose length is NOT a multiple of 4 is not valid + // base64 under any padding rule -- the old regex accepted it anyway. + expectRejected(envelope({ attestationReport: "QUJDQ" }), "attestationReport"); // 5 chars + expectRejected(envelope({ attestationReport: "QUJDQQ" }), "attestationReport"); // 6 chars, unpadded + expect(validateAttestationEnvelope(envelope({ attestationReport: "QUJDQQ==" })).valid).toBe(true); // properly padded }); describe("verification union", () => { @@ -173,10 +236,10 @@ describe("validateAttestationEnvelope (#8541)", () => { it("reports every failing field at once rather than stopping at the first", () => { const errors = expectRejected( - { schemaVersion: 2, teeTechnology: "sgx", runtimeClass: "", measurement: "zz", reportData: "b", attestationReport: "", verification: null }, + { schemaVersion: 2, teeTechnology: "sgx", runtimeClass: "", measurement: "zz", reportData: "b", runId: "", attestationReport: "", verification: null }, "schemaVersion", ); - for (const field of ["teeTechnology", "runtimeClass", "measurement", "reportData", "attestationReport", "verification"]) { + for (const field of ["teeTechnology", "runtimeClass", "measurement", "reportData", "runId", "attestationReport", "verification"]) { expect(errors.some((error) => error.startsWith(field))).toBe(true); } }); diff --git a/test/unit/backfill-decision-labels-core.test.ts b/test/unit/backfill-decision-labels-core.test.ts index d4247604ce..22a272c554 100644 --- a/test/unit/backfill-decision-labels-core.test.ts +++ b/test/unit/backfill-decision-labels-core.test.ts @@ -115,7 +115,7 @@ describe("buildBundle", () => { expect(await contentDigest(record)).toBe(row.record_digest); expect(canonicalJson(record)).toBe(row.record_json); expect(record.configDigest).toBe("backfill:unavailable"); - expect(record.schemaVersion).toBe("3"); // v3 (#8962): + salvageability + expect(record.schemaVersion).toBe("4"); // v4 (#9124/#9135) — bumped past the v3 salvageability shape this test targets } expect(records[0]).toMatchObject({ id: "record:o/r#1@sha1", action: "close", created_at: "2026-07-01T00:00:00.000Z" }); expect(records[1]).toMatchObject({ action: "hold" }); diff --git a/test/unit/close-audit-holdout.test.ts b/test/unit/close-audit-holdout.test.ts index da8e53efd0..e427972531 100644 --- a/test/unit/close-audit-holdout.test.ts +++ b/test/unit/close-audit-holdout.test.ts @@ -1,11 +1,14 @@ import { describe, expect, it, vi } from "vitest"; -import { applyCloseAuditHoldout, holdoutEligibleClose, maybeApplyCloseAuditHoldout } from "../../src/review/close-audit-holdout"; +import { applyCloseAuditHoldout, hmacHexToUnitFloat, holdoutEligibleClose, maybeApplyCloseAuditHoldout } from "../../src/review/close-audit-holdout"; import type { PlannedAgentAction } from "../../src/settings/agent-actions"; import { createTestEnv } from "../helpers/d1"; // #8831: the selective-labels fix. The invariants that make the instrument honest are pinned here: the draw // consumes the FINAL plan, ε=0 is byte-identical, only heuristic auto-closes are eligible, and a hold whose // propensity record failed to write NEVER happens (an unlogged hold would silently bias every estimator). +// #9135: the draw is now HMAC-derived (reproducible from the record, unpredictable without the instance +// secret) rather than Math.random(), and every call reports its decision-time holdout outcome for the +// caller to persist onto the decision record and its replay input. function close(over: Partial = {}): PlannedAgentAction { return { actionClass: "close", closeKind: "heuristic", requiresApproval: false, reason: "ci failing", ...over } as PlannedAgentAction; } @@ -37,25 +40,34 @@ describe("applyCloseAuditHoldout (pure transform)", () => { }); }); +describe("hmacHexToUnitFloat", () => { + it("maps the first 4 bytes of a hex digest to a [0,1) float, at the known endpoints", () => { + expect(hmacHexToUnitFloat("00000000" + "ff".repeat(28))).toBe(0); + expect(hmacHexToUnitFloat("ffffffff" + "00".repeat(28))).toBe(0.9999999997671694); + expect(hmacHexToUnitFloat("80000000" + "00".repeat(28))).toBe(0.5); + }); +}); + describe("maybeApplyCloseAuditHoldout (the full step)", () => { const planWithClose = () => [close()]; - it("ε=0 / absent / non-auto autonomy / no eligible close → plan returned untouched with ZERO writes", async () => { + it("ε=0 / absent / non-auto autonomy / no eligible close → plan returned untouched with ZERO writes, holdout: null", async () => { const env = createTestEnv(); const spy = vi.spyOn(env.DB, "prepare"); - const base = { repoFullName: "o/r", pullNumber: 7, labelSettings: {}, closeAutonomyIsAuto: true }; - expect(await maybeApplyCloseAuditHoldout(env, { ...base, planned: planWithClose(), epsilonPct: 0 })).toEqual(planWithClose()); - expect(await maybeApplyCloseAuditHoldout(env, { ...base, planned: planWithClose(), epsilonPct: undefined })).toEqual(planWithClose()); - expect(await maybeApplyCloseAuditHoldout(env, { ...base, planned: planWithClose(), epsilonPct: 5, closeAutonomyIsAuto: false })).toEqual(planWithClose()); - expect(await maybeApplyCloseAuditHoldout(env, { ...base, planned: [label("x")], epsilonPct: 5 })).toEqual([label("x")]); + const base = { repoFullName: "o/r", pullNumber: 7, headSha: "abc123", labelSettings: {}, closeAutonomyIsAuto: true }; + expect(await maybeApplyCloseAuditHoldout(env, { ...base, planned: planWithClose(), epsilonPct: 0 })).toEqual({ planned: planWithClose(), holdout: null }); + expect(await maybeApplyCloseAuditHoldout(env, { ...base, planned: planWithClose(), epsilonPct: undefined })).toEqual({ planned: planWithClose(), holdout: null }); + expect(await maybeApplyCloseAuditHoldout(env, { ...base, planned: planWithClose(), epsilonPct: 5, closeAutonomyIsAuto: false })).toEqual({ planned: planWithClose(), holdout: null }); + expect(await maybeApplyCloseAuditHoldout(env, { ...base, planned: [label("x")], epsilonPct: 5 })).toEqual({ planned: [label("x")], holdout: null }); expect(spy).not.toHaveBeenCalled(); }); - it("a draw under ε diverts the close, logs the propensity record, and files the pending holdout label", async () => { + it("a draw under ε diverts the close, logs the propensity record, files the pending holdout label, and reports diverted:true", async () => { const env = createTestEnv(); - const next = await maybeApplyCloseAuditHoldout(env, { + const { planned: next, holdout } = await maybeApplyCloseAuditHoldout(env, { repoFullName: "o/r", pullNumber: 7, + headSha: "abc123", planned: planWithClose(), epsilonPct: 5, closeAutonomyIsAuto: true, @@ -63,6 +75,7 @@ describe("maybeApplyCloseAuditHoldout (the full step)", () => { rng: () => 0.01, // 1% < 5% }); expect(next.some((a) => a.actionClass === "close")).toBe(false); + expect(holdout).toEqual({ epsilonPct: 5, draw: 0.01, diverted: true }); const audit = await env.DB.prepare("SELECT metadata_json FROM audit_events WHERE event_type = 'decision_audit_holdout' AND target_key = 'o/r#7'").first<{ metadata_json: string }>(); const metadata = JSON.parse(audit!.metadata_json) as Record; expect(metadata).toMatchObject({ epsilonPct: 5, draw: 0.01, counterfactualAction: "close" }); @@ -70,34 +83,92 @@ describe("maybeApplyCloseAuditHoldout (the full step)", () => { expect(row).toEqual({ verdict: "close", outcome: null, stratum: "holdout_close", status: "pending" }); }); - it("a draw at/above ε lets the close proceed and records nothing", async () => { + it("a draw at/above ε lets the close proceed, records nothing, and reports diverted:false", async () => { const env = createTestEnv(); - const next = await maybeApplyCloseAuditHoldout(env, { + const { planned: next, holdout } = await maybeApplyCloseAuditHoldout(env, { repoFullName: "o/r", pullNumber: 7, + headSha: "abc123", planned: planWithClose(), epsilonPct: 5, closeAutonomyIsAuto: true, rng: () => 0.05, // exactly ε — the held region is [0, ε), so this proceeds }); expect(next.some((a) => a.actionClass === "close")).toBe(true); + expect(holdout).toEqual({ epsilonPct: 5, draw: 0.05, diverted: false }); const n = await env.DB.prepare("SELECT COUNT(*) AS n FROM decision_audit_labels").first<{ n: number }>(); expect(n!.n).toBe(0); }); - it("uses the real RNG when none is injected (the production arm) — ε=100 fires deterministically", async () => { + it("uses the deterministic HMAC-derived draw when none is injected — reproducible for the same (repo, pr, head sha), and ε=100 always diverts", async () => { const env = createTestEnv(); - const next = await maybeApplyCloseAuditHoldout(env, { - repoFullName: "o/r", - pullNumber: 8, + const call = () => + maybeApplyCloseAuditHoldout(env, { + repoFullName: "o/r", + pullNumber: 8, + headSha: "deadbeef", + planned: planWithClose(), + epsilonPct: 100 as never, // parse clamps real config to 20; the module itself only compares + closeAutonomyIsAuto: true, + }); + const first = await call(); + expect(first.planned.some((a) => a.actionClass === "close")).toBe(false); // any draw in [0,1) < 1.0 + expect(first.holdout?.diverted).toBe(true); + + // A SECOND, independent decision for the SAME (repo, pr, head sha) must reproduce the IDENTICAL draw — + // the whole point of deriving it from a recorded seed instead of raw entropy. Use a tiny ε so the test + // actually distinguishes "same draw" from "any draw" (both would divert at ε=100). + const secondEnv = createTestEnv(); + const a = await maybeApplyCloseAuditHoldout(secondEnv, { + repoFullName: "o/r2", + pullNumber: 9, + headSha: "cafef00d", + planned: planWithClose(), + epsilonPct: 100, + closeAutonomyIsAuto: true, + }); + const b = await maybeApplyCloseAuditHoldout(secondEnv, { + repoFullName: "o/r2", + pullNumber: 9, + headSha: "cafef00d", + planned: planWithClose(), + epsilonPct: 100, + closeAutonomyIsAuto: true, + }); + expect(a.holdout?.draw).toBe(b.holdout?.draw); + + // A DIFFERENT head sha for the same repo/PR must NOT reliably reproduce the same draw (different seed). + const c = await maybeApplyCloseAuditHoldout(secondEnv, { + repoFullName: "o/r2", + pullNumber: 9, + headSha: "different-head", planned: planWithClose(), - epsilonPct: 100 as never, // parse clamps real config to 20; the module itself only compares + epsilonPct: 100, closeAutonomyIsAuto: true, }); - expect(next.some((a) => a.actionClass === "close")).toBe(false); // Math.random() < 1.0 always + expect(c.holdout?.draw).not.toBe(a.holdout?.draw); + }); + + it("an absent headSha still derives a deterministic draw (seed falls back to 'unknown')", async () => { + const env = createTestEnv(); + const call = () => maybeApplyCloseAuditHoldout(env, { repoFullName: "o/r3", pullNumber: 5, planned: planWithClose(), epsilonPct: 100, closeAutonomyIsAuto: true }); + const first = await call(); + const second = await call(); + expect(first.holdout?.draw).toBe(second.holdout?.draw); + }); + + it("persists ONE dedicated secret across calls (system_flags), race-safe across a concurrent first write", async () => { + const env = createTestEnv(); + await Promise.all([ + maybeApplyCloseAuditHoldout(env, { repoFullName: "o/a", pullNumber: 1, headSha: "h1", planned: planWithClose(), epsilonPct: 100, closeAutonomyIsAuto: true }), + maybeApplyCloseAuditHoldout(env, { repoFullName: "o/b", pullNumber: 2, headSha: "h2", planned: planWithClose(), epsilonPct: 100, closeAutonomyIsAuto: true }), + ]); + const rows = await env.DB.prepare("SELECT value FROM system_flags WHERE key = 'close_audit_holdout:secret'").all<{ value: string }>(); + expect(rows.results).toHaveLength(1); + expect(rows.results![0]!.value).toMatch(/^[0-9a-f]{64}$/); }); - it("HARD ORDERING RULE: if the propensity record fails to write, the close PROCEEDS unheld", async () => { + it("HARD ORDERING RULE: if the propensity record fails to write, the close PROCEEDS unheld and holdout.diverted is false", async () => { const env = createTestEnv(); const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); const realPrepare = env.DB.prepare.bind(env.DB); @@ -105,15 +176,17 @@ describe("maybeApplyCloseAuditHoldout (the full step)", () => { if (sql.includes("INSERT INTO audit_events") || sql.includes("insert into \"audit_events\"")) throw new Error("ledger down"); return realPrepare(sql); }); - const next = await maybeApplyCloseAuditHoldout(env, { + const { planned: next, holdout } = await maybeApplyCloseAuditHoldout(env, { repoFullName: "o/r", pullNumber: 7, + headSha: "abc123", planned: planWithClose(), epsilonPct: 5, closeAutonomyIsAuto: true, rng: () => 0.0, }); expect(next.some((a) => a.actionClass === "close")).toBe(true); // the instrument degrades, never the gate + expect(holdout).toEqual({ epsilonPct: 5, draw: 0, diverted: false }); expect(warn).toHaveBeenCalled(); vi.restoreAllMocks(); const rows = await env.DB.prepare("SELECT COUNT(*) AS n FROM decision_audit_labels").first<{ n: number }>(); diff --git a/test/unit/crypto.test.ts b/test/unit/crypto.test.ts index 9261bf5a84..0bdde1ea55 100644 --- a/test/unit/crypto.test.ts +++ b/test/unit/crypto.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { createOpaqueToken, hashToken, timingSafeEqual } from "../../src/auth/security"; -import { verifyGitHubSignature, timingSafeEqualHex } from "../../src/utils/crypto"; +import { hmacHex, verifyGitHubSignature, timingSafeEqualHex } from "../../src/utils/crypto"; describe("webhook signature verification", () => { it("accepts valid GitHub HMAC signatures and rejects tampering", async () => { @@ -30,6 +30,14 @@ describe("webhook signature verification", () => { expect(timingSafeEqualHex("ABCD", "abcd")).toBe(true); }); + it("hmacHex is deterministic per (secret, value) and diverges whenever either input differs (#9135)", async () => { + const a = await hmacHex("s1", "v1"); + expect(a).toBe(await hmacHex("s1", "v1")); + expect(a).toMatch(/^[0-9a-f]{64}$/); + expect(a).not.toBe(await hmacHex("s1", "v2")); + expect(a).not.toBe(await hmacHex("s2", "v1")); + }); + it("uses timing-safe token comparisons and one-way token hashes", async () => { await expect(timingSafeEqual("token-a", "token-a")).resolves.toBe(true); await expect(timingSafeEqual("token-a", "token-b")).resolves.toBe(false); diff --git a/test/unit/decision-record.test.ts b/test/unit/decision-record.test.ts index 0e2ee980c8..c45bdc4548 100644 --- a/test/unit/decision-record.test.ts +++ b/test/unit/decision-record.test.ts @@ -57,11 +57,13 @@ function recordInput(over: Partial = {}): Omit { expect(typeof record.decidedAt).toBe("string"); expect(recordDigest).toBe(await contentDigest(record)); // Call sites pass optional-shaped settings fields raw; normalization happens HERE, once. - const { record: bare } = await buildDecisionRecord({ ...recordInput(), gatePack: undefined, ciState: undefined, baseSha: undefined, aiConfidence: undefined }); + const { record: bare } = await buildDecisionRecord({ + ...recordInput(), + gatePack: undefined, + ciState: undefined, + baseSha: undefined, + aiConfidence: undefined, + settingsDigest: undefined, + divertedByHoldout: undefined, + }); expect(bare.gatePack).toBeNull(); expect(bare.ciState).toBeNull(); expect(bare.baseSha).toBeNull(); expect(bare.aiConfidence).toBeNull(); + // #9124/#9135: the two newest optional-normalized fields default the same way — null/false, not undefined. + expect(bare.settingsDigest).toBeNull(); + expect(bare.divertedByHoldout).toBe(false); // #8834: a stated confidence (including explicit 0) survives normalization. const { record: withConf } = await buildDecisionRecord({ ...recordInput(), aiConfidence: 0 }); expect(withConf.aiConfidence).toBe(0); + // #9135: an explicit true is never coerced back to the false default. + const { record: diverted } = await buildDecisionRecord({ ...recordInput(), divertedByHoldout: true }); + expect(diverted.divertedByHoldout).toBe(true); }); it("a re-persist at the SAME (repo, pull, head) is a NEW revisioned row, never an overwrite (#9123)", async () => { @@ -184,7 +200,7 @@ describe("renderDecisionRecordSection", () => { // The head sha keeps its conventional 7-char git-abbreviation — a display convention, not a digest. expect(body).toContain(`\`${record.headSha.slice(0, 7)}\``); - const ai = await buildDecisionRecord(recordInput({ modelId: "claude-sonnet-5", promptDigest: "p".repeat(64), aiConfidence: 0.97 })); + const ai = await buildDecisionRecord(recordInput({ modelIds: ["claude-sonnet-5"], promptDigest: "p".repeat(64), aiConfidence: 0.97 })); const aiBody = renderDecisionRecordSection(ai.record, ai.recordDigest); expect(aiBody).toContain("**model**: claude-sonnet-5"); expect(aiBody).toContain(`\`${"p".repeat(64)}\``); @@ -192,18 +208,40 @@ describe("renderDecisionRecordSection", () => { // Bounded: a record section must stay a small fixed-size block even with three full 64-hex digests inline. expect(aiBody.length).toBeLessThan(900); + // #9124: more than one parsed reviewer joins with "+" — the full set, never a representative one. + const dual = await buildDecisionRecord(recordInput({ modelIds: ["claude-code", "codex"], promptDigest: "p".repeat(64), aiConfidence: 0.8 })); + expect(renderDecisionRecordSection(dual.record, dual.recordDigest)).toContain("**model**: claude-code+codex"); + // Null pack/ci render nothing for those segments; a prompt digest without a model id renders "n/a". - const bare = await buildDecisionRecord(recordInput({ gatePack: null, ciState: null, modelId: null, promptDigest: "q".repeat(64) })); + const bare = await buildDecisionRecord(recordInput({ gatePack: null, ciState: null, modelIds: null, promptDigest: "q".repeat(64) })); const bareBody = renderDecisionRecordSection(bare.record, bare.recordDigest); expect(bareBody).not.toContain("**pack**"); expect(bareBody).not.toContain("**ci**"); expect(bareBody).toContain("**model**: n/a"); expect(bareBody).toContain(`\`${"q".repeat(64)}\``); // Model id present with NO prompt digest: the model line renders without a prompt segment. - const modelOnly = await buildDecisionRecord(recordInput({ modelId: "claude-sonnet-5" })); + const modelOnly = await buildDecisionRecord(recordInput({ modelIds: ["claude-sonnet-5"] })); expect(renderDecisionRecordSection(modelOnly.record, modelOnly.recordDigest)).toContain("**model**: claude-sonnet-5"); expect(renderDecisionRecordSection(modelOnly.record, modelOnly.recordDigest)).not.toContain("**prompt**"); }); + + it("#9135: a diverted-by-holdout decision surfaces the note on the record's own face", async () => { + const notDiverted = await buildDecisionRecord(recordInput()); + expect(renderDecisionRecordSection(notDiverted.record, notDiverted.recordDigest)).not.toContain("**note**"); + const diverted = await buildDecisionRecord(recordInput({ action: "hold", divertedByHoldout: true })); + const body = renderDecisionRecordSection(diverted.record, diverted.recordDigest); + expect(body).toContain("**note**"); + expect(body).toContain("close-audit holdout"); + }); + + it("defends against a genuinely ABSENT (not merely null) field from an older persisted record's JSON", async () => { + const { record, recordDigest } = await buildDecisionRecord(recordInput()); + // Simulate a pre-#9135 stored record.json: divertedByHoldout was never a key at all, not present-as-null. + const preExisting = { ...record } as Partial; + delete preExisting.divertedByHoldout; + const body = renderDecisionRecordSection(preExisting as DecisionRecord, recordDigest); + expect(body).not.toContain("**note**"); // degrades to false, same as an explicit false + }); }); describe("loadDecisionRecordCollapsible", () => { diff --git a/test/unit/decision-replay.test.ts b/test/unit/decision-replay.test.ts index 992d54cb1c..4cb1b5c722 100644 --- a/test/unit/decision-replay.test.ts +++ b/test/unit/decision-replay.test.ts @@ -56,6 +56,67 @@ describe("decision replay (#8838)", () => { expect(outcome).toMatchObject({ verdict: "divergence", stage: "reason_code", expected: "policy_close:doctored", actual: "policy_close:stale_superseded" }); }); + // #9135: the PUBLIC record's `divertedByHoldout` claim and the PRIVATE replay input's own `holdout.diverted` + // outcome are written from the SAME holdout result at the SAME call site (processors.ts) — they can only + // disagree if one was updated without the other, a real bug the pipeline's own re-derivation could never + // catch (the holdout sits entirely outside evaluateGateCheck). + it("stage 4 — holdout_consistency: an unexplained divergence between the record's claim and the replay input's own account is reported, not silently matched", () => { + const [, bundle] = bundles.find(([name]) => name === "ai-consensus-close.json")!; + // Case A: the record says diverted, but the replay input recorded no diversion (or none at all) — a hold + // that was silently mislabeled as an ordinary decision, or the reverse. + const recordClaimsDiverted = { ...replayable(bundle), divertedByHoldout: true }; + const inputSaysNotDiverted = { ...bundle.replayInput, holdout: { epsilonPct: 5, draw: 0.9, diverted: false } }; + expect(replayDecision(recordClaimsDiverted, inputSaysNotDiverted)).toMatchObject({ + verdict: "divergence", + stage: "holdout_consistency", + expected: "true", + actual: "false", + }); + // No holdout recorded at all (undefined) normalizes to false — same divergence. + expect(replayDecision(recordClaimsDiverted, { ...bundle.replayInput, holdout: undefined })).toMatchObject({ + verdict: "divergence", + stage: "holdout_consistency", + }); + + // Case B: the reverse — the record does NOT claim a diversion, but the replay input says it diverted. + const recordClaimsNotDiverted = { ...replayable(bundle), divertedByHoldout: false }; + const inputSaysDiverted = { ...bundle.replayInput, holdout: { epsilonPct: 5, draw: 0.02, diverted: true } }; + expect(replayDecision(recordClaimsNotDiverted, inputSaysDiverted)).toMatchObject({ + verdict: "divergence", + stage: "holdout_consistency", + expected: "false", + actual: "true", + }); + + // Consistent on both sides (including the common absent/absent case, which defaults false on both): MATCH. + expect(replayDecision(recordClaimsNotDiverted, { ...bundle.replayInput, holdout: null }).verdict).toBe("match"); + expect(replayDecision({ ...replayable(bundle) }, bundle.replayInput).verdict).toBe("match"); // divertedByHoldout absent on the record too + expect(replayDecision(recordClaimsDiverted, inputSaysDiverted).verdict).toBe("match"); + }); + + it("persistDecisionReplayInputForGate (#9135): the holdout outcome rides along when supplied, and defaults null otherwise", async () => { + const env = createTestEnv(); + const input = bundles[0]![1].replayInput; + await persistDecisionReplayInputForGate( + env, + "record:o/r#20@s", + { conclusion: "failure", blockers: [{ code: "secret_leak" }], replay: { findings: input.findings, policy: input.policy } }, + "kind", + { epsilonPct: 5, draw: 0.9, diverted: false }, + ); + const withHoldout = await env.DB.prepare("SELECT replay_json FROM decision_replay_inputs WHERE record_id = 'record:o/r#20@s'").first<{ replay_json: string }>(); + expect((JSON.parse(withHoldout!.replay_json) as DecisionReplayInput).holdout).toEqual({ epsilonPct: 5, draw: 0.9, diverted: false }); + + await persistDecisionReplayInputForGate( + env, + "record:o/r#21@s", + { conclusion: "failure", blockers: [{ code: "secret_leak" }], replay: { findings: input.findings, policy: input.policy } }, + "kind", + ); + const withoutHoldout = await env.DB.prepare("SELECT replay_json FROM decision_replay_inputs WHERE record_id = 'record:o/r#21@s'").first<{ replay_json: string }>(); + expect((JSON.parse(withoutHoldout!.replay_json) as DecisionReplayInput).holdout).toBeNull(); + }); + it("deriveDecisionReasonCode: blockerClass beats policy_close beats conclusion — the finalize site's exact mapping", () => { expect(deriveDecisionReasonCode("secret_leak", "stale", "failure")).toBe("secret_leak"); expect(deriveDecisionReasonCode("none", "stale", "failure")).toBe("policy_close:stale"); diff --git a/test/unit/gate-check-policy.test.ts b/test/unit/gate-check-policy.test.ts index 033de574f1..e9d3690960 100644 --- a/test/unit/gate-check-policy.test.ts +++ b/test/unit/gate-check-policy.test.ts @@ -8,6 +8,7 @@ import { evaluateGateCheck } from "../../src/rules/advisory"; import { REVIEW_THREAD_BLOCKER_CODE } from "../../src/review/review-thread-findings"; import { parseFocusManifest, resolveEffectiveSettings } from "../../src/signals/focus-manifest"; import { upsertRepoFocusManifest } from "../../src/signals/focus-manifest-loader"; +import { contentDigest } from "../../src/review/decision-record"; import type { Advisory, PullRequestRecord, RepositorySettings } from "../../src/types"; function settings(over: Partial = {}): RepositorySettings { @@ -47,6 +48,38 @@ describe("gateCheckPolicy — #8176 global close-confidence default-override", ( }); }); +// #9124: `configDigest` at the decision-record call site (src/queue/processors.ts) digests +// `{ policy: gate.replay?.policy ?? settings, untrustworthyRuleCodes }` — the RESOLVED gateCheckPolicy +// output, not raw settings. Pinned here against the exact Monday/Tuesday scenario the issue described: a PR +// closed under one calibrated λ̂ and one held under a different λ̂ must publish DIFFERENT configDigests even +// though raw `settings` (and therefore `settingsDigest`) is byte-identical between the two. +describe("gateCheckPolicy resolved output feeds configDigest (#9124)", () => { + it("a changed calibrated close floor moves the resolved-policy digest even though raw settings is unchanged", async () => { + const raw = settings(); // no explicit gate.aiReview.closeConfidence — the default path #9124 describes + const monday = gateCheckPolicy(raw, null, undefined, null, undefined, undefined, { value: 0.93, calibrated: true }); + const tuesday = gateCheckPolicy(raw, null, undefined, null, undefined, undefined, { value: 0.97, calibrated: true }); + const untrustworthyRuleCodes: string[] = []; + const mondayDigest = await contentDigest({ policy: monday, untrustworthyRuleCodes }); + const tuesdayDigest = await contentDigest({ policy: tuesday, untrustworthyRuleCodes }); + expect(mondayDigest).not.toBe(tuesdayDigest); + // Raw settings — and therefore settingsDigest — is byte-identical between the two: this is exactly the + // gap #9124 fixed (before, configDigest was `contentDigest(settings)` alone, so it could not see this). + expect(await contentDigest(raw)).toBe(await contentDigest(raw)); + + // The untrustworthy-rule-code set the precision breaker consults is folded in too — a repo-agnostic + // input the resolved GateCheckPolicy object itself does not carry. + const sameFloorDifferentRuleCodes = await contentDigest({ policy: monday, untrustworthyRuleCodes: ["ai_consensus_defect"] }); + expect(sameFloorDifferentRuleCodes).not.toBe(mondayDigest); + }); + + it("byte-identical resolved policy + untrustworthy codes reproduce the identical digest", async () => { + const raw = settings({ aiReviewCloseConfidence: 0.9 }); + const a = gateCheckPolicy(raw, null, undefined, null, undefined, undefined, null); + const b = gateCheckPolicy(raw, null, undefined, null, undefined, undefined, null); + expect(await contentDigest({ policy: a, untrustworthyRuleCodes: ["x"] })).toBe(await contentDigest({ policy: b, untrustworthyRuleCodes: ["x"] })); + }); +}); + function missingIssueAdvisory(): Advisory { return { id: "advisory-policy", diff --git a/test/unit/queue-2.test.ts b/test/unit/queue-2.test.ts index 824a1250b6..0fed47bf4b 100644 --- a/test/unit/queue-2.test.ts +++ b/test/unit/queue-2.test.ts @@ -366,10 +366,10 @@ describe("queue processors", () => { expect(closeAudit?.n).toBe(0); const pr18 = await getPullRequest(env, "owner/agent-repo", 18); expect(pr18?.state).toBe("open"); - // The v3 decision record carries the boundary evidence. + // The decision record carries the boundary evidence. const record = await env.DB.prepare("select record_json from decision_records where repo_full_name = ? and pull_number = ?").bind("owner/agent-repo", 18).first<{ record_json: string }>(); const parsedRecord = JSON.parse(record!.record_json) as { schemaVersion: string; salvageability: { score: number; factors: string[] } | null }; - expect(parsedRecord.schemaVersion).toBe("3"); + expect(parsedRecord.schemaVersion).toBe("4"); // v4 (#9124/#9135) — bumped past the v3 salvageability shape this test targets expect(parsedRecord.salvageability?.score).toBe(70); expect(parsedRecord.salvageability?.factors.join(" ")).toContain("mechanical defect class"); // #8838: the replay input persisted beside the record, and the decision re-derives bit-exactly from it. @@ -381,6 +381,151 @@ describe("queue processors", () => { expect(outcome.verdict).toBe("match"); }); + it("#9124: the decision record threads the finding's real modelIds/promptDigest, the live CI aggregate, and both digests — never the hardcoded nulls the pre-fix record shipped", async () => { + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async () => ({ response: JSON.stringify({ assessment: "Looks fine.", blockers: [], nits: [], suggestions: [] }) }) } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + await upsertInstallation(env, { action: "created", installation: { id: 9001, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: {}, events: [] } }); + await upsertRepositoryFromGitHub(env, { name: "agent-repo", full_name: "owner/agent-repo", private: false, owner: { login: "owner" } }, 9001); + await upsertRepositorySettings(env, { repoFullName: "owner/agent-repo", autonomy: { close: "auto" }, gatePack: "oss-anti-slop" }); + await upsertRepoFocusManifest(env, "owner/agent-repo", { settings: { checkRunMode: "off", commentMode: "off", publicSurface: "off", reviewCheckMode: "required", aiReviewMode: "block" } }); + await upsertPullRequestFromGitHub(env, "owner/agent-repo", { number: 19, title: "Real defect PR", state: "open", user: { login: "contributor" }, head: { sha: "d19" }, labels: [], body: "Closes #1" }); + await upsertPullRequestFile(env, { repoFullName: "owner/agent-repo", pullNumber: 19, path: "src/a.ts", status: "modified", additions: 1, deletions: 0, changes: 1, payload: { patch: "@@\n+export const ok = value.length;" } }); + const inputFingerprint = await cachedSubFloorDefectFingerprint("Real defect PR"); + // #9124: the cached finding carries modelIds/promptDigest exactly as ai-review-orchestration.ts now + // attaches them to a FRESH ai_consensus_defect finding (parsedReviewModelIds + systemPromptDigest) — the + // cache is a faithful stand-in for that shape, so this proves the processors.ts wiring reads them + // straight through rather than re-deriving or dropping them. + await putCachedAiReview(env, "owner/agent-repo", 19, "d19", "block", { + notes: "cached review", + reviewerCount: 2, + // 0.95 sits ABOVE the default 0.93 floor: no salvageability hold, so this one-shot-closes cleanly. + findings: [{ code: "ai_consensus_defect", severity: "critical", title: "Unused import", detail: "unused import join from node:path is dead code.", confidence: 0.95, modelIds: ["claude-code", "codex"], promptDigest: "p".repeat(64) }], + metadata: { inputFingerprint }, + }); + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/pulls/19/files")) return Response.json([{ filename: "src/a.ts", status: "modified", additions: 1, deletions: 0, changes: 1, patch: "@@\n+export const ok = value.length;" }]); + if (url.endsWith("/pulls/19") && init?.method === "PATCH") return Response.json({ number: 19, state: "closed" }); + if (url.endsWith("/pulls/19")) return Response.json({ number: 19, title: "Real defect PR", state: "open", user: { login: "contributor" }, head: { sha: "d19" }, labels: [], body: "Closes #1" }); + if (url.includes("/commits/d19/check-runs")) return Response.json({ total_count: 0, check_runs: [] }); + if (url.includes("/commits/d19/status")) return Response.json({ state: "success", statuses: [] }); + if (url.endsWith("/pulls/19/reviews") && init?.method === "POST") return Response.json({ id: 1 }); + if (url.endsWith("/pulls/19/reviews")) return Response.json([]); + if (url.includes("/issues/1")) return Response.json({ number: 1, title: "Issue", state: "open", labels: [], user: { login: "reporter" } }); + if (url.includes("/branches/")) return Response.json({ protected: false, protection: { required_status_checks: { contexts: [] } } }); + return Response.json({}); + }); + vi.setSystemTime(new Date("2026-05-28T02:00:00.000Z")); + + await sweepAndDrainPerPr(env, "owner/agent-repo"); + + const record = await env.DB.prepare("select record_json from decision_records where repo_full_name = ? and pull_number = ?").bind("owner/agent-repo", 19).first<{ record_json: string }>(); + const parsed = JSON.parse(record!.record_json) as { + schemaVersion: string; + action: string; + modelIds: string[] | null; + promptDigest: string | null; + ciState: string | null; + configDigest: string; + settingsDigest: string | null; + divertedByHoldout: boolean; + }; + expect(parsed.schemaVersion).toBe("4"); + expect(parsed.action).toBe("close"); // above-floor confidence, no salvageability config -> a one-shot close DECISION + // The exact requirement: modelId is non-null whenever an AI judgment shaped the decision. + expect(parsed.modelIds).toEqual(["claude-code", "codex"]); + expect(parsed.promptDigest).toBe("p".repeat(64)); + // #9124: ciState is now populated from the live CI aggregate in scope at the call site, never the old + // hardcoded null — the exact resolved value depends on the mocked check-run/status shape, which is not + // this test's concern (see gate-outcomes-focused tests for that mapping). + expect(parsed.ciState).not.toBeNull(); + // Both digests are real 64-hex sha256 values, and distinct fields (configDigest != settingsDigest is not + // asserted here since they may coincide for a simple policy -- what matters is both are populated). + expect(parsed.configDigest).toMatch(/^[0-9a-f]{64}$/); + expect(parsed.settingsDigest).toMatch(/^[0-9a-f]{64}$/); + // No closeAuditHoldoutPct configured -> the holdout never draws -> never diverted. + expect(parsed.divertedByHoldout).toBe(false); + }); + + it("#9135: with gate.closeAuditHoldoutPct set, a would-close is diverted to a hold and the decision record + replay input both say so", async () => { + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async () => ({ response: JSON.stringify({ assessment: "Looks fine.", blockers: [], nits: [], suggestions: [] }) }) } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "100000", + }); + // The production draw is HMAC(instance secret, seed) — deterministic given a KNOWN secret. Pre-seed the + // instrument's dedicated system_flags secret so the draw for this test's exact seed + // (record:owner/agent-repo#20@d20) is known ahead of time (~0.097), rather than depending on a randomly + // generated secret and hoping it lands under the threshold. + await env.DB.prepare("INSERT INTO system_flags (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)") + .bind("close_audit_holdout:secret", "e9dca8960f9c80ba4e290e5f0947dea5d1d0153703ab389d3bd995706a979a73") + .run(); + await upsertInstallation(env, { action: "created", installation: { id: 9001, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: {}, events: [] } }); + await upsertRepositoryFromGitHub(env, { name: "agent-repo", full_name: "owner/agent-repo", private: false, owner: { login: "owner" } }, 9001); + await upsertRepositorySettings(env, { repoFullName: "owner/agent-repo", autonomy: { close: "auto" }, gatePack: "oss-anti-slop" }); + // gate.closeAuditHoldoutPct is manifest-only (config-as-code), valid range 0-20 (#8831). 20 with the + // pre-seeded secret above yields draw ≈0.097 < 0.20 — a real, deterministic diversion. + await upsertRepoFocusManifest(env, "owner/agent-repo", { settings: { checkRunMode: "off", commentMode: "off", publicSurface: "off", reviewCheckMode: "required", aiReviewMode: "block" }, gate: { closeAuditHoldoutPct: 20 } }); + await upsertPullRequestFromGitHub(env, "owner/agent-repo", { number: 20, title: "Holdout-diverted PR", state: "open", user: { login: "contributor" }, head: { sha: "d20" }, labels: [], body: "Closes #1" }); + await upsertPullRequestFile(env, { repoFullName: "owner/agent-repo", pullNumber: 20, path: "src/a.ts", status: "modified", additions: 1, deletions: 0, changes: 1, payload: { patch: "@@\n+export const ok = value.length;" } }); + const inputFingerprint = await cachedSubFloorDefectFingerprint("Holdout-diverted PR"); + await putCachedAiReview(env, "owner/agent-repo", 20, "d20", "block", { + notes: "cached review", + reviewerCount: 2, + // Above-floor confidence: without the holdout this would one-shot-close cleanly, same as the #9124 test. + findings: [{ code: "ai_consensus_defect", severity: "critical", title: "Unused import", detail: "unused import join from node:path is dead code.", confidence: 0.95, modelIds: ["claude-code", "codex"], promptDigest: "p".repeat(64) }], + metadata: { inputFingerprint }, + }); + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/pulls/20/files")) return Response.json([{ filename: "src/a.ts", status: "modified", additions: 1, deletions: 0, changes: 1, patch: "@@\n+export const ok = value.length;" }]); + if (url.endsWith("/pulls/20") && init?.method === "PATCH") return Response.json({ number: 20, state: "closed" }); + if (url.endsWith("/pulls/20")) return Response.json({ number: 20, title: "Holdout-diverted PR", state: "open", user: { login: "contributor" }, head: { sha: "d20" }, labels: [], body: "Closes #1" }); + if (url.includes("/commits/d20/check-runs")) return Response.json({ total_count: 0, check_runs: [] }); + if (url.includes("/commits/d20/status")) return Response.json({ state: "success", statuses: [] }); + if (url.endsWith("/pulls/20/reviews") && init?.method === "POST") return Response.json({ id: 1 }); + if (url.endsWith("/pulls/20/reviews")) return Response.json([]); + if (url.includes("/issues/1")) return Response.json({ number: 1, title: "Issue", state: "open", labels: [], user: { login: "reporter" } }); + if (url.includes("/issues/20/labels")) return Response.json([{ name: "manual-review" }]); + if (url.includes("/branches/")) return Response.json({ protected: false, protection: { required_status_checks: { contexts: [] } } }); + return Response.json({}); + }); + vi.setSystemTime(new Date("2026-05-28T02:00:00.000Z")); + + await sweepAndDrainPerPr(env, "owner/agent-repo"); + + // The holdout diverted the close: the recorded disposition is a hold, never a close. + const closeAudit = await env.DB.prepare("select count(*) as n from audit_events where event_type = ? and detail like ?").bind("agent.action.close", "%#20%").first<{ n: number }>(); + expect(closeAudit?.n).toBe(0); + const holdoutAudit = await env.DB.prepare("select count(*) as n from audit_events where event_type = 'decision_audit_holdout' and target_key = 'owner/agent-repo#20'").first<{ n: number }>(); + expect(holdoutAudit?.n).toBe(1); + + const record = await env.DB.prepare("select id, reason_code, record_json from decision_records where repo_full_name = ? and pull_number = ?").bind("owner/agent-repo", 20).first<{ id: string; reason_code: string; record_json: string }>(); + const parsed = JSON.parse(record!.record_json) as { action: string; divertedByHoldout: boolean }; + expect(parsed.action).toBe("hold"); + expect(parsed.divertedByHoldout).toBe(true); + + // The private replay input carries the SAME holdout outcome the public record's claim rests on — + // exactly the pair `holdout_consistency` (decision-replay.ts) checks. + const replayRow = await env.DB.prepare("select replay_json from decision_replay_inputs where record_id = ?").bind(record!.id).first<{ replay_json: string }>(); + const replayInput = JSON.parse(replayRow!.replay_json) as { holdout: { epsilonPct: number; draw: number; diverted: boolean } | null }; + expect(replayInput.holdout?.diverted).toBe(true); + expect(replayInput.holdout?.epsilonPct).toBe(20); + expect(replayInput.holdout?.draw).toBeLessThan(0.2); + const { replayDecision } = await import("../../src/review/decision-replay"); + const outcome = replayDecision({ id: record!.id, reasonCode: record!.reason_code, action: parsed.action, divertedByHoldout: parsed.divertedByHoldout }, replayInput as never); + expect(outcome.verdict).toBe("match"); + }); + it("#9009: auto-clears manual-review once AI-review lock contention (the pass that applied it) resolves on a later pass", async () => { const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), @@ -581,7 +726,9 @@ describe("queue processors", () => { await putCachedAiReview(env, "owner/agent-repo", 9, "c9", "block", { notes: "cached review", reviewerCount: 2, - findings: [{ code: "ai_consensus_defect", severity: "critical", title: "Cached defect", detail: "Cached critical defect.", confidence: 0.3 }], + // #9124: modelIds/promptDigest are the shape a FRESH ai_consensus_defect finding now carries + // (ai-review-orchestration.ts); the cached finding mirrors it so the decision record below inherits it. + findings: [{ code: "ai_consensus_defect", severity: "critical", title: "Cached defect", detail: "Cached critical defect.", confidence: 0.3, modelIds: ["claude-code"], promptDigest: "f".repeat(64) }], metadata: { inputFingerprint }, }); vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { @@ -620,10 +767,11 @@ describe("queue processors", () => { // #8834: an AI-judgment-shaped decision persists its confidence + prompt-template commitment in the // decision record — the row every future calibration read keys on. const record = await env.DB.prepare("select record_json from decision_records where repo_full_name = 'owner/agent-repo' and pull_number = 9 order by created_at desc limit 1").first<{ record_json: string }>(); - const parsedRecord = JSON.parse(record!.record_json) as { aiConfidence: number | null; promptDigest: string | null; schemaVersion: string }; - expect(parsedRecord.schemaVersion).toBe("3"); // v3 (#8962): + salvageability + const parsedRecord = JSON.parse(record!.record_json) as { aiConfidence: number | null; promptDigest: string | null; modelIds: string[] | null; schemaVersion: string }; + expect(parsedRecord.schemaVersion).toBe("4"); // v4 (#9124/#9135): configDigest/promptDigest/modelIds/ciState + divertedByHoldout expect(parsedRecord.aiConfidence).toBe(0.3); // the cached sub-floor defect's calibrated confidence - expect(typeof parsedRecord.promptDigest).toBe("string"); + expect(parsedRecord.promptDigest).toBe("f".repeat(64)); + expect(parsedRecord.modelIds).toEqual(["claude-code"]); }); it("posts the 🟪 reviewing placeholder before the AI review runs, then overwrites it with the verdict (#reviewing-placeholder)", async () => {