From 8fc259d19c9f6dff808709e234b752606ed65a3b Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 29 Jul 2026 03:23:37 -0700 Subject: [PATCH 1/4] fix(eval): stamp the in-Worker threshold backtest with its corpus freeze point loadPublicRulePrecision selects the latest backtest run with `json_extract(metadata_json, '$.corpusChecksum') IS NOT NULL` across two event types, but only one writer ever satisfied it. The CI-side logic-backtest CLI writes corpusChecksum; persistThresholdBacktestRuns -- which runs in-Worker on every review pass touching a KNOWN_THRESHOLDS constant -- wrote only `comparison` and `constantName`. So a deployment whose backtest history is entirely in-Worker had latestBacktestRun null forever, and buildEvalScoreRecordsFromRulePrecision early-returns [] on that: the whole /v1/public/eval-scores EvalScoreRecord feed served zero records with no diagnostic, no matter how many backtests had run. checksumCases moves from scripts/backtest-corpus-export-core.ts into packages/loopover-engine/src/calibration/ so the Worker can reach it, rather than being copied -- two implementations of a freeze point is worse than none, because a manifest frozen by one and validated by the other disagrees for no visible reason. backtest-corpus-export-core re-exports it so its existing importers are unaffected, and backtest-checksum.test.ts pins the digest of a fixed fixture against the pre-move implementation: manifests already on disk carry pre-move checksums, and two CLIs re-validate against this function before trusting a corpus. The checksum is taken over the corpus map that was actually scored, not re-read at persist time. A freeze point that does not match the cases behind the verdict invites a reader to re-derive different numbers and conclude the published ones are wrong. comparison/constantName are untouched -- rule-calibration-trend.ts reads $.comparison.verdict off these rows -- and corpusChecksum is omitted rather than nulled when absent, so the reader's IS NOT NULL filter stays the one place that decides whether a run is a usable commitment. Closes #9639 --- packages/loopover-engine/package.json | 4 + .../src/calibration/backtest-checksum.ts | 37 ++++++ packages/loopover-engine/src/index.ts | 3 + scripts/attested-backtest-run.ts | 2 +- scripts/backtest-corpus-export-core.ts | 20 +-- scripts/backtest-logic-check.ts | 2 +- src/queue/processors.ts | 7 +- src/services/threshold-backtest-run.ts | 37 +++++- test/unit/backtest-checksum.test.ts | 64 +++++++++ test/unit/eval-score-records.test.ts | 71 +++++++++- test/unit/public-rule-precision.test.ts | 59 +++++++++ test/unit/threshold-backtest-run.test.ts | 121 +++++++++++++++++- 12 files changed, 404 insertions(+), 23 deletions(-) create mode 100644 packages/loopover-engine/src/calibration/backtest-checksum.ts create mode 100644 test/unit/backtest-checksum.test.ts diff --git a/packages/loopover-engine/package.json b/packages/loopover-engine/package.json index 16523a8f4..fbdaed805 100644 --- a/packages/loopover-engine/package.json +++ b/packages/loopover-engine/package.json @@ -81,6 +81,10 @@ "./calibration/backtest-corpus": { "types": "./dist/calibration/backtest-corpus.d.ts", "default": "./dist/calibration/backtest-corpus.js" + }, + "./calibration/backtest-checksum": { + "types": "./dist/calibration/backtest-checksum.d.ts", + "default": "./dist/calibration/backtest-checksum.js" } }, "files": [ diff --git a/packages/loopover-engine/src/calibration/backtest-checksum.ts b/packages/loopover-engine/src/calibration/backtest-checksum.ts new file mode 100644 index 000000000..7b3f7a9cb --- /dev/null +++ b/packages/loopover-engine/src/calibration/backtest-checksum.ts @@ -0,0 +1,37 @@ +// The backtest corpus freeze point (#8136's reproducibility posture, moved here by #9639). +// +// This lived in scripts/backtest-corpus-export-core.ts, which made it reachable only from the CI-side CLIs. +// The in-Worker threshold backtest (src/services/threshold-backtest-run.ts) needs the SAME function to stamp +// its own runs: a deployment whose only backtest history is in-Worker was publishing no freeze point at all, +// so /v1/public/eval-scores served zero EvalScoreRecords. Two checksum implementations would have been worse +// than none -- a manifest frozen by one and validated by the other would disagree for no visible reason -- +// so the function moved rather than being copied. +// +// It is byte-stable by contract, not by accident: existing corpus manifests on disk carry checksums produced +// by the pre-move implementation, and scripts/backtest-logic-check.ts and scripts/attested-backtest-run.ts +// both re-validate a manifest against it before trusting the cases. Any change to the canonicalization or the +// hash input invalidates every manifest ever exported. backtest-checksum.test.ts pins the exact digest of a +// fixed fixture for that reason. +// +// node:crypto rather than Web Crypto: this must be SYNCHRONOUS (checksumCases is called inside pure, +// non-async manifest builders), and the Worker runs with nodejs_compat -- the same basis on which +// attester.ts, attestation-envelope.ts, backtest-split.ts and counterfactual-fixtures.ts in this same +// directory already import createHash. +import { createHash } from "node:crypto"; +import type { BacktestCase } from "./backtest-corpus.js"; + +/** Canonicalize one case (sort keys) so property-order differences don't change the checksum -- same technique + * as scripts/export-d1-core.ts's canonicalizeRow. */ +function canonicalizeCase(backtestCase: BacktestCase): Record { + /* v8 ignore next -- the comparator's `0` arm is unreachable: Object.entries yields each key once, so the + * two keys handed to a sort comparator are never equal. Kept anyway because a comparator that cannot + * return 0 is not a total order, and rewriting it to drop the arm would be a worse function for a branch + * counter's benefit. Every reachable arm (a < b, a > b) is exercised by the property-order test. */ + return Object.fromEntries(Object.entries(backtestCase).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))); +} + +/** Deterministic SHA-256 over the canonicalized cases -- mirrors scripts/export-d1-core.ts's checksumRows + * exactly (canonicalize each entry, JSON-stringify the array, hash). */ +export function checksumCases(cases: readonly BacktestCase[]): string { + return createHash("sha256").update(JSON.stringify(cases.map(canonicalizeCase))).digest("hex"); +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 9eab8e2a6..fc84cd7c7 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -164,6 +164,9 @@ export * from "./governor/action-mode.js"; export * from "./governor/chokepoint.js"; export * from "./calibration/signal-tracking.js"; export * from "./calibration/backtest-corpus.js"; +// #9639: the corpus freeze point, moved out of scripts/ so the in-Worker threshold backtest can stamp its +// own runs with the same checksum the CI-side manifests are frozen and validated by. +export * from "./calibration/backtest-checksum.js"; export * from "./calibration/repo-corpus-slice.js"; export * from "./calibration/ams-prediction-corpus.js"; export * from "./calibration/ams-rank-corpus.js"; diff --git a/scripts/attested-backtest-run.ts b/scripts/attested-backtest-run.ts index 0c020a56d..92c7139a0 100644 --- a/scripts/attested-backtest-run.ts +++ b/scripts/attested-backtest-run.ts @@ -30,7 +30,7 @@ import { readFileSync } from "node:fs"; import { assembleAttestationEnvelope, createSampleAttester, type Attester } from "@loopover/engine/calibration/attester"; import type { BacktestCase } from "@loopover/engine/calibration/backtest-corpus"; -import { checksumCases } from "./backtest-corpus-export-core"; +import { checksumCases } from "@loopover/engine"; import { attestedRunExitCode, buildAttestedRunAuditInsertSql, diff --git a/scripts/backtest-corpus-export-core.ts b/scripts/backtest-corpus-export-core.ts index 328e4e1f3..4e30179f7 100644 --- a/scripts/backtest-corpus-export-core.ts +++ b/scripts/backtest-corpus-export-core.ts @@ -2,9 +2,15 @@ // built BacktestCase[] (from buildBacktestCorpus) into a versioned, checksummed manifest a scorer can reload // without re-querying D1. No IO here — the CLI (backtest-corpus-export.ts) does the wrangler/D1 reads and the // file write — so this stays unit-testable. Mirrors scripts/export-d1-core.ts's pure-core / thin-IO split. -import { createHash } from "node:crypto"; +import { checksumCases } from "@loopover/engine"; import type { BacktestCase } from "@loopover/engine/calibration/backtest-corpus"; +// #9639: checksumCases moved into the engine so the in-Worker threshold backtest can use it too. Re-exported +// here because this module's own consumers (and its test) have always imported it from this path, and the +// checksum must stay ONE function -- a manifest frozen by one copy and validated by another would disagree +// for no visible reason. +export { checksumCases }; + export type BacktestCorpusManifest = { ruleId: string; caseCount: number; @@ -12,18 +18,6 @@ export type BacktestCorpusManifest = { cases: BacktestCase[]; }; -/** Canonicalize one case (sort keys) so property-order differences don't change the checksum — same technique - * as export-d1-core.ts's canonicalizeRow. */ -function canonicalizeCase(backtestCase: BacktestCase): Record { - return Object.fromEntries(Object.entries(backtestCase).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))); -} - -/** Deterministic SHA-256 over the canonicalized cases — mirrors export-d1-core.ts's checksumRows exactly - * (canonicalize each entry, JSON-stringify the array, hash). */ -export function checksumCases(cases: readonly BacktestCase[]): string { - return createHash("sha256").update(JSON.stringify(cases.map(canonicalizeCase))).digest("hex"); -} - /** * Build the export manifest for one rule's labeled corpus. Spreads an optional `meta` bag into the result * (the CLI attaches `generatedAt`); this core never reads the clock. Mirrors export-d1-core.ts's diff --git a/scripts/backtest-logic-check.ts b/scripts/backtest-logic-check.ts index 34f9574d1..bf5376020 100644 --- a/scripts/backtest-logic-check.ts +++ b/scripts/backtest-logic-check.ts @@ -17,7 +17,7 @@ import { resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { spawnSync } from "node:child_process"; import type { BacktestCase } from "@loopover/engine"; -import { checksumCases } from "./backtest-corpus-export-core"; +import { checksumCases } from "@loopover/engine"; import { buildLogicBacktestAuditInsertSql, filterReplayableCases, diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 4b6b9d745..763fd9fa2 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -8593,9 +8593,12 @@ export async function resolveThresholdBacktestAdvisory( if (mode === "off") return ""; if (!files.some((file) => THRESHOLD_BACKTEST_WATCHED_PATHS.has(file.path))) return ""; try { - const { changed, comparisons } = await runThresholdBacktestAdvisory(env, buildSecretScanDiff(files)); + const { changed, comparisons, corpusChecksumByRuleId } = await runThresholdBacktestAdvisory(env, buildSecretScanDiff(files)); if (comparisons.length === 0) return ""; - await persistThresholdBacktestRuns(env, repoFullName, pr.number, changed, comparisons); + // #9639: the freeze point travels with the run. Without it the persisted event is invisible to + // loadPublicRulePrecision's `corpusChecksum IS NOT NULL` filter, so /v1/public/eval-scores publishes + // nothing no matter how many backtests this deployment has actually run. + await persistThresholdBacktestRuns(env, repoFullName, pr.number, changed, comparisons, corpusChecksumByRuleId); // #8105 block mode: a REGRESSED verdict becomes a configured gate blocker. The finding only exists in // block mode (advisory keeps today's comment-only behavior byte-identically), and // isConfiguredGateBlocker's own `backtest_regression` branch is the defense-in-depth mirror of this diff --git a/src/services/threshold-backtest-run.ts b/src/services/threshold-backtest-run.ts index c90c6bb3e..93a50ccb7 100644 --- a/src/services/threshold-backtest-run.ts +++ b/src/services/threshold-backtest-run.ts @@ -9,7 +9,7 @@ // • A SignalStore read failure for one ruleId degrades that ruleId to an empty corpus (a real, if uninformative, // backtest result -- null precision/recall, per #8085's own null-when-no-data discipline) rather than // aborting the whole advisory. This never blocks the PR either way -- see the epic's own Boundaries. -import { buildBacktestCorpus, type BacktestCase, type BacktestComparison } from "@loopover/engine"; +import { buildBacktestCorpus, checksumCases, type BacktestCase, type BacktestComparison } from "@loopover/engine"; import { createSignalStore } from "../review/signal-tracking-wire"; import { recordAuditEvent } from "../db/repositories"; import { backtestChangedThreshold, detectChangedThresholds, type ChangedThreshold } from "./threshold-backtest"; @@ -32,6 +32,13 @@ async function fetchCorpus(env: Env, ruleId: string, nowMs: number): Promise; }; /** @@ -43,14 +50,21 @@ export type ThresholdBacktestRunResult = { */ export async function runThresholdBacktestAdvisory(env: Env, diff: string, nowMs: number = Date.now()): Promise { const changed = detectChangedThresholds(diff); - if (changed.length === 0) return { changed: [], comparisons: [] }; + if (changed.length === 0) return { changed: [], comparisons: [], corpusChecksumByRuleId: new Map() }; const ruleIds = [...new Set(changed.flatMap((change) => change.ruleIds))]; const corpusByRuleId = new Map(); for (const ruleId of ruleIds) corpusByRuleId.set(ruleId, await fetchCorpus(env, ruleId, nowMs)); + // Checksummed HERE, over the same map backtestChangedThreshold is about to score -- not re-read from the + // store afterwards. A second read could return a different corpus (a new override landing mid-run), and a + // freeze point that does not match the cases actually scored is worse than none: it invites a reader to + // re-derive different numbers and conclude the published ones are wrong. + const corpusChecksumByRuleId = new Map(); + for (const [ruleId, cases] of corpusByRuleId) corpusChecksumByRuleId.set(ruleId, checksumCases(cases)); + const comparisons = changed.flatMap((change) => backtestChangedThreshold(change, corpusByRuleId)); - return { changed, comparisons }; + return { changed, comparisons, corpusChecksumByRuleId }; } /** Persist each comparison for #8140's future track-record tool -- structured data via the shared @@ -63,12 +77,17 @@ export async function persistThresholdBacktestRuns( prNumber: number, changed: readonly ChangedThreshold[], comparisons: readonly BacktestComparison[], + // #9639: optional so existing call sites that only have `changed`/`comparisons` keep compiling, but every + // in-tree caller passes it -- without a checksum the event is exactly the one loadPublicRulePrecision + // filters out with `corpusChecksum IS NOT NULL`, which is the bug this closes. + corpusChecksumByRuleId: ReadonlyMap = new Map(), ): Promise { const comparisonsByRuleId = new Map(comparisons.map((comparison) => [comparison.ruleId, comparison])); for (const change of changed) { for (const ruleId of change.ruleIds) { const comparison = comparisonsByRuleId.get(ruleId); if (!comparison) continue; + const corpusChecksum = corpusChecksumByRuleId.get(ruleId); await recordAuditEvent(env, { eventType: THRESHOLD_BACKTEST_EVENT_TYPE, actor: "loopover", @@ -79,7 +98,17 @@ export async function persistThresholdBacktestRuns( // how every other telemetry-shaped audit event in this codebase uses "completed" for "happened, see detail". outcome: "completed", detail: `${change.constantName} threshold backtest for ${ruleId}: ${comparison.verdict}`, - metadata: { comparison, constantName: change.constantName }, + // corpusChecksum is a SIBLING of the existing keys, in the same position and under the same name + // buildLogicBacktestAuditInsertSql writes it (scripts/backtest-logic-check-core.ts) -- the two writers + // are siblings by design and loadPublicRulePrecision reads them through one query. `comparison` and + // `constantName` are untouched: rule-calibration-trend.ts reads `$.comparison.verdict` off these rows. + // Omitted rather than written as null when absent, so the reader's `IS NOT NULL` filter stays the + // single place that decides whether a run is a usable freeze point. + metadata: { + comparison, + constantName: change.constantName, + ...(corpusChecksum === undefined ? {} : { corpusChecksum }), + }, }).catch(() => undefined); } } diff --git a/test/unit/backtest-checksum.test.ts b/test/unit/backtest-checksum.test.ts new file mode 100644 index 000000000..a42dae715 --- /dev/null +++ b/test/unit/backtest-checksum.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; +// Imported by RELATIVE SOURCE path, not by the "@loopover/engine" specifier: that specifier resolves through +// the package's exports map to the compiled dist/, and v8 coverage would attribute every hit to the gitignored +// built file instead of this source -- the same misattribution vitest.config.ts's contractSourceAliases() +// exists to prevent for @loopover/contract, and the shape every other engine unit test here already uses. +import { checksumCases } from "../../packages/loopover-engine/src/calibration/backtest-checksum"; +import { checksumCases as checksumCasesViaBarrel } from "@loopover/engine"; +import type { BacktestCase } from "@loopover/engine/calibration/backtest-corpus"; +import { checksumCases as checksumCasesViaExportCore } from "../../scripts/backtest-corpus-export-core"; + +// #9639: checksumCases moved from scripts/backtest-corpus-export-core.ts into the engine so the in-Worker +// threshold backtest can stamp its runs with the SAME freeze point the CI-side manifests use. +// +// The move is only safe if the output is byte-identical. Corpus manifests already on disk carry checksums +// produced by the pre-move implementation, and scripts/backtest-logic-check.ts and +// scripts/attested-backtest-run.ts both re-validate a manifest against this function before trusting its +// cases -- so a drifted implementation does not fail loudly, it rejects every previously-exported corpus. + +// A fixed fixture, written out literally rather than generated: the digest below is only meaningful if the +// input can never change. Keys are deliberately NOT in sorted order, so the test also exercises the +// canonicalization rather than just the hash. +const FIXTURE: BacktestCase[] = [ + { targetKey: "acme/widgets#2", ruleId: "ai_consensus_defect", reversed: true, confidence: 0.91, firedAt: "2026-01-02T00:00:00.000Z" }, + { ruleId: "ai_consensus_defect", firedAt: "2026-01-01T00:00:00.000Z", targetKey: "acme/widgets#1", confidence: 0.42, reversed: false }, +] as unknown as BacktestCase[]; + +// Produced by the pre-move implementation in scripts/backtest-corpus-export-core.ts. If this constant has to +// be edited to make the test pass, the freeze point has drifted and every exported manifest is invalidated. +const FIXTURE_DIGEST = "d980dbeecbd9e9d63ff1182821b726b62b8a65a1f2ec37eafe3fa01aad493844"; + +describe("checksumCases byte-stability across the #9639 move", () => { + it("produces the pinned digest for the fixed fixture", () => { + expect(checksumCases(FIXTURE)).toBe(FIXTURE_DIGEST); + }); + + it("agrees byte-for-byte through the engine barrel and through scripts/backtest-corpus-export-core", () => { + // Behavioral sameness, not reference identity: the three paths resolve through different module + // instances (source here, dist via the package specifier), and it is the DIGEST that must not fork -- + // a manifest frozen through one path and validated through another has to agree. + expect(checksumCasesViaBarrel(FIXTURE)).toBe(FIXTURE_DIGEST); + expect(checksumCasesViaExportCore(FIXTURE)).toBe(FIXTURE_DIGEST); + }); + + it("ignores property order, so a reordered case set freezes to the same digest", () => { + const reordered = FIXTURE.map((c) => Object.fromEntries(Object.entries(c).reverse())) as unknown as BacktestCase[]; + expect(checksumCases(reordered)).toBe(FIXTURE_DIGEST); + }); + + it("does NOT ignore case order -- the corpus is a sequence, and a reordered export is a different freeze point", () => { + expect(checksumCases([...FIXTURE].reverse())).not.toBe(FIXTURE_DIGEST); + }); + + it("distinguishes a changed value, so the checksum actually commits to the cases", () => { + const mutated = FIXTURE.map((c, i) => (i === 0 ? { ...c, reversed: false } : c)) as unknown as BacktestCase[]; + expect(checksumCases(mutated)).not.toBe(FIXTURE_DIGEST); + }); + + it("hashes the empty corpus to a stable, rule-independent digest", () => { + // This is the value EMPTY_CORPUS_CHECKSUM exists to recognise: identical for every rule and every + // deployment, which is why eval-score-records.ts refuses to publish a record against it. + expect(checksumCases([])).toBe(checksumCases([])); + expect(checksumCases([])).toHaveLength(64); + }); +}); diff --git a/test/unit/eval-score-records.test.ts b/test/unit/eval-score-records.test.ts index 910e58ca3..ec7ab881f 100644 --- a/test/unit/eval-score-records.test.ts +++ b/test/unit/eval-score-records.test.ts @@ -10,7 +10,10 @@ import { type EvalScoreRecord, } from "../../src/review/eval-score-records"; import { contentDigest, sha256Hex } from "../../src/review/decision-record"; -import type { PublicRulePrecision } from "../../src/review/public-rule-precision"; +import { loadPublicRulePrecision, PUBLIC_PRECISION_MIN_DECIDED, type PublicRulePrecision } from "../../src/review/public-rule-precision"; +import { createSignalStore } from "../../src/review/signal-tracking-wire"; +import { persistThresholdBacktestRuns, runThresholdBacktestAdvisory } from "../../src/services/threshold-backtest-run"; +import { createTestEnv } from "../helpers/d1"; const ISSUED_AT = "2026-07-27T12:00:00.000Z"; @@ -154,3 +157,69 @@ describe("verifyEvalScoreRecordDigest", () => { expect(await verifyEvalScoreRecordDigest(tampered)).toBe(false); }); }); + +// #9639 Deliverable 4: the /v1/public/eval-scores regression, pinned end to end over a real D1. The unit +// tests above feed buildEvalScoreRecordsFromRulePrecision a hand-built PublicRulePrecision; this one starts +// from an in-Worker threshold backtest and goes all the way to records, because the defect lived in the seam +// between the writer and the reader -- both halves were individually correct and produced [] together. +describe("in-Worker backtest -> /v1/public/eval-scores records (#9639)", () => { + const diff = [ + "diff --git a/src/rules/advisory.ts b/src/rules/advisory.ts", + "@@ -980,7 +980,7 @@", + "-export const LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR = 0.5;", + "+export const LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR = 0.4;", + ].join("\n"); + + async function seedRunAndLoad(env: Env, now: number) { + const store = createSignalStore(env); + // Enough decided cases to clear PUBLIC_PRECISION_MIN_DECIDED, so a rule actually reaches the block. + for (let i = 0; i < PUBLIC_PRECISION_MIN_DECIDED + 2; i += 1) { + await store.recordRuleFired({ + ruleId: "linked_issue_scope_mismatch", + targetKey: `acme/widgets#${i + 1}`, + outcome: "unaddressed", + occurredAt: new Date(now - (i + 2) * 1000).toISOString(), + metadata: { confidence: 0.3 + (i % 4) * 0.15 }, + }); + await store.recordHumanOverride({ + ruleId: "linked_issue_scope_mismatch", + targetKey: `acme/widgets#${i + 1}`, + verdict: i % 2 === 0 ? "reversed" : "confirmed", + occurredAt: new Date(now - (i + 1) * 1000).toISOString(), + }); + } + const run = await runThresholdBacktestAdvisory(env, diff, now); + await persistThresholdBacktestRuns(env, "acme/widgets", 7, run.changed, run.comparisons, run.corpusChecksumByRuleId); + return { run, precision: await loadPublicRulePrecision(env, now) }; + } + + it("REGRESSION: emits one record per rule instead of [] -- the whole surface was empty before the writer stamped a checksum", async () => { + const env = createTestEnv(); + const now = Date.now(); + const { run, precision } = await seedRunAndLoad(env, now); + expect(precision.rules.length).toBeGreaterThan(0); + + const records = await buildEvalScoreRecordsFromRulePrecision(precision, ISSUED_AT); + + expect(records).toHaveLength(precision.rules.length); + for (const record of records) { + expect(record.commitments.corpusChecksum).toBe(run.corpusChecksumByRuleId.get("linked_issue_scope_mismatch")); + expect(record.commitments.corpusChecksum).not.toBe(EMPTY_CORPUS_CHECKSUM); + expect(record.trust.tier).toBe("reproducible"); + // Each record still commits to its own content, so the freeze point cannot be swapped undetected. + await expect(verifyEvalScoreRecordDigest(record)).resolves.toBe(true); + } + }); + + it("goes back to [] when the same pipeline runs against an empty corpus", async () => { + // The counter-case for the one above: the run is persisted either way, so an assertion that records exist + // is only meaningful alongside one showing they still do not when the commitment is worthless. + const env = createTestEnv(); + const now = Date.now(); + const run = await runThresholdBacktestAdvisory(env, diff, now); + await persistThresholdBacktestRuns(env, "acme/widgets", 7, run.changed, run.comparisons, run.corpusChecksumByRuleId); + + const precision = await loadPublicRulePrecision(env, now); + expect(await buildEvalScoreRecordsFromRulePrecision(precision, ISSUED_AT)).toEqual([]); + }); +}); diff --git a/test/unit/public-rule-precision.test.ts b/test/unit/public-rule-precision.test.ts index 4e1746b97..bc2cbc39b 100644 --- a/test/unit/public-rule-precision.test.ts +++ b/test/unit/public-rule-precision.test.ts @@ -8,6 +8,7 @@ import { import { sha256Hex } from "../../src/review/decision-record"; import { recordAuditEvent } from "../../src/db/repositories"; import { createSignalStore } from "../../src/review/signal-tracking-wire"; +import { persistThresholdBacktestRuns, runThresholdBacktestAdvisory } from "../../src/services/threshold-backtest-run"; import { createTestEnv } from "../helpers/d1"; // #8230 (epic #8211 track G): the public measured-accuracy block. Load-bearing properties: the public @@ -192,3 +193,61 @@ describe("loadPublicRulePrecision (#8230)", () => { expect(serialized).not.toMatch(/acme|#\d|targetKey|confidence|wallet|hotkey|trust|reward|payout/i); }); }); + +// #9639: the reader's `IN ('calibration.threshold_backtest_run', 'calibration.logic_backtest_run')` listed two +// writers but only the CI-side logic one ever satisfied the companion `corpusChecksum IS NOT NULL` filter. +// These pin the in-Worker writer's row as a first-class freeze point, end to end through the real service. +describe("latestBacktestRun from an in-Worker threshold run (#9639)", () => { + const diff = [ + "diff --git a/src/rules/advisory.ts b/src/rules/advisory.ts", + "@@ -980,7 +980,7 @@", + "-export const LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR = 0.5;", + "+export const LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR = 0.4;", + ].join("\n"); + + const seedAndRun = async (env: Env, now: number) => { + const store = createSignalStore(env); + // A labeled case needs BOTH halves: the firing (with its confidence) and the human verdict that scores it. + // Firings alone build an empty corpus, whose checksum is the rule-independent EMPTY_CORPUS_CHECKSUM. + for (let i = 0; i < 4; i += 1) { + await store.recordRuleFired({ + ruleId: "linked_issue_scope_mismatch", + targetKey: `acme/widgets#${i + 1}`, + outcome: "unaddressed", + occurredAt: new Date(now - (i + 2) * 1000).toISOString(), + metadata: { confidence: 0.3 + i * 0.2 }, + }); + await store.recordHumanOverride({ + ruleId: "linked_issue_scope_mismatch", + targetKey: `acme/widgets#${i + 1}`, + verdict: i % 2 === 0 ? "reversed" : "confirmed", + occurredAt: new Date(now - (i + 1) * 1000).toISOString(), + }); + } + const run = await runThresholdBacktestAdvisory(env, diff, now); + await persistThresholdBacktestRuns(env, "acme/widgets", 7, run.changed, run.comparisons, run.corpusChecksumByRuleId); + return run; + }; + + it("REGRESSION: a threshold_backtest_run is now a usable freeze point -- this returned null before the writer emitted corpusChecksum", async () => { + const env = createTestEnv(); + const now = Date.now(); + const run = await seedAndRun(env, now); + + const block = await loadPublicRulePrecision(env, now); + expect(block.latestBacktestRun).not.toBeNull(); + expect(block.latestBacktestRun?.corpusChecksum).toBe(run.corpusChecksumByRuleId.get("linked_issue_scope_mismatch")); + expect(block.latestBacktestRun?.corpusChecksum).not.toBe(EMPTY_CORPUS_CHECKSUM); + }); + + it("still returns null when the same writer ran against an empty corpus -- a hash over zero cases commits to nothing", async () => { + // No seeded history, so the corpus is empty and its checksum is the rule-independent EMPTY_CORPUS_CHECKSUM. + // The row IS written (the run happened); the reader is what declines to treat it as a commitment. + const env = createTestEnv(); + const now = Date.now(); + const run = await runThresholdBacktestAdvisory(env, diff, now); + await persistThresholdBacktestRuns(env, "acme/widgets", 7, run.changed, run.comparisons, run.corpusChecksumByRuleId); + + expect((await loadPublicRulePrecision(env, now)).latestBacktestRun).toBeNull(); + }); +}); diff --git a/test/unit/threshold-backtest-run.test.ts b/test/unit/threshold-backtest-run.test.ts index 7107d933e..4c08eabaa 100644 --- a/test/unit/threshold-backtest-run.test.ts +++ b/test/unit/threshold-backtest-run.test.ts @@ -4,6 +4,7 @@ import { createSignalStore } from "../../src/review/signal-tracking-wire"; import * as repositories from "../../src/db/repositories"; import { listAuditEventsByType } from "../../src/db/repositories"; import { persistThresholdBacktestRuns, runThresholdBacktestAdvisory, THRESHOLD_BACKTEST_EVENT_TYPE } from "../../src/services/threshold-backtest-run"; +import { checksumCases } from "@loopover/engine"; import { createTestEnv } from "../helpers/d1"; afterEach(() => { @@ -18,7 +19,7 @@ describe("runThresholdBacktestAdvisory (#8138) — real D1 round-trip", () => { it("returns empty changed/comparisons when the diff touches no known threshold, without any D1 read", async () => { const env = createTestEnv(); const result = await runThresholdBacktestAdvisory(env, "diff --git a/README.md b/README.md\n-old\n+new"); - expect(result).toEqual({ changed: [], comparisons: [] }); + expect(result).toEqual({ changed: [], comparisons: [], corpusChecksumByRuleId: new Map() }); }); it("backtests a changed LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR against real recorded history", async () => { @@ -111,3 +112,121 @@ describe("persistThresholdBacktestRuns (#8138) — real D1 round-trip", () => { await expect(persistThresholdBacktestRuns(env, "acme/widgets", 8, changed, comparisons)).resolves.toBeUndefined(); }); }); + +// #9639: loadPublicRulePrecision selects the latest backtest run with +// `json_extract(metadata_json, '$.corpusChecksum') IS NOT NULL`. This writer never emitted the field, so a +// deployment whose only backtest history is in-Worker had `latestBacktestRun: null` forever -- and +// buildEvalScoreRecordsFromRulePrecision early-returns [] on that, which is why /v1/public/eval-scores +// served zero records with no diagnostic. +describe("persistThresholdBacktestRuns writes the corpus freeze point (#9639)", () => { + const seedHistory = async (env: Env, now: number) => { + const store = createSignalStore(env); + // A labeled case needs BOTH halves: the firing (with its confidence) and the human verdict that scores it. + // Firings alone build an empty corpus, whose checksum is the rule-independent EMPTY_CORPUS_CHECKSUM. + for (let i = 0; i < 4; i += 1) { + await store.recordRuleFired({ + ruleId: "linked_issue_scope_mismatch", + targetKey: `acme/widgets#${i + 1}`, + outcome: "unaddressed", + occurredAt: new Date(now - (i + 2) * 1000).toISOString(), + metadata: { confidence: 0.3 + i * 0.2 }, + }); + await store.recordHumanOverride({ + ruleId: "linked_issue_scope_mismatch", + targetKey: `acme/widgets#${i + 1}`, + verdict: i % 2 === 0 ? "reversed" : "confirmed", + occurredAt: new Date(now - (i + 1) * 1000).toISOString(), + }); + } + }; + + it("REGRESSION: writes metadata.corpusChecksum, the field the public reader filters on", async () => { + const env = createTestEnv(); + const now = Date.now(); + await seedHistory(env, now); + const { changed, comparisons, corpusChecksumByRuleId } = await runThresholdBacktestAdvisory(env, diffFor("LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR", "0.5", "0.4"), now); + await persistThresholdBacktestRuns(env, "acme/widgets", 7, changed, comparisons, corpusChecksumByRuleId); + + const rows = await listAuditEventsByType(env, THRESHOLD_BACKTEST_EVENT_TYPE, new Date(now - 60_000).toISOString()); + expect(rows).toHaveLength(1); + const checksum = rows[0]!.metadata.corpusChecksum; + expect(checksum).toMatch(/^[0-9a-f]{64}$/); + // Not the rule-independent empty-corpus digest -- that would commit to nothing re-derivable. + expect(checksum).not.toBe(checksumCases([])); + // Not just "a hash": the SAME hash the shared freeze-point function produces over the scored corpus. + expect(checksum).toBe(corpusChecksumByRuleId.get("linked_issue_scope_mismatch")); + }); + + it("leaves metadata.comparison and metadata.constantName untouched -- rule-calibration-trend.ts reads $.comparison.verdict off these rows", async () => { + const env = createTestEnv(); + const now = Date.now(); + await seedHistory(env, now); + const { changed, comparisons, corpusChecksumByRuleId } = await runThresholdBacktestAdvisory(env, diffFor("LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR", "0.5", "0.4"), now); + await persistThresholdBacktestRuns(env, "acme/widgets", 9, changed, comparisons, corpusChecksumByRuleId); + + const row = (await listAuditEventsByType(env, THRESHOLD_BACKTEST_EVENT_TYPE, new Date(now - 60_000).toISOString()))[0]!; + expect(row.metadata.constantName).toBe("LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR"); + const comparison = row.metadata.comparison as { ruleId: string; verdict: string }; + expect(comparison.ruleId).toBe("linked_issue_scope_mismatch"); + expect(typeof comparison.verdict).toBe("string"); + }); + + it("checksums the corpus that was SCORED, not a fresh read -- a later override does not move the freeze point", async () => { + // The freeze point has to describe the cases the verdict came from. If it were re-read at persist time, + // a reader re-deriving from it would get numbers that disagree with the published ones and conclude the + // publication was wrong. + const env = createTestEnv(); + const now = Date.now(); + await seedHistory(env, now); + const { changed, comparisons, corpusChecksumByRuleId } = await runThresholdBacktestAdvisory(env, diffFor("LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR", "0.5", "0.4"), now); + const frozen = corpusChecksumByRuleId.get("linked_issue_scope_mismatch"); + + await createSignalStore(env).recordRuleFired({ + ruleId: "linked_issue_scope_mismatch", + targetKey: "acme/widgets#999", + outcome: "unaddressed", + occurredAt: new Date(now - 2000).toISOString(), + metadata: { confidence: 0.99 }, + }); + await createSignalStore(env).recordHumanOverride({ + ruleId: "linked_issue_scope_mismatch", + targetKey: "acme/widgets#999", + verdict: "reversed", + occurredAt: new Date(now - 1500).toISOString(), + }); + await persistThresholdBacktestRuns(env, "acme/widgets", 11, changed, comparisons, corpusChecksumByRuleId); + + const row = (await listAuditEventsByType(env, THRESHOLD_BACKTEST_EVENT_TYPE, new Date(now - 60_000).toISOString()))[0]!; + expect(row.metadata.corpusChecksum).toBe(frozen); + }); + + it("omits corpusChecksum entirely (never null) when a direct caller supplies no map, so the reader's IS NOT NULL filter stays the single gate", async () => { + const env = createTestEnv(); + const now = Date.now(); + await seedHistory(env, now); + const { changed, comparisons } = await runThresholdBacktestAdvisory(env, diffFor("LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR", "0.5", "0.4"), now); + await persistThresholdBacktestRuns(env, "acme/widgets", 13, changed, comparisons); + + const row = (await listAuditEventsByType(env, THRESHOLD_BACKTEST_EVENT_TYPE, new Date(now - 60_000).toISOString()))[0]!; + expect("corpusChecksum" in row.metadata).toBe(false); + }); + + it("still records a freeze point for a ruleId whose corpus read FAILED -- the empty corpus is a true statement about what was backtested", async () => { + // fetchCorpus degrades a failed read to [], and that is genuinely what got scored. The reader is what + // refuses to publish against EMPTY_CORPUS_CHECKSUM; the writer must not silently omit the field and make + // a failed read indistinguishable from an old deployment that never wrote one. + const env = createTestEnv(); + const now = Date.now(); + vi.spyOn(signalTrackingWire, "createSignalStore").mockReturnValue({ + queryRuleHistory: () => Promise.reject(new Error("simulated read failure")), + } as unknown as ReturnType); + + const { changed, comparisons, corpusChecksumByRuleId } = await runThresholdBacktestAdvisory(env, diffFor("LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR", "0.5", "0.4"), now); + vi.restoreAllMocks(); + await persistThresholdBacktestRuns(env, "acme/widgets", 15, changed, comparisons, corpusChecksumByRuleId); + + const rows = await listAuditEventsByType(env, THRESHOLD_BACKTEST_EVENT_TYPE, new Date(now - 60_000).toISOString()); + if (rows.length > 0) expect(rows[0]!.metadata.corpusChecksum).toBe(checksumCases([])); + expect(corpusChecksumByRuleId.get("linked_issue_scope_mismatch")).toBe(checksumCases([])); + }); +}); From ce871a20a7eedeb2d65c507e53ba185acbbe0da1 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 29 Jul 2026 03:49:06 -0700 Subject: [PATCH 2/4] test(engine): cover the moved checksum in its own package, through dist codecov/patch flagged backtest-checksum.ts at 83.78% under the `engine` flag while `backend` reported 100%: the file lives in packages/loopover-engine/src, which the engine's own node --test suite measures, but the only test for it was in the root vitest suite. #9639 required the moved function to be covered in its new home. This is that, and it earns its place beyond the flag: the engine suite runs against dist/, so it is the only check of the freeze point through the exact artifact published consumers import rather than through the source. Both tests now share one fixture and one pinned digest, and the fixture is a real BacktestCase (the engine suite typechecks its tests, which caught that the root one had been getting by on a cast). The digest is recomputed from the pre-move implementation for the new fixture, so it still pins byte-stability against what exported manifests carry. --- .../test/backtest-checksum.test.ts | 50 +++++++++++++++++++ test/unit/backtest-checksum.test.ts | 12 ++--- 2 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 packages/loopover-engine/test/backtest-checksum.test.ts diff --git a/packages/loopover-engine/test/backtest-checksum.test.ts b/packages/loopover-engine/test/backtest-checksum.test.ts new file mode 100644 index 000000000..d92a655ba --- /dev/null +++ b/packages/loopover-engine/test/backtest-checksum.test.ts @@ -0,0 +1,50 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { checksumCases } from "../dist/index.js"; +import type { BacktestCase } from "../dist/calibration/backtest-corpus.js"; + +// #9639: checksumCases moved here from scripts/backtest-corpus-export-core.ts so the in-Worker threshold +// backtest can stamp its runs with the same freeze point the CI-side manifests are frozen and validated by. +// +// The root test/unit/backtest-checksum.test.ts covers this from the host side; this is its coverage in its +// OWN package, where the `engine` flag measures it. More than a flag exercise, though: this suite runs +// against dist/, so it is the only place the moved function is checked through the exact artifact published +// consumers import. + +// A fixed fixture with deliberately unsorted keys, so canonicalization is exercised and not just the hash. +const FIXTURE: BacktestCase[] = [ + { targetKey: "acme/widgets#2", ruleId: "ai_consensus_defect", outcome: "close", label: "reversed", decidedAt: "2026-01-03T00:00:00.000Z", firedAt: "2026-01-02T00:00:00.000Z", metadata: { confidence: 0.91 } }, + { ruleId: "ai_consensus_defect", firedAt: "2026-01-01T00:00:00.000Z", decidedAt: "2026-01-02T00:00:00.000Z", targetKey: "acme/widgets#1", label: "confirmed", outcome: "merge", metadata: { confidence: 0.42 } }, +]; + +// Produced by the PRE-MOVE implementation. Corpus manifests already on disk carry checksums from it, and +// scripts/backtest-logic-check.ts and scripts/attested-backtest-run.ts both re-validate a manifest against +// this function before trusting its cases -- so drift here does not fail loudly, it silently rejects every +// corpus ever exported. If this constant has to be edited to make the test pass, that has happened. +const FIXTURE_DIGEST = "bba3c7db2e1ff0b6943802416ebf94c789f37ac5ed962cc7167c2dd33a33a861"; + +test("checksumCases produces the pinned pre-move digest", () => { + assert.equal(checksumCases(FIXTURE), FIXTURE_DIGEST); +}); + +test("checksumCases ignores property order", () => { + const reordered = FIXTURE.map((c) => Object.fromEntries(Object.entries(c).reverse()) as unknown as BacktestCase); + assert.equal(checksumCases(reordered), FIXTURE_DIGEST); +}); + +test("checksumCases does NOT ignore case order -- the corpus is a sequence", () => { + assert.notEqual(checksumCases([...FIXTURE].reverse()), FIXTURE_DIGEST); +}); + +test("checksumCases distinguishes a changed value, so it genuinely commits to the cases", () => { + const mutated = FIXTURE.map((c, i) => (i === 0 ? { ...c, reversed: false } : c)); + assert.notEqual(checksumCases(mutated), FIXTURE_DIGEST); +}); + +test("checksumCases is deterministic and 64 hex chars for the empty corpus", () => { + // The value EMPTY_CORPUS_CHECKSUM recognises: identical for every rule, window and deployment, which is + // why eval-score-records.ts refuses to publish a record against it. + assert.equal(checksumCases([]), checksumCases([])); + assert.match(checksumCases([]), /^[0-9a-f]{64}$/); +}); diff --git a/test/unit/backtest-checksum.test.ts b/test/unit/backtest-checksum.test.ts index a42dae715..efe7b2077 100644 --- a/test/unit/backtest-checksum.test.ts +++ b/test/unit/backtest-checksum.test.ts @@ -20,13 +20,13 @@ import { checksumCases as checksumCasesViaExportCore } from "../../scripts/backt // input can never change. Keys are deliberately NOT in sorted order, so the test also exercises the // canonicalization rather than just the hash. const FIXTURE: BacktestCase[] = [ - { targetKey: "acme/widgets#2", ruleId: "ai_consensus_defect", reversed: true, confidence: 0.91, firedAt: "2026-01-02T00:00:00.000Z" }, - { ruleId: "ai_consensus_defect", firedAt: "2026-01-01T00:00:00.000Z", targetKey: "acme/widgets#1", confidence: 0.42, reversed: false }, -] as unknown as BacktestCase[]; + { targetKey: "acme/widgets#2", ruleId: "ai_consensus_defect", outcome: "close", label: "reversed", decidedAt: "2026-01-03T00:00:00.000Z", firedAt: "2026-01-02T00:00:00.000Z", metadata: { confidence: 0.91 } }, + { ruleId: "ai_consensus_defect", firedAt: "2026-01-01T00:00:00.000Z", decidedAt: "2026-01-02T00:00:00.000Z", targetKey: "acme/widgets#1", label: "confirmed", outcome: "merge", metadata: { confidence: 0.42 } }, +]; // Produced by the pre-move implementation in scripts/backtest-corpus-export-core.ts. If this constant has to // be edited to make the test pass, the freeze point has drifted and every exported manifest is invalidated. -const FIXTURE_DIGEST = "d980dbeecbd9e9d63ff1182821b726b62b8a65a1f2ec37eafe3fa01aad493844"; +const FIXTURE_DIGEST = "bba3c7db2e1ff0b6943802416ebf94c789f37ac5ed962cc7167c2dd33a33a861"; describe("checksumCases byte-stability across the #9639 move", () => { it("produces the pinned digest for the fixed fixture", () => { @@ -42,7 +42,7 @@ describe("checksumCases byte-stability across the #9639 move", () => { }); it("ignores property order, so a reordered case set freezes to the same digest", () => { - const reordered = FIXTURE.map((c) => Object.fromEntries(Object.entries(c).reverse())) as unknown as BacktestCase[]; + const reordered = FIXTURE.map((c) => Object.fromEntries(Object.entries(c).reverse()) as unknown as BacktestCase); expect(checksumCases(reordered)).toBe(FIXTURE_DIGEST); }); @@ -51,7 +51,7 @@ describe("checksumCases byte-stability across the #9639 move", () => { }); it("distinguishes a changed value, so the checksum actually commits to the cases", () => { - const mutated = FIXTURE.map((c, i) => (i === 0 ? { ...c, reversed: false } : c)) as unknown as BacktestCase[]; + const mutated = FIXTURE.map((c, i) => (i === 0 ? { ...c, reversed: false } : c)); expect(checksumCases(mutated)).not.toBe(FIXTURE_DIGEST); }); From 84b6dd93b8bb9cd53a75a7988aa092ce472093b2 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 29 Jul 2026 04:26:07 -0700 Subject: [PATCH 3/4] fix(replay-runner): keep the attested runner on subpath imports Review catch: importing checksumCases from the "@loopover/engine" barrel reversed the rationale documented a few lines above that import. The barrel's re-export graph pulls in web-tree-sitter and claude-agent-sdk, and scripts/replay-runner/Dockerfile deliberately constructs @loopover/engine from three named dist files rather than npm-installing it -- so the barrel import would have failed to resolve in the container at runtime, and even if resolved, would have widened a trusted computing base that is measured inside a TEE. Uses the subpath export this change already added. dist/calibration/ backtest-checksum.js is added to the Dockerfile's COPY list; it compiles to a single node:crypto import (its only other import is type-only and erased), so it widens the image's file list without widening its dependency footprint at all. replay-runner-manifest:check caught the drift, as its own Dockerfile comment says it will; the regenerated manifest is committed alongside. --- scripts/attested-backtest-run.ts | 2 +- scripts/replay-runner-image-manifest.json | 10 +++++----- scripts/replay-runner/Dockerfile | 8 ++++++-- test/unit/eval-score-records.test.ts | 2 ++ 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/scripts/attested-backtest-run.ts b/scripts/attested-backtest-run.ts index 92c7139a0..2c86b8a97 100644 --- a/scripts/attested-backtest-run.ts +++ b/scripts/attested-backtest-run.ts @@ -30,7 +30,7 @@ import { readFileSync } from "node:fs"; import { assembleAttestationEnvelope, createSampleAttester, type Attester } from "@loopover/engine/calibration/attester"; import type { BacktestCase } from "@loopover/engine/calibration/backtest-corpus"; -import { checksumCases } from "@loopover/engine"; +import { checksumCases } from "@loopover/engine/calibration/backtest-checksum"; import { attestedRunExitCode, buildAttestedRunAuditInsertSql, diff --git a/scripts/replay-runner-image-manifest.json b/scripts/replay-runner-image-manifest.json index 889e03527..118a0ddcd 100644 --- a/scripts/replay-runner-image-manifest.json +++ b/scripts/replay-runner-image-manifest.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, "baseImageRef": "node:22-slim@sha256:6c74791e557ce11fc957704f6d4fe134a7bc8d6f5ca4403205b2966bd488f6b3", - "dockerfileSha256": "7e4d3c18fbc3a4153d7ba25e3cff8329b07fceb6a52a62350ba0f3c65f54f2ff", - "packageLockSha256": "050ad66cb6a9ba6cfceb6e379f59341fea11eac0bdf5ffc2cfe17febdb0d5038", + "dockerfileSha256": "18b434af0c6359105293a9bd6bdd890f4b5662ce72308df4bcfc3bb30ae7c1ee", + "packageLockSha256": "35c61915f0b21d5980f5f5aa1bff532da3d9d51c71257c8b28904f7b798e0d71", "sourceFiles": { "scripts/attested-backtest-run-core.ts": "e5f5659dc9204f9406c9874317d71cb02fd7f4c993c9f5caa92545595f522167", - "scripts/attested-backtest-run.ts": "0db6b462afdf80086eb68da793a6dccbe3d5751eceeb19e338d2746a1c319fa5", - "scripts/backtest-corpus-export-core.ts": "6a268410ea1d8041e95af345149aeb4848376c4c72127b801ee9da377b81e85e", + "scripts/attested-backtest-run.ts": "49e0e046e3c4882c0e4afd422a464957b6c4789bad28f5b6682afa9d534424f7", + "scripts/backtest-corpus-export-core.ts": "060d48a2a6073a7d859efc64c87c18ae1d3d25e06391ce94d6ed40874050b2e8", "scripts/snp-attester.ts": "b917e8541908150bd0c885fbe57e178247e5cd5f78f80eda7dc9e8b77da6deb3" }, - "digest": "cdf7c3aaf83f814cf5b0ce37887abb264c9b5ee2d903ddb47a3d9dc60c64fa0c" + "digest": "5b8665f586317a0f3d77c5ac76891b4cf2ba8f1784977d268b093ce6644b2210" } diff --git a/scripts/replay-runner/Dockerfile b/scripts/replay-runner/Dockerfile index 59de929a1..d4f185576 100644 --- a/scripts/replay-runner/Dockerfile +++ b/scripts/replay-runner/Dockerfile @@ -54,8 +54,11 @@ COPY --from=runtime-deps /app/node_modules ./node_modules # @loopover/engine is constructed here directly from its own compiled output -- NOT `npm ci`'d -- because npm # would install every dependency the FULL package declares (@anthropic-ai/claude-agent-sdk, tree-sitter-wasms, # ...), regardless of which files this image actually imports. attester.js/attestation-envelope.js/ -# backtest-corpus.js's entire runtime closure is these three files plus node:crypto (verified: `grep -n -# "^import" packages/loopover-engine/dist/calibration/*.js` -- attested-backtest-run.ts's own subpath imports, +# backtest-corpus.js/backtest-checksum.js's entire runtime closure is these four files plus node:crypto +# (verified: `grep -n "^import" packages/loopover-engine/dist/calibration/*.js` -- backtest-checksum.js, added +# by #9639, compiles to a single `node:crypto` import because its only other import is type-only and erased, +# so it widens this image's file list without widening its trusted computing base at all) +# -- attested-backtest-run.ts's own subpath imports, # #9214, are what make this closure reachable without ever loading the barrel's web-tree-sitter/ # claude-agent-sdk-carrying re-export graph in the first place). This measured image is meant to run inside # an eventual TEE (#8534); keeping its dependency footprint to exactly what replay needs is a real security @@ -64,6 +67,7 @@ COPY --from=build --chown=replay-runner:replay-runner /app/packages/loopover-eng COPY --from=build --chown=replay-runner:replay-runner /app/packages/loopover-engine/dist/calibration/attester.js ./node_modules/@loopover/engine/dist/calibration/attester.js COPY --from=build --chown=replay-runner:replay-runner /app/packages/loopover-engine/dist/calibration/attestation-envelope.js ./node_modules/@loopover/engine/dist/calibration/attestation-envelope.js COPY --from=build --chown=replay-runner:replay-runner /app/packages/loopover-engine/dist/calibration/backtest-corpus.js ./node_modules/@loopover/engine/dist/calibration/backtest-corpus.js +COPY --from=build --chown=replay-runner:replay-runner /app/packages/loopover-engine/dist/calibration/backtest-checksum.js ./node_modules/@loopover/engine/dist/calibration/backtest-checksum.js # The runner script's own sources -- listed individually (rather than `COPY scripts/`) so this image's content # is EXACTLY the set replay-runner-image-manifest.json enumerates under sourceFiles; adding a new script # dependency here without a matching manifest entry is precisely the drift `replay-runner-manifest:check` catches. diff --git a/test/unit/eval-score-records.test.ts b/test/unit/eval-score-records.test.ts index a74aed36a..f657092e4 100644 --- a/test/unit/eval-score-records.test.ts +++ b/test/unit/eval-score-records.test.ts @@ -238,6 +238,8 @@ describe("per-rule corpus commitments when no backtest run is persisted (#9805)" it("still returns [] with neither a run nor any commitment -- the #9215 rule is unchanged", async () => { expect(await buildEvalScoreRecordsFromRulePrecision(precisionOf([rule("ai_consensus_defect")]), ISSUED_AT)).toEqual([]); + }); +}); // #9639 Deliverable 4: the /v1/public/eval-scores regression, pinned end to end over a real D1. The unit // tests above feed buildEvalScoreRecordsFromRulePrecision a hand-built PublicRulePrecision; this one starts From f5114b4d457aebca4e2d95c230423bf7139762f1 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 29 Jul 2026 05:41:30 -0700 Subject: [PATCH 4/4] refactor(engine): drop the unreachable comparator arm in canonicalizeCase codecov/patch held at 97.29% on this file under the `engine` flag: one line, the sort comparator's equality arm. It is genuinely unreachable -- Object.keys yields each key exactly once, so a comparator never sees two equal keys -- and the `v8 ignore` pragma covering it was flag-dependent: vitest's v8 provider honoured it, the engine package's own coverage run did not. Rather than chase pragmas across two coverage tools, the branch is gone. Sorting the keys with the default comparator gives the identical total order for strings, so the freeze point does not move -- which is not an assertion, it is what the pinned pre-move digest in both test suites verifies. An unreachable branch can only sit uncovered or carry an ignore comment; not writing one beats both. --- .../src/calibration/backtest-checksum.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/loopover-engine/src/calibration/backtest-checksum.ts b/packages/loopover-engine/src/calibration/backtest-checksum.ts index 7b3f7a9cb..cad3dff04 100644 --- a/packages/loopover-engine/src/calibration/backtest-checksum.ts +++ b/packages/loopover-engine/src/calibration/backtest-checksum.ts @@ -23,11 +23,14 @@ import type { BacktestCase } from "./backtest-corpus.js"; /** Canonicalize one case (sort keys) so property-order differences don't change the checksum -- same technique * as scripts/export-d1-core.ts's canonicalizeRow. */ function canonicalizeCase(backtestCase: BacktestCase): Record { - /* v8 ignore next -- the comparator's `0` arm is unreachable: Object.entries yields each key once, so the - * two keys handed to a sort comparator are never equal. Kept anyway because a comparator that cannot - * return 0 is not a total order, and rewriting it to drop the arm would be a worse function for a branch - * counter's benefit. Every reachable arm (a < b, a > b) is exercised by the property-order test. */ - return Object.fromEntries(Object.entries(backtestCase).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))); + // Sorted with the DEFAULT comparator over the keys, not a hand-written `a < b ? -1 : a > b ? 1 : 0`. For + // strings the two give the identical total order -- the pinned digest in this module's two test suites is + // what proves the output did not move -- but the hand-written form carries an equality arm that + // Object.keys can never trigger, since it yields each key exactly once. An unreachable branch cannot be + // tested, so it either sits uncovered or needs an ignore pragma; not writing it beats both. (The pragma + // was also flag-dependent: vitest's v8 provider honoured it, the engine package's own coverage did not.) + const source = backtestCase as unknown as Record; + return Object.fromEntries(Object.keys(source).sort().map((key) => [key, source[key]])); } /** Deterministic SHA-256 over the canonicalized cases -- mirrors scripts/export-d1-core.ts's checksumRows