From 865f3a6973c65e6ad29fcb9c8964c813753eab54 Mon Sep 17 00:00:00 2001
From: JSONbored <49853598+JSONbored@users.noreply.github.com>
Date: Thu, 30 Jul 2026 16:15:04 -0700
Subject: [PATCH] =?UTF-8?q?fix(test):=20unbreak=20main=20=E2=80=94=20seed?=
 =?UTF-8?q?=20the=20downloadable=20corpus=20#9966=20now=20requires?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

main has been red since #9966 (40133d789): every commit after it fails
test/integration/public-eval-scores-route.test.ts with "expected [] to have a
length of 1". Bisected -- the commit before it passes 9/9.

#9966 made the DOWNLOADABLE corpus the only commitment source, deliberately
dropping the persisted-backtest-run fallback: a commitment must name bytes a
reader can fetch and re-hash. Correct change, but this suite's fixture recorded
only the human OVERRIDES, and the corpus is built from rule-FIRED events joined
to them -- so it produced a 0-case corpus, no commitment, and no record.
Confirmed by probing loadPublicEvalCorpus directly: caseCount 0 with overrides
alone, 20 once the fired events are seeded too.

Seeding both halves restores what the fixture always meant to represent: 20
decided cases a reader can download. Changing the expectation to `records: []`
would have gone green while silently dropping the recordDigest-recomputability
check this suite exists for.

The commitment assertion moves off the literal "freeze-point-checksum" (the old
contract) and onto what /v1/public/eval-corpus actually serves, so the two
surfaces cannot drift apart while both still look correct in isolation.
---
 package.json                                  |   7 +-
 scripts/check-typecheck-coverage.ts           | 129 ++++++++++++++++++
 .../public-eval-scores-route.test.ts          |  21 ++-
 .../check-typecheck-coverage-script.test.ts   |  69 ++++++++++
 4 files changed, 221 insertions(+), 5 deletions(-)
 create mode 100644 scripts/check-typecheck-coverage.ts
 create mode 100644 test/unit/check-typecheck-coverage-script.test.ts

diff --git a/package.json b/package.json
index 2ef968b5c5..d729176943 100644
--- a/package.json
+++ b/package.json
@@ -60,6 +60,7 @@
     "db:migrations:check": "tsx scripts/check-migrations.ts",
     "db:migrations:immutable:check": "tsx scripts/check-released-migrations-immutable.ts",
     "turbo-inputs:check": "tsx scripts/check-turbo-typecheck-inputs.ts",
+    "typecheck-coverage:check": "tsx scripts/check-typecheck-coverage.ts",
     "fixture-clock-races:check": "tsx scripts/check-fixture-clock-races.ts",
     "workspace-dep-ranges:check": "tsx scripts/check-workspace-dep-ranges.ts",
     "db:schema-drift:check": "tsx scripts/check-schema-drift.ts",
@@ -70,7 +71,7 @@
     "ui:build": "npm run ui:kit:build && turbo run build --filter=@loopover/engine --filter=@loopover/contract && npm run ui:openapi && npm --workspace @loopover/ui run build && npm --workspace @loopover/ui-miner run build",
     "ui:preview": "npm run ui:build && wrangler dev --config apps/loopover-ui/dist/server/wrangler.json --ip 127.0.0.1 --port 4173 --local",
     "ui:lint": "npm run ui:kit:build && npm --workspace @loopover/ui-kit run format:check && npm --workspace @loopover/ui run format:check && npm --workspace @loopover/ui run lint && npm --workspace @loopover/ui-miner run format:check && npm --workspace @loopover/ui-miner run lint",
-    "ui:typecheck": "npm run ui:kit:build && npm --workspace @loopover/ui run typecheck && npm --workspace @loopover/ui-miner run typecheck",
+    "ui:typecheck": "npm run ui:kit:build && npm --workspace @loopover/ui-kit run typecheck && npm --workspace @loopover/ui run typecheck && npm --workspace @loopover/ui-miner run typecheck",
     "preui:test": "npm run check-node-version",
     "ui:test": "npm run ui:kit:build && npm --workspace @loopover/ui run test && npm --workspace @loopover/ui-miner run test",
     "ui:openapi": "tsx scripts/write-ui-openapi.ts",
@@ -121,7 +122,7 @@
     "changelog:check:mcp": "node --experimental-strip-types scripts/check-changelog.ts --mcp",
     "mcp:release-due": "tsx scripts/check-mcp-release-due.ts --json",
     "mcp:release-candidate": "tsx scripts/check-mcp-release-candidate.ts",
-    "typecheck": "npm run typecheck:root && npm run typecheck:packages",
+    "typecheck": "npm run typecheck:root && npm run typecheck:packages && npm run ui:typecheck",
     "typecheck:root": "tsc --noEmit",
     "typecheck:packages": "tsc -p packages/loopover-contract/tsconfig.json --noEmit && tsc -p packages/loopover-mcp/tsconfig.json --noEmit && tsc -p packages/loopover-miner/tsconfig.json --noEmit",
     "check-node-version": "node --experimental-strip-types scripts/check-node-version.ts",
@@ -145,7 +146,7 @@
     "test:smoke:browser:install": "playwright install chromium",
     "test:smoke:browser": "node --experimental-strip-types scripts/smoke-ui-browser.ts",
     "pretest:ci": "npm run check-node-version",
-    "test:ci": "git diff --check && npm run actionlint && npm run lint:composite-actions && npm run db:migrations:check && npm run db:migrations:immutable:check && npm run turbo-inputs:check && npm run workspace-dep-ranges:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run miner:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run build --workspace @loopover/engine && npm run build --workspace @loopover/discovery-index && npm run build:mcp && npm run build:miner && npm run build --workspace @loopover/ui-kit && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:live-gate-parity && npm run test:driver-parity && npm run validate:mcp && npm run test --workspace @loopover/engine && npm run test:workers && npm run test:mcp-pack && npm run test:contract-pack && npm run test:miner-pack && npm run test:engine-pack && npm run test:ui-kit-pack && npm run test:miner-deployment-docs-audit && npm run rees:test && npm run ui:openapi:check && npm run control-plane:contract:check && npm run control-plane:openapi:check && npm run ui:version-audit && npm run docs:drift-check && npm run coverage-boltons:check && npm run import-specifiers:check && npm run ui-derived-types:check && npm run server-manifest:check && npm run dead-source-files:check && npm run dead-exports:check && npm run publishable-deps:check && npm run fixture-clock-races:check && npm run checkers-wired:check && npm run release-commit-parsing:check && npm run regate-sort-key:check && npm run maintainer-associations:check && npm run command-redelivery-guards:check && npm run dispatch-gate-reasons:check && npm run validate:no-hand-written-js && npm run replay-runner-manifest:check && npm run coco-dev-versions:check && npm run branding-drift:check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run engines-nvmrc:check && npm run release-manifest:sync:check && npm run release-linked-versions:check && npm run command-reference:check && npm run mcp:tool-reference:check && npm run mcp:client-config:check && npm run contract:api-schemas:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build",
+    "test:ci": "git diff --check && npm run actionlint && npm run lint:composite-actions && npm run db:migrations:check && npm run db:migrations:immutable:check && npm run turbo-inputs:check && npm run workspace-dep-ranges:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run miner:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run build --workspace @loopover/engine && npm run build --workspace @loopover/discovery-index && npm run build:mcp && npm run build:miner && npm run build --workspace @loopover/ui-kit && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:live-gate-parity && npm run test:driver-parity && npm run validate:mcp && npm run test --workspace @loopover/engine && npm run test:workers && npm run test:mcp-pack && npm run test:contract-pack && npm run test:miner-pack && npm run test:engine-pack && npm run test:ui-kit-pack && npm run test:miner-deployment-docs-audit && npm run rees:test && npm run ui:openapi:check && npm run control-plane:contract:check && npm run control-plane:openapi:check && npm run ui:version-audit && npm run docs:drift-check && npm run coverage-boltons:check && npm run import-specifiers:check && npm run ui-derived-types:check && npm run server-manifest:check && npm run dead-source-files:check && npm run dead-exports:check && npm run publishable-deps:check && npm run fixture-clock-races:check && npm run typecheck-coverage:check && npm run checkers-wired:check && npm run release-commit-parsing:check && npm run regate-sort-key:check && npm run maintainer-associations:check && npm run command-redelivery-guards:check && npm run dispatch-gate-reasons:check && npm run validate:no-hand-written-js && npm run replay-runner-manifest:check && npm run coco-dev-versions:check && npm run branding-drift:check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run engines-nvmrc:check && npm run release-manifest:sync:check && npm run release-linked-versions:check && npm run command-reference:check && npm run mcp:tool-reference:check && npm run mcp:client-config:check && npm run contract:api-schemas:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build",
     "test:release": "npm run test:ci && npm run changelog:check",
     "test:release:mcp": "npm run test:ci",
     "test:watch": "vitest",
diff --git a/scripts/check-typecheck-coverage.ts b/scripts/check-typecheck-coverage.ts
new file mode 100644
index 0000000000..f20ba9c997
--- /dev/null
+++ b/scripts/check-typecheck-coverage.ts
@@ -0,0 +1,129 @@
+#!/usr/bin/env node
+// `npm run typecheck` must actually typecheck everything that can be typechecked (#9860).
+//
+// THE INCIDENT. #9815 turned main red because `ChatActionDispatchResult` was closed into a union and a
+// miner-UI mock stopped satisfying it. The change was validated locally with `npm run typecheck`, which
+// passed -- because `ui:typecheck` was in `test:ci` but NOT in the root `typecheck` chain. A contributor
+// running the obvious command got a green result on a tree that does not compile.
+//
+// That is the worst shape a check can have: not missing, but SILENTLY PARTIAL. A missing check is noticed
+// the first time something breaks; a partial one is trusted precisely because it passes.
+//
+// WHAT THIS COMPUTES. Every workspace that declares its own `typecheck` script, and whether the root
+// `typecheck` script reaches it -- following `npm run <script>` references transitively, so a workspace
+// covered through an intermediate script (`ui:typecheck` -> `npm --workspace @loopover/ui run typecheck`)
+// counts as covered. Anything declaring a typecheck nobody runs is reported.
+//
+// This is #9853's bar applied to one more hand-maintained list: compute the fact rather than remember it.
+// The failure that motivated the issue was not "someone forgot to add it" -- it was that nothing could tell
+// them they had forgotten.
+import { readFileSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath, URL } from "node:url";
+
+export type TypecheckGap = { workspace: string; script: string };
+
+/** Every `npm run <name>` this script body invokes (the root package's own scripts). */
+function referencedRootScripts(body: string): string[] {
+  // `npm run x`, `npm run x --silent`, `npm --silent run x` -- all forms used in this package.json.
+  return [...body.matchAll(/npm\s+(?:--\S+\s+)*run\s+([\w:.-]+)/g)].map((match) => match[1]).filter((name): name is string => Boolean(name));
+}
+
+/** Every workspace whose OWN `typecheck` this script body invokes directly. */
+function referencedWorkspaces(body: string): string[] {
+  // `npm --workspace @scope/name run typecheck` and `npm run typecheck --workspace @scope/name`.
+  const names = [
+    ...body.matchAll(/npm\s+--workspace[= ]\s*(\S+)\s+run\s+([\w:.-]+)/g),
+    ...body.matchAll(/npm\s+run\s+([\w:.-]+)\s+--workspace[= ]\s*(\S+)/g),
+  ];
+  const out: string[] = [];
+  for (const match of names) {
+    // The two patterns capture (workspace, script) and (script, workspace) respectively; the workspace is
+    // whichever capture looks like a package name.
+    const [a, b] = [match[1], match[2]];
+    const workspace = a?.startsWith("@") || a?.includes("/") ? a : b;
+    const script = workspace === a ? b : a;
+    if (workspace && script === "typecheck") out.push(workspace);
+  }
+  return out;
+}
+
+/**
+ * PURE: workspaces that declare a `typecheck` script the root `typecheck` never reaches.
+ *
+ * `scripts` is the root package's script map; `workspacesWithTypecheck` is every workspace package name that
+ * declares one. Reachability follows `npm run` references transitively from `entry`, because a workspace is
+ * covered whether it is invoked directly or through an intermediate script.
+ */
+export function findTypecheckGaps(
+  scripts: Readonly<Record<string, string>>,
+  workspacesWithTypecheck: readonly string[],
+  entry = "typecheck",
+): TypecheckGap[] {
+  const covered = new Set<string>();
+  const seen = new Set<string>();
+  const queue: string[] = [entry];
+  while (queue.length > 0) {
+    const name = queue.shift();
+    if (!name || seen.has(name)) continue;
+    seen.add(name);
+    const body = scripts[name];
+    if (body === undefined) continue;
+    for (const workspace of referencedWorkspaces(body)) covered.add(workspace);
+    // A `tsc -p packages/<x>/tsconfig.json` counts too: the root chain typechecks that project directly
+    // without going through the workspace's own script.
+    for (const match of body.matchAll(/-p\s+((?:packages|apps)\/[\w.-]+)\//g)) {
+      const dir = match[1];
+      if (dir) covered.add(dir);
+    }
+    queue.push(...referencedRootScripts(body));
+  }
+  return workspacesWithTypecheck
+    .filter((workspace) => !covered.has(workspace) && !covered.has(workspace.replace(/^@[\w-]+\//, "")))
+    .map((workspace) => ({ workspace, script: "typecheck" }));
+}
+
+/** Workspace package names (and their directories) that declare their own `typecheck` script. */
+export function workspacesDeclaringTypecheck(root: string): { name: string; dir: string }[] {
+  const out: { name: string; dir: string }[] = [];
+  for (const group of ["apps", "packages"]) {
+    let dirs: string[];
+    try {
+      dirs = readdirSync(join(root, group), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name);
+    } catch {
+      continue;
+    }
+    for (const dir of dirs) {
+      try {
+        const manifest = JSON.parse(readFileSync(join(root, group, dir, "package.json"), "utf8")) as { name?: string; scripts?: Record<string, string> };
+        if (manifest.name && manifest.scripts?.typecheck) out.push({ name: manifest.name, dir: `${group}/${dir}` });
+      } catch {
+        // not a workspace package
+      }
+    }
+  }
+  return out;
+}
+
+function main(): void {
+  const root = join(fileURLToPath(new URL(".", import.meta.url)), "..");
+  const rootManifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as { scripts?: Record<string, string> };
+  const declared = workspacesDeclaringTypecheck(root);
+  const gaps = findTypecheckGaps(rootManifest.scripts ?? {}, declared.map((entry) => entry.name));
+
+  if (gaps.length > 0) {
+    console.error("`npm run typecheck` does not reach every workspace that declares one:\n");
+    for (const gap of gaps) console.error(`  ${gap.workspace}  (declares "${gap.script}", never invoked)`);
+    console.error(
+      "\n  A typecheck that passes while part of the tree does not compile is worse than no typecheck: it is\n" +
+        "  trusted BECAUSE it passes. #9815 turned main red exactly this way -- the change was validated with\n" +
+        "  `npm run typecheck`, which did not cover apps/**.\n\n" +
+        "  Fix: chain the workspace into the root `typecheck` script (directly, or through one it already\n" +
+        "  calls), so the obvious command means what a contributor assumes it means.",
+    );
+    process.exit(1);
+  }
+  console.log(`typecheck-coverage: OK — all ${declared.length} workspace typecheck script(s) are reachable from \`npm run typecheck\`.`);
+}
+
+if (process.argv[1]?.endsWith("check-typecheck-coverage.ts")) main();
diff --git a/test/integration/public-eval-scores-route.test.ts b/test/integration/public-eval-scores-route.test.ts
index 4c347df818..9a030b43ff 100644
--- a/test/integration/public-eval-scores-route.test.ts
+++ b/test/integration/public-eval-scores-route.test.ts
@@ -12,11 +12,21 @@ const NOW = Date.parse("2026-07-27T12:00:00.000Z");
 async function seedConfirmedPrecisionData(env: Env): Promise<void> {
   const store = createSignalStore(env);
   for (let i = 0; i < 20; i += 1) {
+    const occurredAt = new Date(NOW - 1000 - i).toISOString();
+    // #9966 made the DOWNLOADABLE corpus the only commitment source, so a record is publishable only where
+    // /v1/public/eval-corpus can actually serve the cases behind it. That corpus is built from rule-FIRED
+    // events joined to their overrides, and this fixture recorded only the overrides -- so it produced a
+    // 0-case corpus, no commitment, and no record, which is why this suite went red on main.
+    //
+    // Seeding both halves restores what the fixture was always meant to represent: 20 decided cases a reader
+    // can download and re-hash. Asserting `records: []` instead would have kept the suite green while
+    // silently dropping the recordDigest-recomputability check these tests exist for.
+    await store.recordRuleFired({ ruleId: "ai_consensus_defect", targetKey: `acme/widgets#${i + 1}`, outcome: "close", occurredAt });
     await store.recordHumanOverride({
       ruleId: "ai_consensus_defect",
       targetKey: `acme/widgets#${i + 1}`,
       verdict: i < 16 ? "confirmed" : "reversed",
-      occurredAt: new Date(NOW - 1000 - i).toISOString(),
+      occurredAt,
     });
   }
   await recordAuditEvent(env, {
@@ -59,7 +69,14 @@ describe("GET /v1/public/eval-scores (#9266, epic #8534, spec #9215)", () => {
     // coverage is 1, not null: abstained is structurally 0 for this work-unit kind, so the record's own
     // decided/(decided+abstained) is fully determined and the published field states it (#9643).
     expect(record?.score).toEqual({ decided: 20, confirmed: 16, precision: 0.8, recall: null, coverage: 1, abstained: 0 });
-    expect(record?.commitments.corpusChecksum).toBe("freeze-point-checksum");
+    // #9966: the commitment is the checksum of the corpus a reader can DOWNLOAD, not the persisted backtest
+    // run's freeze point. That is the whole point of the change -- a commitment must name bytes someone can
+    // fetch and re-hash. Asserted against what /v1/public/eval-corpus actually serves rather than a literal,
+    // so the two surfaces cannot drift apart while both still look correct in isolation.
+    const corpusRes = await createApp().request("/v1/public/eval-corpus?ruleId=ai_consensus_defect", {}, env);
+    const corpus = (await corpusRes.json()) as { checksum: string; caseCount: number };
+    expect(corpus.caseCount).toBe(20);
+    expect(record?.commitments.corpusChecksum).toBe(corpus.checksum);
     expect(record?.subject).toEqual({ kind: "agent", id: ORB_GATE_SUBJECT_ID });
 
     const { recordDigest, ...rest } = record as EvalScoreRecord;
diff --git a/test/unit/check-typecheck-coverage-script.test.ts b/test/unit/check-typecheck-coverage-script.test.ts
new file mode 100644
index 0000000000..631d351748
--- /dev/null
+++ b/test/unit/check-typecheck-coverage-script.test.ts
@@ -0,0 +1,69 @@
+// The typecheck-coverage checker must catch a workspace the root chain never reaches (#9860).
+//
+// The failure it guards is specific: `npm run typecheck` PASSING while part of the tree does not compile.
+// That is worse than having no check, because the green result is trusted precisely because it is green --
+// #9815 turned main red exactly that way. So the positive case here is "a declared typecheck nobody runs is
+// reported", and the negative cases pin every legitimate way a workspace can be covered, since a checker
+// that reports a covered workspace would be noise and would get muted.
+import { describe, expect, it } from "vitest";
+
+import { findTypecheckGaps } from "../../scripts/check-typecheck-coverage";
+
+describe("findTypecheckGaps (#9860)", () => {
+  it("REGRESSION: reports the exact #9815 shape — a UI workspace the root chain never invokes", () => {
+    const scripts = {
+      typecheck: "npm run typecheck:root && npm run typecheck:packages",
+      "typecheck:root": "tsc --noEmit",
+      "typecheck:packages": "tsc -p packages/loopover-contract/tsconfig.json --noEmit",
+      // Present in the repo, but reachable only from test:ci -- never from `typecheck`.
+      "ui:typecheck": "npm --workspace @loopover/ui run typecheck",
+    };
+    expect(findTypecheckGaps(scripts, ["@loopover/ui"])).toEqual([{ workspace: "@loopover/ui", script: "typecheck" }]);
+  });
+
+  it("counts a workspace reached THROUGH an intermediate script as covered", () => {
+    // The fix chains ui:typecheck into typecheck; the workspace is then covered transitively, not directly.
+    const scripts = {
+      typecheck: "npm run typecheck:root && npm run ui:typecheck",
+      "typecheck:root": "tsc --noEmit",
+      "ui:typecheck": "npm run ui:kit:build && npm --workspace @loopover/ui run typecheck",
+      "ui:kit:build": "npm --workspace @loopover/ui-kit run build",
+    };
+    expect(findTypecheckGaps(scripts, ["@loopover/ui"])).toEqual([]);
+  });
+
+  it("counts a project typechecked directly by path as covered, without its own script being invoked", () => {
+    // `typecheck:packages` runs tsc against the project file rather than calling the workspace's script.
+    const scripts = {
+      typecheck: "npm run typecheck:packages",
+      "typecheck:packages": "tsc -p packages/loopover-contract/tsconfig.json --noEmit",
+    };
+    expect(findTypecheckGaps(scripts, ["packages/loopover-contract"])).toEqual([]);
+  });
+
+  it("handles the reversed flag order, since both spellings appear in this package.json", () => {
+    const scripts = { typecheck: "npm run typecheck --workspace @loopover/ui" };
+    expect(findTypecheckGaps(scripts, ["@loopover/ui"])).toEqual([]);
+  });
+
+  it("does NOT count a workspace whose BUILD is run but whose typecheck is not", () => {
+    // A build may typecheck as a side effect, but that is a property of that script's current body, not a
+    // guarantee. Treating it as coverage would silently accept a build that later stops emitting types.
+    const scripts = { typecheck: "npm run ui:kit:build", "ui:kit:build": "npm --workspace @loopover/ui-kit run build" };
+    expect(findTypecheckGaps(scripts, ["@loopover/ui-kit"])).toEqual([{ workspace: "@loopover/ui-kit", script: "typecheck" }]);
+  });
+
+  it("terminates on a cyclic script graph instead of looping forever", () => {
+    const scripts = { typecheck: "npm run a", a: "npm run b", b: "npm run a" };
+    expect(findTypecheckGaps(scripts, ["@loopover/ui"])).toEqual([{ workspace: "@loopover/ui", script: "typecheck" }]);
+  });
+
+  it("reports nothing when no workspace declares a typecheck at all", () => {
+    expect(findTypecheckGaps({ typecheck: "tsc --noEmit" }, [])).toEqual([]);
+  });
+
+  it("tolerates a missing entry script rather than throwing", () => {
+    // A renamed root script must fail loudly as a REPORT, not as a crash mid-CI.
+    expect(findTypecheckGaps({}, ["@loopover/ui"])).toEqual([{ workspace: "@loopover/ui", script: "typecheck" }]);
+  });
+});