Skip to content

fix(test): unbreak main — seed the downloadable corpus #9966 now requires #9974

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JSONbored merged 1 commit into from
Jul 31, 2026
+221 −5
Merged

fix(test): unbreak main — seed the downloadable corpus #9966 now requires #9974

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
"db:migrations:check": "tsx scripts/check-migrations.ts",
"db:migrations:immutable:check": "tsx scripts/check-released-migrations-immutable.ts",
"turbo-inputs:check": "tsx scripts/check-turbo-typecheck-inputs.ts",
"typecheck-coverage:check": "tsx scripts/check-typecheck-coverage.ts",
"fixture-clock-races:check": "tsx scripts/check-fixture-clock-races.ts",
"workspace-dep-ranges:check": "tsx scripts/check-workspace-dep-ranges.ts",
"db:schema-drift:check": "tsx scripts/check-schema-drift.ts",
Expand All @@ -70,7 +71,7 @@
"ui:build": "npm run ui:kit:build && turbo run build --filter=@loopover/engine --filter=@loopover/contract && npm run ui:openapi && npm --workspace @loopover/ui run build && npm --workspace @loopover/ui-miner run build",
"ui:preview": "npm run ui:build && wrangler dev --config apps/loopover-ui/dist/server/wrangler.json --ip 127.0.0.1 --port 4173 --local",
"ui:lint": "npm run ui:kit:build && npm --workspace @loopover/ui-kit run format:check && npm --workspace @loopover/ui run format:check && npm --workspace @loopover/ui run lint && npm --workspace @loopover/ui-miner run format:check && npm --workspace @loopover/ui-miner run lint",
"ui:typecheck": "npm run ui:kit:build && npm --workspace @loopover/ui run typecheck && npm --workspace @loopover/ui-miner run typecheck",
"ui:typecheck": "npm run ui:kit:build && npm --workspace @loopover/ui-kit run typecheck && npm --workspace @loopover/ui run typecheck && npm --workspace @loopover/ui-miner run typecheck",
"preui:test": "npm run check-node-version",
"ui:test": "npm run ui:kit:build && npm --workspace @loopover/ui run test && npm --workspace @loopover/ui-miner run test",
"ui:openapi": "tsx scripts/write-ui-openapi.ts",
Expand Down Expand Up @@ -121,7 +122,7 @@
"changelog:check:mcp": "node --experimental-strip-types scripts/check-changelog.ts --mcp",
"mcp:release-due": "tsx scripts/check-mcp-release-due.ts --json",
"mcp:release-candidate": "tsx scripts/check-mcp-release-candidate.ts",
"typecheck": "npm run typecheck:root && npm run typecheck:packages",
"typecheck": "npm run typecheck:root && npm run typecheck:packages && npm run ui:typecheck",
"typecheck:root": "tsc --noEmit",
"typecheck:packages": "tsc -p packages/loopover-contract/tsconfig.json --noEmit && tsc -p packages/loopover-mcp/tsconfig.json --noEmit && tsc -p packages/loopover-miner/tsconfig.json --noEmit",
"check-node-version": "node --experimental-strip-types scripts/check-node-version.ts",
Expand All @@ -145,7 +146,7 @@
"test:smoke:browser:install": "playwright install chromium",
"test:smoke:browser": "node --experimental-strip-types scripts/smoke-ui-browser.ts",
"pretest:ci": "npm run check-node-version",
"test:ci": "git diff --check && npm run actionlint && npm run lint:composite-actions && npm run db:migrations:check && npm run db:migrations:immutable:check && npm run turbo-inputs:check && npm run workspace-dep-ranges:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run miner:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run build --workspace @loopover/engine && npm run build --workspace @loopover/discovery-index && npm run build:mcp && npm run build:miner && npm run build --workspace @loopover/ui-kit && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:live-gate-parity && npm run test:driver-parity && npm run validate:mcp && npm run test --workspace @loopover/engine && npm run test:workers && npm run test:mcp-pack && npm run test:contract-pack && npm run test:miner-pack && npm run test:engine-pack && npm run test:ui-kit-pack && npm run test:miner-deployment-docs-audit && npm run rees:test && npm run ui:openapi:check && npm run control-plane:contract:check && npm run control-plane:openapi:check && npm run ui:version-audit && npm run docs:drift-check && npm run coverage-boltons:check && npm run import-specifiers:check && npm run ui-derived-types:check && npm run server-manifest:check && npm run dead-source-files:check && npm run dead-exports:check && npm run publishable-deps:check && npm run fixture-clock-races:check && npm run checkers-wired:check && npm run release-commit-parsing:check && npm run regate-sort-key:check && npm run maintainer-associations:check && npm run command-redelivery-guards:check && npm run dispatch-gate-reasons:check && npm run validate:no-hand-written-js && npm run replay-runner-manifest:check && npm run coco-dev-versions:check && npm run branding-drift:check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run engines-nvmrc:check && npm run release-manifest:sync:check && npm run release-linked-versions:check && npm run command-reference:check && npm run mcp:tool-reference:check && npm run mcp:client-config:check && npm run contract:api-schemas:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build",
"test:ci": "git diff --check && npm run actionlint && npm run lint:composite-actions && npm run db:migrations:check && npm run db:migrations:immutable:check && npm run turbo-inputs:check && npm run workspace-dep-ranges:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run miner:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run build --workspace @loopover/engine && npm run build --workspace @loopover/discovery-index && npm run build:mcp && npm run build:miner && npm run build --workspace @loopover/ui-kit && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:live-gate-parity && npm run test:driver-parity && npm run validate:mcp && npm run test --workspace @loopover/engine && npm run test:workers && npm run test:mcp-pack && npm run test:contract-pack && npm run test:miner-pack && npm run test:engine-pack && npm run test:ui-kit-pack && npm run test:miner-deployment-docs-audit && npm run rees:test && npm run ui:openapi:check && npm run control-plane:contract:check && npm run control-plane:openapi:check && npm run ui:version-audit && npm run docs:drift-check && npm run coverage-boltons:check && npm run import-specifiers:check && npm run ui-derived-types:check && npm run server-manifest:check && npm run dead-source-files:check && npm run dead-exports:check && npm run publishable-deps:check && npm run fixture-clock-races:check && npm run typecheck-coverage:check && npm run checkers-wired:check && npm run release-commit-parsing:check && npm run regate-sort-key:check && npm run maintainer-associations:check && npm run command-redelivery-guards:check && npm run dispatch-gate-reasons:check && npm run validate:no-hand-written-js && npm run replay-runner-manifest:check && npm run coco-dev-versions:check && npm run branding-drift:check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run engines-nvmrc:check && npm run release-manifest:sync:check && npm run release-linked-versions:check && npm run command-reference:check && npm run mcp:tool-reference:check && npm run mcp:client-config:check && npm run contract:api-schemas:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build",
"test:release": "npm run test:ci && npm run changelog:check",
"test:release:mcp": "npm run test:ci",
"test:watch": "vitest",
Expand Down
129 changes: 129 additions & 0 deletions scripts/check-typecheck-coverage.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
#!/usr/bin/env node
// `npm run typecheck` must actually typecheck everything that can be typechecked (#9860).
//
// THE INCIDENT. #9815 turned main red because `ChatActionDispatchResult` was closed into a union and a
// miner-UI mock stopped satisfying it. The change was validated locally with `npm run typecheck`, which
// passed -- because `ui:typecheck` was in `test:ci` but NOT in the root `typecheck` chain. A contributor
// running the obvious command got a green result on a tree that does not compile.
//
// That is the worst shape a check can have: not missing, but SILENTLY PARTIAL. A missing check is noticed
// the first time something breaks; a partial one is trusted precisely because it passes.
//
// WHAT THIS COMPUTES. Every workspace that declares its own `typecheck` script, and whether the root
// `typecheck` script reaches it -- following `npm run <script>` references transitively, so a workspace
// covered through an intermediate script (`ui:typecheck` -> `npm --workspace @loopover/ui run typecheck`)
// counts as covered. Anything declaring a typecheck nobody runs is reported.
//
// This is #9853's bar applied to one more hand-maintained list: compute the fact rather than remember it.
// The failure that motivated the issue was not "someone forgot to add it" -- it was that nothing could tell
// them they had forgotten.
import { readFileSync, readdirSync } from "node:fs";
import { join } from "node:path";
import { fileURLToPath, URL } from "node:url";

export type TypecheckGap = { workspace: string; script: string };

/** Every `npm run <name>` this script body invokes (the root package's own scripts). */
function referencedRootScripts(body: string): string[] {
// `npm run x`, `npm run x --silent`, `npm --silent run x` -- all forms used in this package.json.
return [...body.matchAll(/npm\s+(?:--\S+\s+)*run\s+([\w:.-]+)/g)].map((match) => match[1]).filter((name): name is string => Boolean(name));
}

/** Every workspace whose OWN `typecheck` this script body invokes directly. */
function referencedWorkspaces(body: string): string[] {
// `npm --workspace @scope/name run typecheck` and `npm run typecheck --workspace @scope/name`.
const names = [
...body.matchAll(/npm\s+--workspace[= ]\s*(\S+)\s+run\s+([\w:.-]+)/g),
...body.matchAll(/npm\s+run\s+([\w:.-]+)\s+--workspace[= ]\s*(\S+)/g),
];
const out: string[] = [];
for (const match of names) {
// The two patterns capture (workspace, script) and (script, workspace) respectively; the workspace is
// whichever capture looks like a package name.
const [a, b] = [match[1], match[2]];
const workspace = a?.startsWith("@") || a?.includes("/") ? a : b;
const script = workspace === a ? b : a;
if (workspace && script === "typecheck") out.push(workspace);
}
return out;
}

/**
* PURE: workspaces that declare a `typecheck` script the root `typecheck` never reaches.
*
* `scripts` is the root package's script map; `workspacesWithTypecheck` is every workspace package name that
* declares one. Reachability follows `npm run` references transitively from `entry`, because a workspace is
* covered whether it is invoked directly or through an intermediate script.
*/
export function findTypecheckGaps(
scripts: Readonly<Record<string, string>>,
workspacesWithTypecheck: readonly string[],
entry = "typecheck",
): TypecheckGap[] {
const covered = new Set<string>();
const seen = new Set<string>();
const queue: string[] = [entry];
while (queue.length > 0) {
const name = queue.shift();
if (!name || seen.has(name)) continue;
seen.add(name);
const body = scripts[name];
if (body === undefined) continue;
for (const workspace of referencedWorkspaces(body)) covered.add(workspace);
// A `tsc -p packages/<x>/tsconfig.json` counts too: the root chain typechecks that project directly
// without going through the workspace's own script.
for (const match of body.matchAll(/-p\s+((?:packages|apps)\/[\w.-]+)\//g)) {
const dir = match[1];
if (dir) covered.add(dir);
}
queue.push(...referencedRootScripts(body));
}
return workspacesWithTypecheck
.filter((workspace) => !covered.has(workspace) && !covered.has(workspace.replace(/^@[\w-]+\//, "")))
.map((workspace) => ({ workspace, script: "typecheck" }));
}

/** Workspace package names (and their directories) that declare their own `typecheck` script. */
export function workspacesDeclaringTypecheck(root: string): { name: string; dir: string }[] {
const out: { name: string; dir: string }[] = [];
for (const group of ["apps", "packages"]) {
let dirs: string[];
try {
dirs = readdirSync(join(root, group), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name);
} catch {
continue;
}
for (const dir of dirs) {
try {
const manifest = JSON.parse(readFileSync(join(root, group, dir, "package.json"), "utf8")) as { name?: string; scripts?: Record<string, string> };
if (manifest.name && manifest.scripts?.typecheck) out.push({ name: manifest.name, dir: `${group}/${dir}` });
} catch {
// not a workspace package
}
}
}
return out;
}

function main(): void {
const root = join(fileURLToPath(new URL(".", import.meta.url)), "..");
const rootManifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as { scripts?: Record<string, string> };
const declared = workspacesDeclaringTypecheck(root);
const gaps = findTypecheckGaps(rootManifest.scripts ?? {}, declared.map((entry) => entry.name));

if (gaps.length > 0) {
console.error("`npm run typecheck` does not reach every workspace that declares one:\n");
for (const gap of gaps) console.error(` ${gap.workspace} (declares "${gap.script}", never invoked)`);
console.error(
"\n A typecheck that passes while part of the tree does not compile is worse than no typecheck: it is\n" +
" trusted BECAUSE it passes. #9815 turned main red exactly this way -- the change was validated with\n" +
" `npm run typecheck`, which did not cover apps/**.\n\n" +
" Fix: chain the workspace into the root `typecheck` script (directly, or through one it already\n" +
" calls), so the obvious command means what a contributor assumes it means.",
);
process.exit(1);
}
console.log(`typecheck-coverage: OK — all ${declared.length} workspace typecheck script(s) are reachable from \`npm run typecheck\`.`);
}

if (process.argv[1]?.endsWith("check-typecheck-coverage.ts")) main();
21 changes: 19 additions & 2 deletions test/integration/public-eval-scores-route.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,21 @@ const NOW = Date.parse("2026-07-27T12:00:00.000Z");
async function seedConfirmedPrecisionData(env: Env): Promise<void> {
const store = createSignalStore(env);
for (let i = 0; i < 20; i += 1) {
const occurredAt = new Date(NOW - 1000 - i).toISOString();
// #9966 made the DOWNLOADABLE corpus the only commitment source, so a record is publishable only where
// /v1/public/eval-corpus can actually serve the cases behind it. That corpus is built from rule-FIRED
// events joined to their overrides, and this fixture recorded only the overrides -- so it produced a
// 0-case corpus, no commitment, and no record, which is why this suite went red on main.
//
// Seeding both halves restores what the fixture was always meant to represent: 20 decided cases a reader
// can download and re-hash. Asserting `records: []` instead would have kept the suite green while
// silently dropping the recordDigest-recomputability check these tests exist for.
await store.recordRuleFired({ ruleId: "ai_consensus_defect", targetKey: `acme/widgets#${i + 1}`, outcome: "close", occurredAt });
await store.recordHumanOverride({
ruleId: "ai_consensus_defect",
targetKey: `acme/widgets#${i + 1}`,
verdict: i < 16 ? "confirmed" : "reversed",
occurredAt: new Date(NOW - 1000 - i).toISOString(),
occurredAt,
});
}
await recordAuditEvent(env, {
Expand Down Expand Up @@ -59,7 +69,14 @@ describe("GET /v1/public/eval-scores (#9266, epic #8534, spec #9215)", () => {
// coverage is 1, not null: abstained is structurally 0 for this work-unit kind, so the record's own
// decided/(decided+abstained) is fully determined and the published field states it (#9643).
expect(record?.score).toEqual({ decided: 20, confirmed: 16, precision: 0.8, recall: null, coverage: 1, abstained: 0 });
expect(record?.commitments.corpusChecksum).toBe("freeze-point-checksum");
// #9966: the commitment is the checksum of the corpus a reader can DOWNLOAD, not the persisted backtest
// run's freeze point. That is the whole point of the change -- a commitment must name bytes someone can
// fetch and re-hash. Asserted against what /v1/public/eval-corpus actually serves rather than a literal,
// so the two surfaces cannot drift apart while both still look correct in isolation.
const corpusRes = await createApp().request("/v1/public/eval-corpus?ruleId=ai_consensus_defect", {}, env);
const corpus = (await corpusRes.json()) as { checksum: string; caseCount: number };
expect(corpus.caseCount).toBe(20);
expect(record?.commitments.corpusChecksum).toBe(corpus.checksum);
expect(record?.subject).toEqual({ kind: "agent", id: ORB_GATE_SUBJECT_ID });

const { recordDigest, ...rest } = record as EvalScoreRecord;
Expand Down
Loading
Loading