diff --git a/packages/loopover-engine/src/calibration/benchmark-score.ts b/packages/loopover-engine/src/calibration/benchmark-score.ts new file mode 100644 index 000000000..9806646f5 --- /dev/null +++ b/packages/loopover-engine/src/calibration/benchmark-score.ts @@ -0,0 +1,218 @@ +// Benchmark proposal scorer (#9262, harness #9216, epic #8534) — multi-class scoring built ON the existing +// confusion-matrix and Pareto-floor primitives, deliberately NOT beside them. +// +// REUSE, NOT REIMPLEMENTATION. `scoreBacktest` (backtest-score.ts) and `compareBacktestScores` +// (backtest-compare.ts) remain the scoring core; this module is an adapter that reshapes (proposal, ground +// truth) pairs into the exact inputs those functions already take. A second scoring implementation is +// precisely the drift this module exists to prevent, so the anti-drift guarantee is mechanical rather than +// aspirational: an equivalent binary case scored here and scored through the internal backtest path produces +// the identical report, asserted directly in the tests. +// +// The reuse works by borrowing the primitive's positive-class slot per action: for action A, the "positive" +// class is "the realized action was A" and the classifier's answer is "the agent proposed A". The primitive +// names that slot `reversed`/`confirmed` because its first caller scored rule reversals; the name is +// vestigial here and never surfaces in this module's own output. The `ruleId` slot carries the ACTION name, +// which makes each per-action report self-labeling and makes `compareBacktestScores`' rule-mismatch throw +// double as a guard against accidentally comparing two different actions. +// +// AGGREGATION — the recorded decision (#9262 requirement 2). MACRO is the headline; MICRO is published +// alongside. They answer different questions and the difference is not cosmetic here: +// +// Micro pools the counts across actions. Under single-label multi-class scoring — one prediction per work +// unit — every decided unit contributes exactly one TP or one FP, so micro precision, micro recall and +// plain accuracy are all the SAME number. That makes it an honest "how often was it right overall", and +// also makes it dominated by whichever action is most frequent. In this corpus that is overwhelmingly +// `merge`, so an agent that answers "merge" to everything scores well on micro while being worthless. +// +// Macro averages the per-action metrics, so each action counts equally regardless of frequency, and the +// answer-merge-to-everything agent is immediately exposed by its floor-level `close`/`request_changes` +// numbers. Since the benchmark's question is "can this agent make MAINTAINER decisions" — including the +// rare, expensive ones — macro is the number that answers it, and therefore the headline. +// +// Both are published because a benchmark that reports only its headline invites the reader to reconstruct +// the other one wrongly. An action with no realized instances has null metrics and is EXCLUDED from the +// macro mean rather than counted as 0 — a metric nobody could measure must not drag an average down. +// +// Same purity contract as the rest of this module family: no IO, no randomness, no wall-clock reads. + +import type { BacktestCase } from "./backtest-corpus.js"; +import { scoreBacktest, type BacktestScoreReport } from "./backtest-score.js"; +import { compareBacktestScores, type BacktestComparison } from "./backtest-compare.js"; +import type { BenchmarkActionKind, BenchmarkProposal } from "./benchmark-proposal.js"; +import { scoreableGroundTruths, type BenchmarkGroundTruthSet } from "./benchmark-ground-truth.js"; + +/** Every action scored, in a fixed order so two reports are directly comparable field by field. */ +export const SCORED_ACTIONS: readonly BenchmarkActionKind[] = ["merge", "close", "request_changes", "label", "hold"]; + +export type BenchmarkScoreReport = { + schemaVersion: 1; + snapshotRef: string; + /** WHO was scored — carried through from the proposals, opaque here exactly as in #9215's EvalScoreRecord. */ + subjectId: string; + /** One-vs-rest report per action, keyed by action, each produced by the shared `scoreBacktest`. */ + perAction: Record; + /** The HEADLINE (see the module header's recorded decision). `actionsScored` is how many actions had a + * non-null metric and therefore entered the mean — a macro number over 2 of 5 actions is a different + * claim than one over 5, and hiding that would be the kind of unexamined average this file argues against. */ + macro: { precision: number | null; recall: number | null; actionsScored: number }; + /** Pooled counts. Under single-label scoring micro precision === micro recall === accuracy; all three are + * the same number and it is reported once, honestly labeled, rather than three times as if independent. */ + micro: { precision: number | null; recall: number | null; accuracy: number | null }; + /** #9215's coverage semantics. Abstentions are NEVER folded into errors: they lower coverage, which is a + * different (and recoverable) thing than being wrong. */ + coverage: { + decided: number; + abstained: number; + /** `decided / (decided + abstained)`; null when the agent faced nothing at all — never 0, which would + * read as "answered nothing it was asked" rather than "was asked nothing". */ + coverage: number | null; + /** Ground-truth units excluded before scoring began (#9261's `unresolved`) — published so a reader can + * see the denominator shrink rather than discovering it in a footnote. */ + unresolvedExcluded: number; + /** Proposals for work units not in this snapshot's ground truth. Ignored for scoring (a submitter + * cannot inflate anything by padding), but COUNTED, because a nonzero value means the agent is + * answering a different question than the one asked. */ + unscorableProposals: number; + }; +}; + +function ratio(numerator: number, denominator: number): number | null { + return denominator > 0 ? Math.round((numerator / denominator) * 1000) / 1000 : null; +} + +/** Mean of the non-null values, or null when none are — an unmeasurable metric leaves the average rather + * than entering it as 0. */ +function macroMean(values: ReadonlyArray): { mean: number | null; counted: number } { + const present = values.filter((value): value is number => value !== null); + if (present.length === 0) return { mean: null, counted: 0 }; + return { mean: Math.round((present.reduce((sum, value) => sum + value, 0) / present.length) * 1000) / 1000, counted: present.length }; +} + +/** + * Score one agent's proposals for one snapshot against #9261's realized ground truth. + * + * Denominator discipline, in the order it is applied: + * 1. `unresolved` ground truth leaves entirely (#9261) — never a correct abstention, never an error. + * 2. A scoreable unit the agent DID NOT answer counts as an abstention, identically to an explicit + * `{kind: "abstain"}`. Silence and a declared abstention are the same act; scoring them differently + * would reward whichever one an agent's emitter happened to produce. + * 3. Abstentions lower coverage and are absent from every confusion-matrix count. + */ +export function scoreBenchmarkProposals(input: { + subjectId: string; + groundTruth: BenchmarkGroundTruthSet; + proposals: readonly BenchmarkProposal[]; +}): BenchmarkScoreReport { + const scoreable = scoreableGroundTruths(input.groundTruth); + const scoreableIds = new Set(scoreable.map((truth) => truth.workUnitId)); + + // Last proposal per work unit wins, so a resubmission is a correction rather than a double entry. + const proposalByUnit = new Map(); + let unscorableProposals = 0; + for (const proposal of input.proposals) { + if (!scoreableIds.has(proposal.workUnitId)) { + unscorableProposals += 1; + continue; + } + proposalByUnit.set(proposal.workUnitId, proposal); + } + + // The decided set: scoreable units the agent actually answered with an action. + const decided: Array<{ realized: BenchmarkActionKind; predicted: BenchmarkActionKind }> = []; + let abstained = 0; + for (const truth of scoreable) { + const proposal = proposalByUnit.get(truth.workUnitId); + if (!proposal || proposal.prediction.kind === "abstain") { + abstained += 1; + continue; + } + decided.push({ realized: truth.action, predicted: proposal.prediction.action.kind }); + } + + // One-vs-rest through the SHARED primitive. The synthetic cases carry the action in the `ruleId` slot so + // each report is self-labeling and a cross-action comparison throws rather than silently succeeding. + const perAction = {} as Record; + for (const action of SCORED_ACTIONS) { + const cases: BacktestCase[] = decided.map((pair, index) => ({ + ruleId: action, + targetKey: String(index), + outcome: pair.realized, + label: pair.realized === action ? "reversed" : "confirmed", + firedAt: input.groundTruth.frozenAt, + decidedAt: input.groundTruth.horizonEnd, + metadata: { predicted: pair.predicted }, + })); + perAction[action] = scoreBacktest(action, cases, (backtestCase) => + (backtestCase.metadata as { predicted: BenchmarkActionKind }).predicted === action ? "reversed" : "confirmed", + ); + } + + const macroPrecision = macroMean(SCORED_ACTIONS.map((action) => perAction[action].precision)); + const macroRecall = macroMean(SCORED_ACTIONS.map((action) => perAction[action].recall)); + // Pooled: exactly one TP or FP per decided unit, so this single number IS precision, recall and accuracy. + const correct = decided.filter((pair) => pair.predicted === pair.realized).length; + const micro = ratio(correct, decided.length); + + return { + schemaVersion: 1, + snapshotRef: input.groundTruth.snapshotRef, + subjectId: input.subjectId, + perAction, + macro: { + precision: macroPrecision.mean, + recall: macroRecall.mean, + // The two means are taken over the same actions whenever both are defined; report the precision + // side's count, which is the one the headline precision is an average of. + actionsScored: macroPrecision.counted, + }, + micro: { precision: micro, recall: micro, accuracy: micro }, + coverage: { + decided: decided.length, + abstained, + coverage: ratio(decided.length, decided.length + abstained), + unresolvedExcluded: input.groundTruth.coverage.unresolved, + unscorableProposals, + }, + }; +} + +export type BenchmarkComparison = { + subjectId: string; + perAction: Record; + regressedActions: BenchmarkActionKind[]; + improvedActions: BenchmarkActionKind[]; + /** The Pareto floor, extended to the multi-class case (#9262 requirement 4): ANY regressed action decides + * the verdict, even alongside improvements elsewhere. Gaining on `merge` while losing on `close` is a + * trade, and the floor's entire purpose is that a trade is not a win. */ + verdict: "improved" | "regressed" | "unchanged"; +}; + +/** + * Compare two benchmark score reports under the Pareto floor, per action, via the SHARED comparator. + * + * Every per-action verdict comes from `compareBacktestScores`, so the null-handling ("unknown stays + * unknown": an axis with a null on either side is excluded from both lists) is inherited rather than + * re-derived. Throws on a subject mismatch — comparing two different agents' reports as if they were one + * agent's before/after is a caller bug, the same posture the primitive takes on a rule mismatch. + */ +export function compareBenchmarkScores(baseline: BenchmarkScoreReport, candidate: BenchmarkScoreReport): BenchmarkComparison { + if (baseline.subjectId !== candidate.subjectId) { + throw new Error(`cannot compare benchmark scores for different subjects: ${baseline.subjectId} vs ${candidate.subjectId}`); + } + const perAction = {} as Record; + const regressedActions: BenchmarkActionKind[] = []; + const improvedActions: BenchmarkActionKind[] = []; + for (const action of SCORED_ACTIONS) { + const comparison = compareBacktestScores(baseline.perAction[action], candidate.perAction[action]); + perAction[action] = comparison; + if (comparison.verdict === "regressed") regressedActions.push(action); + else if (comparison.verdict === "improved") improvedActions.push(action); + } + return { + subjectId: baseline.subjectId, + perAction, + regressedActions, + improvedActions, + verdict: regressedActions.length > 0 ? "regressed" : improvedActions.length > 0 ? "improved" : "unchanged", + }; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 0ddea29fb..5989875d6 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -185,6 +185,7 @@ export * from "./calibration/attestation-envelope.js"; export * from "./calibration/attester.js"; export * from "./calibration/benchmark-proposal.js"; export * from "./calibration/benchmark-ground-truth.js"; +export * from "./calibration/benchmark-score.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/benchmark-score.test.ts b/packages/loopover-engine/test/benchmark-score.test.ts new file mode 100644 index 000000000..8d4a4f66b --- /dev/null +++ b/packages/loopover-engine/test/benchmark-score.test.ts @@ -0,0 +1,258 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + compareBenchmarkScores, + deriveBenchmarkGroundTruth, + scoreBacktest, + scoreBenchmarkProposals, + SCORED_ACTIONS, + type BenchmarkAction, + type BenchmarkActionKind, + type BenchmarkProposal, + type BenchmarkScoreReport, +} from "../dist/index.js"; + +// #9262 (harness #9216, epic #8534): multi-class scoring built ON the shared confusion-matrix and +// Pareto-floor primitives. The load-bearing test here is the ANTI-DRIFT guard — an equivalent binary case +// scored through this adapter and through `scoreBacktest` directly must produce the identical report, which +// is what makes "benchmark scores are the same kind of number as the internal backtest's" mechanical rather +// than a claim in a comment. + +const FROZEN = "2026-07-01T00:00:00.000Z"; +const SNAPSHOT = "a1".repeat(32); +const SUBJECT = "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty"; + +function at(days: number): string { + return new Date(Date.parse(FROZEN) + days * 86_400_000).toISOString(); +} + +/** Ground truth from a compact [workUnitId, realizedAction] list; `null` means no action (unresolved). */ +function truthFrom(pairs: ReadonlyArray<[string, BenchmarkActionKind | null]>) { + return deriveBenchmarkGroundTruth({ + snapshotRef: SNAPSHOT, + frozenAt: FROZEN, + horizonDays: 14, + workUnitIds: pairs.map(([id]) => id), + events: pairs.flatMap(([id, action]) => (action ? [{ workUnitId: id, action, occurredAt: at(2) }] : [])), + }); +} + +/** Proposals from a compact [workUnitId, predictedAction | "abstain"] list. */ +function proposalsFrom(pairs: ReadonlyArray<[string, BenchmarkActionKind | "abstain"]>): BenchmarkProposal[] { + return pairs.map(([workUnitId, choice]) => ({ + schemaVersion: 1, + benchmarkId: "bench-1", + snapshotRef: SNAPSHOT, + workUnitId, + subject: { kind: "agent", id: SUBJECT }, + prediction: choice === "abstain" ? { kind: "abstain" as const } : { kind: "act" as const, action: actionOf(choice) }, + })); +} + +function actionOf(kind: BenchmarkActionKind): BenchmarkAction { + if (kind === "close") return { kind, reasonClass: "defective" }; + if (kind === "request_changes") return { kind, blockingConcern: "concern" }; + if (kind === "label") return { kind, labels: ["bug"] }; + return { kind }; +} + +test("ANTI-DRIFT: an equivalent binary case scores identically through this adapter and through scoreBacktest", () => { + // Two realized merges and two realized closes; the agent gets one of each right. Scored here as the + // one-vs-rest `merge` report, and independently through the shared primitive on the same four cases. + const groundTruth = truthFrom([["u1", "merge"], ["u2", "merge"], ["u3", "close"], ["u4", "close"]]); + const proposals = proposalsFrom([["u1", "merge"], ["u2", "close"], ["u3", "merge"], ["u4", "close"]]); + const report = scoreBenchmarkProposals({ subjectId: SUBJECT, groundTruth, proposals }); + + const predictions: BenchmarkActionKind[] = ["merge", "close", "merge", "close"]; + const realized: BenchmarkActionKind[] = ["merge", "merge", "close", "close"]; + const direct = scoreBacktest( + "merge", + realized.map((actual, index) => ({ + ruleId: "merge", + targetKey: String(index), + outcome: actual, + label: actual === "merge" ? ("reversed" as const) : ("confirmed" as const), + firedAt: FROZEN, + decidedAt: groundTruth.horizonEnd, + metadata: { index }, + })), + (backtestCase) => (predictions[(backtestCase.metadata as { index: number }).index] === "merge" ? "reversed" : "confirmed"), + ); + assert.deepEqual(report.perAction.merge, direct); + // And the counts are the ones a human would write down by hand: TP=u1, FP=u3, TN=u4, FN=u2. + assert.deepEqual( + { tp: direct.truePositive, fp: direct.falsePositive, tn: direct.trueNegative, fn: direct.falseNegative }, + { tp: 1, fp: 1, tn: 1, fn: 1 }, + ); +}); + +test("scores every action one-vs-rest, each report self-labeled by its action", () => { + const groundTruth = truthFrom([["u1", "merge"], ["u2", "close"], ["u3", "request_changes"], ["u4", "label"], ["u5", "hold"]]); + const report = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth, + proposals: proposalsFrom([["u1", "merge"], ["u2", "close"], ["u3", "request_changes"], ["u4", "label"], ["u5", "hold"]]), + }); + for (const action of SCORED_ACTIONS) { + assert.equal(report.perAction[action].ruleId, action); + assert.equal(report.perAction[action].caseCount, 5); + assert.equal(report.perAction[action].precision, 1); + assert.equal(report.perAction[action].recall, 1); + } + assert.deepEqual(report.macro, { precision: 1, recall: 1, actionsScored: 5 }); + assert.deepEqual(report.micro, { precision: 1, recall: 1, accuracy: 1 }); +}); + +test("REGRESSION: macro exposes the answer-merge-to-everything agent that micro flatters", () => { + // Eight merges, two closes — the skew the module header warns about. The agent answers "merge" always. + const pairs: Array<[string, BenchmarkActionKind]> = [ + ["u1", "merge"], ["u2", "merge"], ["u3", "merge"], ["u4", "merge"], + ["u5", "merge"], ["u6", "merge"], ["u7", "merge"], ["u8", "merge"], + ["u9", "close"], ["u10", "close"], + ]; + const report = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth: truthFrom(pairs), + proposals: proposalsFrom(pairs.map(([id]) => [id, "merge"] as [string, BenchmarkActionKind])), + }); + // Micro (=accuracy) says 80% — respectable-looking for an agent that never made a decision. + assert.equal(report.micro.accuracy, 0.8); + // Macro sees the floor-level `close` recall (0 of 2 caught) and drags the headline down accordingly. + assert.equal(report.perAction.close.recall, 0); + assert.ok(report.macro.recall !== null && report.macro.recall < 0.6, `macro recall ${report.macro.recall}`); +}); + +test("an action with no realized instances is EXCLUDED from the macro mean, never counted as 0", () => { + // Only merges are realized, and the agent gets them all right. `close`/`label`/... have no positives, so + // their recall is null and must leave the average rather than dragging a perfect agent to 0.2. + const report = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth: truthFrom([["u1", "merge"], ["u2", "merge"]]), + proposals: proposalsFrom([["u1", "merge"], ["u2", "merge"]]), + }); + assert.equal(report.perAction.close.recall, null); + assert.equal(report.macro.recall, 1); + // ...and the count says the headline is an average over ONE action, not five — the claim is legible. + const scoredRecallActions = SCORED_ACTIONS.filter((action) => report.perAction[action].recall !== null); + assert.deepEqual(scoredRecallActions, ["merge"]); +}); + +test("coverage: abstentions lower coverage and never enter the confusion matrix as errors", () => { + const report = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth: truthFrom([["u1", "merge"], ["u2", "close"], ["u3", "merge"], ["u4", "close"]]), + proposals: proposalsFrom([["u1", "merge"], ["u2", "abstain"], ["u3", "merge"]]), + }); + // u2 declared an abstention; u4 was simply never answered — the SAME act, scored the same way. + assert.deepEqual(report.coverage, { + decided: 2, + abstained: 2, + coverage: 0.5, + unresolvedExcluded: 0, + unscorableProposals: 0, + }); + // Two decided cases only: an abstention is absent from every count, so precision stays perfect. + assert.equal(report.perAction.merge.caseCount, 2); + assert.equal(report.perAction.merge.precision, 1); + assert.equal(report.perAction.close.falseNegative, 0); +}); + +test("unresolved ground truth leaves the denominator before scoring begins, and is reported", () => { + const report = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth: truthFrom([["u1", "merge"], ["u2", null], ["u3", null]]), + // The agent answered the unresolved units too; they are excluded regardless of what it said. + proposals: proposalsFrom([["u1", "merge"], ["u2", "close"], ["u3", "merge"]]), + }); + assert.equal(report.coverage.decided, 1); + assert.equal(report.coverage.unresolvedExcluded, 2); + // A proposal for an excluded unit is unscorable, not free credit and not an error. + assert.equal(report.coverage.unscorableProposals, 2); + assert.equal(report.perAction.merge.caseCount, 1); +}); + +test("a proposal for a work unit outside the snapshot is counted as unscorable, never scored", () => { + const report = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth: truthFrom([["u1", "merge"]]), + proposals: proposalsFrom([["u1", "merge"], ["not-in-snapshot", "merge"], ["also-not", "close"]]), + }); + assert.equal(report.coverage.unscorableProposals, 2); + assert.equal(report.perAction.merge.caseCount, 1); +}); + +test("a resubmitted proposal is a correction: the LAST one for a work unit wins", () => { + const report = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth: truthFrom([["u1", "merge"]]), + proposals: proposalsFrom([["u1", "close"], ["u1", "merge"]]), + }); + assert.equal(report.perAction.merge.truePositive, 1); + assert.equal(report.coverage.decided, 1); +}); + +test("an agent that faced nothing reports NULL coverage, never 0", () => { + const report = scoreBenchmarkProposals({ subjectId: SUBJECT, groundTruth: truthFrom([]), proposals: [] }); + assert.deepEqual(report.coverage, { decided: 0, abstained: 0, coverage: null, unresolvedExcluded: 0, unscorableProposals: 0 }); + assert.deepEqual(report.macro, { precision: null, recall: null, actionsScored: 0 }); + assert.deepEqual(report.micro, { precision: null, recall: null, accuracy: null }); +}); + +test("the report carries the snapshot and subject it commits to", () => { + const report = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth: truthFrom([["u1", "merge"]]), + proposals: proposalsFrom([["u1", "merge"]]), + }); + assert.equal(report.schemaVersion, 1); + assert.equal(report.snapshotRef, SNAPSHOT); + assert.equal(report.subjectId, SUBJECT); +}); + +test("REGRESSION: the Pareto floor holds across actions — gaining on merge while losing on close is REGRESSED", () => { + const groundTruth = truthFrom([ + ["u1", "merge"], ["u2", "merge"], ["u3", "merge"], + ["u4", "close"], ["u5", "close"], ["u6", "close"], + ]); + // Baseline catches 1 of 3 merges (recall 0.33) and all 3 closes (recall 1). The candidate buys perfect + // merge recall by spending close recall: 3 of 3 merges, 1 of 3 closes. Precision stays 1 on both actions + // in both reports, so recall alone moves and the trade is unambiguous. + const baseline = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth, + proposals: proposalsFrom([["u1", "merge"], ["u2", "hold"], ["u3", "hold"], ["u4", "close"], ["u5", "close"], ["u6", "close"]]), + }); + const candidate = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth, + proposals: proposalsFrom([["u1", "merge"], ["u2", "merge"], ["u3", "merge"], ["u4", "close"], ["u5", "hold"], ["u6", "hold"]]), + }); + assert.equal(baseline.perAction.merge.recall, 0.3333333333333333); + assert.equal(candidate.perAction.merge.recall, 1); + assert.equal(baseline.perAction.close.recall, 1); + assert.equal(candidate.perAction.close.recall, 0.3333333333333333); + const comparison = compareBenchmarkScores(baseline, candidate); + assert.ok(comparison.improvedActions.includes("merge"), "merge should have improved"); + assert.ok(comparison.regressedActions.includes("close"), "close should have regressed"); + // A trade is not a win — the entire point of the floor. + assert.equal(comparison.verdict, "regressed"); +}); + +test("comparison verdicts: improved when only gains, unchanged when identical, per-action detail preserved", () => { + const groundTruth = truthFrom([["u1", "merge"], ["u2", "merge"]]); + const worse = scoreBenchmarkProposals({ subjectId: SUBJECT, groundTruth, proposals: proposalsFrom([["u1", "merge"], ["u2", "close"]]) }); + const better = scoreBenchmarkProposals({ subjectId: SUBJECT, groundTruth, proposals: proposalsFrom([["u1", "merge"], ["u2", "merge"]]) }); + assert.equal(compareBenchmarkScores(worse, better).verdict, "improved"); + assert.equal(compareBenchmarkScores(better, better).verdict, "unchanged"); + const detail = compareBenchmarkScores(worse, better); + assert.equal(detail.perAction.merge.verdict, "improved"); + for (const action of SCORED_ACTIONS) assert.equal(detail.perAction[action].ruleId, action); +}); + +test("comparing two DIFFERENT subjects throws — that is a caller bug, not a valid comparison", () => { + const groundTruth = truthFrom([["u1", "merge"]]); + const mine = scoreBenchmarkProposals({ subjectId: "agent-a", groundTruth, proposals: proposalsFrom([["u1", "merge"]]) }); + const theirs: BenchmarkScoreReport = { ...mine, subjectId: "agent-b" }; + assert.throws(() => compareBenchmarkScores(mine, theirs), /different subjects/); +});