From 6e9a31daeb98382af35412c7f819b37d6dcdbce8 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:38:22 -0700 Subject: [PATCH] feat(benchmark): multi-class proposal scorer on the shared primitives (#9262) Scoring built ON scoreBacktest and compareBacktestScores, deliberately not beside them: this module is an adapter that reshapes (proposal, ground truth) pairs into the exact inputs those functions already take. A second scoring implementation is the drift the issue exists to prevent, so the guarantee is mechanical -- an equivalent binary case scored through this adapter and through scoreBacktest directly produces the identical report, asserted in the tests. The reuse borrows the primitive's positive-class slot per action: for action A the positive class is "the realized action was A" and the classifier answers "the agent proposed A". The `ruleId` slot carries the action name, which makes each per-action report self-labeling and makes compareBacktestScores' rule-mismatch throw double as a guard against comparing two different actions. AGGREGATION -- the recorded decision, in the module header. Macro is the headline, micro published alongside. Under single-label scoring micro precision, micro recall and accuracy are all the same number and it is reported once, honestly labeled, rather than three times as if independent; being pooled, it is dominated by the most frequent action, so an agent answering "merge" to everything scores 0.8 on this corpus while being worthless. Macro weights each action equally and exposes exactly that agent through its floor-level close/request_changes numbers, which is why it answers the benchmark's actual question. An action with no realized instances has null metrics and is EXCLUDED from the macro mean rather than counted as 0, and actionsScored publishes how many entered it. Coverage follows #9215: coverage = decided / (decided + abstained); abstentions lower coverage and never enter a confusion-matrix count. Silence and a declared abstention are the same act and are scored the same way. #9261's unresolved units leave the denominator before scoring starts, and proposals for units outside the snapshot are counted as unscorable -- ignored for scoring, so padding inflates nothing, but visible, because a nonzero count means the agent answered a different question than the one asked. The Pareto floor extends to the multi-class case: ANY regressed action decides the verdict, so gaining on merge while losing on close is a trade, and a trade is not a win. Closes #9262 --- .../src/calibration/benchmark-score.ts | 218 +++++++++++++++ packages/loopover-engine/src/index.ts | 1 + .../test/benchmark-score.test.ts | 258 ++++++++++++++++++ 3 files changed, 477 insertions(+) create mode 100644 packages/loopover-engine/src/calibration/benchmark-score.ts create mode 100644 packages/loopover-engine/test/benchmark-score.test.ts diff --git a/packages/loopover-engine/src/calibration/benchmark-score.ts b/packages/loopover-engine/src/calibration/benchmark-score.ts new file mode 100644 index 000000000..9806646f5 --- /dev/null +++ b/packages/loopover-engine/src/calibration/benchmark-score.ts @@ -0,0 +1,218 @@ +// Benchmark proposal scorer (#9262, harness #9216, epic #8534) — multi-class scoring built ON the existing +// confusion-matrix and Pareto-floor primitives, deliberately NOT beside them. +// +// REUSE, NOT REIMPLEMENTATION. `scoreBacktest` (backtest-score.ts) and `compareBacktestScores` +// (backtest-compare.ts) remain the scoring core; this module is an adapter that reshapes (proposal, ground +// truth) pairs into the exact inputs those functions already take. A second scoring implementation is +// precisely the drift this module exists to prevent, so the anti-drift guarantee is mechanical rather than +// aspirational: an equivalent binary case scored here and scored through the internal backtest path produces +// the identical report, asserted directly in the tests. +// +// The reuse works by borrowing the primitive's positive-class slot per action: for action A, the "positive" +// class is "the realized action was A" and the classifier's answer is "the agent proposed A". The primitive +// names that slot `reversed`/`confirmed` because its first caller scored rule reversals; the name is +// vestigial here and never surfaces in this module's own output. The `ruleId` slot carries the ACTION name, +// which makes each per-action report self-labeling and makes `compareBacktestScores`' rule-mismatch throw +// double as a guard against accidentally comparing two different actions. +// +// AGGREGATION — the recorded decision (#9262 requirement 2). MACRO is the headline; MICRO is published +// alongside. They answer different questions and the difference is not cosmetic here: +// +// Micro pools the counts across actions. Under single-label multi-class scoring — one prediction per work +// unit — every decided unit contributes exactly one TP or one FP, so micro precision, micro recall and +// plain accuracy are all the SAME number. That makes it an honest "how often was it right overall", and +// also makes it dominated by whichever action is most frequent. In this corpus that is overwhelmingly +// `merge`, so an agent that answers "merge" to everything scores well on micro while being worthless. +// +// Macro averages the per-action metrics, so each action counts equally regardless of frequency, and the +// answer-merge-to-everything agent is immediately exposed by its floor-level `close`/`request_changes` +// numbers. Since the benchmark's question is "can this agent make MAINTAINER decisions" — including the +// rare, expensive ones — macro is the number that answers it, and therefore the headline. +// +// Both are published because a benchmark that reports only its headline invites the reader to reconstruct +// the other one wrongly. An action with no realized instances has null metrics and is EXCLUDED from the +// macro mean rather than counted as 0 — a metric nobody could measure must not drag an average down. +// +// Same purity contract as the rest of this module family: no IO, no randomness, no wall-clock reads. + +import type { BacktestCase } from "./backtest-corpus.js"; +import { scoreBacktest, type BacktestScoreReport } from "./backtest-score.js"; +import { compareBacktestScores, type BacktestComparison } from "./backtest-compare.js"; +import type { BenchmarkActionKind, BenchmarkProposal } from "./benchmark-proposal.js"; +import { scoreableGroundTruths, type BenchmarkGroundTruthSet } from "./benchmark-ground-truth.js"; + +/** Every action scored, in a fixed order so two reports are directly comparable field by field. */ +export const SCORED_ACTIONS: readonly BenchmarkActionKind[] = ["merge", "close", "request_changes", "label", "hold"]; + +export type BenchmarkScoreReport = { + schemaVersion: 1; + snapshotRef: string; + /** WHO was scored — carried through from the proposals, opaque here exactly as in #9215's EvalScoreRecord. */ + subjectId: string; + /** One-vs-rest report per action, keyed by action, each produced by the shared `scoreBacktest`. */ + perAction: Record; + /** The HEADLINE (see the module header's recorded decision). `actionsScored` is how many actions had a + * non-null metric and therefore entered the mean — a macro number over 2 of 5 actions is a different + * claim than one over 5, and hiding that would be the kind of unexamined average this file argues against. */ + macro: { precision: number | null; recall: number | null; actionsScored: number }; + /** Pooled counts. Under single-label scoring micro precision === micro recall === accuracy; all three are + * the same number and it is reported once, honestly labeled, rather than three times as if independent. */ + micro: { precision: number | null; recall: number | null; accuracy: number | null }; + /** #9215's coverage semantics. Abstentions are NEVER folded into errors: they lower coverage, which is a + * different (and recoverable) thing than being wrong. */ + coverage: { + decided: number; + abstained: number; + /** `decided / (decided + abstained)`; null when the agent faced nothing at all — never 0, which would + * read as "answered nothing it was asked" rather than "was asked nothing". */ + coverage: number | null; + /** Ground-truth units excluded before scoring began (#9261's `unresolved`) — published so a reader can + * see the denominator shrink rather than discovering it in a footnote. */ + unresolvedExcluded: number; + /** Proposals for work units not in this snapshot's ground truth. Ignored for scoring (a submitter + * cannot inflate anything by padding), but COUNTED, because a nonzero value means the agent is + * answering a different question than the one asked. */ + unscorableProposals: number; + }; +}; + +function ratio(numerator: number, denominator: number): number | null { + return denominator > 0 ? Math.round((numerator / denominator) * 1000) / 1000 : null; +} + +/** Mean of the non-null values, or null when none are — an unmeasurable metric leaves the average rather + * than entering it as 0. */ +function macroMean(values: ReadonlyArray): { mean: number | null; counted: number } { + const present = values.filter((value): value is number => value !== null); + if (present.length === 0) return { mean: null, counted: 0 }; + return { mean: Math.round((present.reduce((sum, value) => sum + value, 0) / present.length) * 1000) / 1000, counted: present.length }; +} + +/** + * Score one agent's proposals for one snapshot against #9261's realized ground truth. + * + * Denominator discipline, in the order it is applied: + * 1. `unresolved` ground truth leaves entirely (#9261) — never a correct abstention, never an error. + * 2. A scoreable unit the agent DID NOT answer counts as an abstention, identically to an explicit + * `{kind: "abstain"}`. Silence and a declared abstention are the same act; scoring them differently + * would reward whichever one an agent's emitter happened to produce. + * 3. Abstentions lower coverage and are absent from every confusion-matrix count. + */ +export function scoreBenchmarkProposals(input: { + subjectId: string; + groundTruth: BenchmarkGroundTruthSet; + proposals: readonly BenchmarkProposal[]; +}): BenchmarkScoreReport { + const scoreable = scoreableGroundTruths(input.groundTruth); + const scoreableIds = new Set(scoreable.map((truth) => truth.workUnitId)); + + // Last proposal per work unit wins, so a resubmission is a correction rather than a double entry. + const proposalByUnit = new Map(); + let unscorableProposals = 0; + for (const proposal of input.proposals) { + if (!scoreableIds.has(proposal.workUnitId)) { + unscorableProposals += 1; + continue; + } + proposalByUnit.set(proposal.workUnitId, proposal); + } + + // The decided set: scoreable units the agent actually answered with an action. + const decided: Array<{ realized: BenchmarkActionKind; predicted: BenchmarkActionKind }> = []; + let abstained = 0; + for (const truth of scoreable) { + const proposal = proposalByUnit.get(truth.workUnitId); + if (!proposal || proposal.prediction.kind === "abstain") { + abstained += 1; + continue; + } + decided.push({ realized: truth.action, predicted: proposal.prediction.action.kind }); + } + + // One-vs-rest through the SHARED primitive. The synthetic cases carry the action in the `ruleId` slot so + // each report is self-labeling and a cross-action comparison throws rather than silently succeeding. + const perAction = {} as Record; + for (const action of SCORED_ACTIONS) { + const cases: BacktestCase[] = decided.map((pair, index) => ({ + ruleId: action, + targetKey: String(index), + outcome: pair.realized, + label: pair.realized === action ? "reversed" : "confirmed", + firedAt: input.groundTruth.frozenAt, + decidedAt: input.groundTruth.horizonEnd, + metadata: { predicted: pair.predicted }, + })); + perAction[action] = scoreBacktest(action, cases, (backtestCase) => + (backtestCase.metadata as { predicted: BenchmarkActionKind }).predicted === action ? "reversed" : "confirmed", + ); + } + + const macroPrecision = macroMean(SCORED_ACTIONS.map((action) => perAction[action].precision)); + const macroRecall = macroMean(SCORED_ACTIONS.map((action) => perAction[action].recall)); + // Pooled: exactly one TP or FP per decided unit, so this single number IS precision, recall and accuracy. + const correct = decided.filter((pair) => pair.predicted === pair.realized).length; + const micro = ratio(correct, decided.length); + + return { + schemaVersion: 1, + snapshotRef: input.groundTruth.snapshotRef, + subjectId: input.subjectId, + perAction, + macro: { + precision: macroPrecision.mean, + recall: macroRecall.mean, + // The two means are taken over the same actions whenever both are defined; report the precision + // side's count, which is the one the headline precision is an average of. + actionsScored: macroPrecision.counted, + }, + micro: { precision: micro, recall: micro, accuracy: micro }, + coverage: { + decided: decided.length, + abstained, + coverage: ratio(decided.length, decided.length + abstained), + unresolvedExcluded: input.groundTruth.coverage.unresolved, + unscorableProposals, + }, + }; +} + +export type BenchmarkComparison = { + subjectId: string; + perAction: Record; + regressedActions: BenchmarkActionKind[]; + improvedActions: BenchmarkActionKind[]; + /** The Pareto floor, extended to the multi-class case (#9262 requirement 4): ANY regressed action decides + * the verdict, even alongside improvements elsewhere. Gaining on `merge` while losing on `close` is a + * trade, and the floor's entire purpose is that a trade is not a win. */ + verdict: "improved" | "regressed" | "unchanged"; +}; + +/** + * Compare two benchmark score reports under the Pareto floor, per action, via the SHARED comparator. + * + * Every per-action verdict comes from `compareBacktestScores`, so the null-handling ("unknown stays + * unknown": an axis with a null on either side is excluded from both lists) is inherited rather than + * re-derived. Throws on a subject mismatch — comparing two different agents' reports as if they were one + * agent's before/after is a caller bug, the same posture the primitive takes on a rule mismatch. + */ +export function compareBenchmarkScores(baseline: BenchmarkScoreReport, candidate: BenchmarkScoreReport): BenchmarkComparison { + if (baseline.subjectId !== candidate.subjectId) { + throw new Error(`cannot compare benchmark scores for different subjects: ${baseline.subjectId} vs ${candidate.subjectId}`); + } + const perAction = {} as Record; + const regressedActions: BenchmarkActionKind[] = []; + const improvedActions: BenchmarkActionKind[] = []; + for (const action of SCORED_ACTIONS) { + const comparison = compareBacktestScores(baseline.perAction[action], candidate.perAction[action]); + perAction[action] = comparison; + if (comparison.verdict === "regressed") regressedActions.push(action); + else if (comparison.verdict === "improved") improvedActions.push(action); + } + return { + subjectId: baseline.subjectId, + perAction, + regressedActions, + improvedActions, + verdict: regressedActions.length > 0 ? "regressed" : improvedActions.length > 0 ? "improved" : "unchanged", + }; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 0ddea29fb..5989875d6 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -185,6 +185,7 @@ export * from "./calibration/attestation-envelope.js"; export * from "./calibration/attester.js"; export * from "./calibration/benchmark-proposal.js"; export * from "./calibration/benchmark-ground-truth.js"; +export * from "./calibration/benchmark-score.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/benchmark-score.test.ts b/packages/loopover-engine/test/benchmark-score.test.ts new file mode 100644 index 000000000..8d4a4f66b --- /dev/null +++ b/packages/loopover-engine/test/benchmark-score.test.ts @@ -0,0 +1,258 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + compareBenchmarkScores, + deriveBenchmarkGroundTruth, + scoreBacktest, + scoreBenchmarkProposals, + SCORED_ACTIONS, + type BenchmarkAction, + type BenchmarkActionKind, + type BenchmarkProposal, + type BenchmarkScoreReport, +} from "../dist/index.js"; + +// #9262 (harness #9216, epic #8534): multi-class scoring built ON the shared confusion-matrix and +// Pareto-floor primitives. The load-bearing test here is the ANTI-DRIFT guard — an equivalent binary case +// scored through this adapter and through `scoreBacktest` directly must produce the identical report, which +// is what makes "benchmark scores are the same kind of number as the internal backtest's" mechanical rather +// than a claim in a comment. + +const FROZEN = "2026-07-01T00:00:00.000Z"; +const SNAPSHOT = "a1".repeat(32); +const SUBJECT = "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty"; + +function at(days: number): string { + return new Date(Date.parse(FROZEN) + days * 86_400_000).toISOString(); +} + +/** Ground truth from a compact [workUnitId, realizedAction] list; `null` means no action (unresolved). */ +function truthFrom(pairs: ReadonlyArray<[string, BenchmarkActionKind | null]>) { + return deriveBenchmarkGroundTruth({ + snapshotRef: SNAPSHOT, + frozenAt: FROZEN, + horizonDays: 14, + workUnitIds: pairs.map(([id]) => id), + events: pairs.flatMap(([id, action]) => (action ? [{ workUnitId: id, action, occurredAt: at(2) }] : [])), + }); +} + +/** Proposals from a compact [workUnitId, predictedAction | "abstain"] list. */ +function proposalsFrom(pairs: ReadonlyArray<[string, BenchmarkActionKind | "abstain"]>): BenchmarkProposal[] { + return pairs.map(([workUnitId, choice]) => ({ + schemaVersion: 1, + benchmarkId: "bench-1", + snapshotRef: SNAPSHOT, + workUnitId, + subject: { kind: "agent", id: SUBJECT }, + prediction: choice === "abstain" ? { kind: "abstain" as const } : { kind: "act" as const, action: actionOf(choice) }, + })); +} + +function actionOf(kind: BenchmarkActionKind): BenchmarkAction { + if (kind === "close") return { kind, reasonClass: "defective" }; + if (kind === "request_changes") return { kind, blockingConcern: "concern" }; + if (kind === "label") return { kind, labels: ["bug"] }; + return { kind }; +} + +test("ANTI-DRIFT: an equivalent binary case scores identically through this adapter and through scoreBacktest", () => { + // Two realized merges and two realized closes; the agent gets one of each right. Scored here as the + // one-vs-rest `merge` report, and independently through the shared primitive on the same four cases. + const groundTruth = truthFrom([["u1", "merge"], ["u2", "merge"], ["u3", "close"], ["u4", "close"]]); + const proposals = proposalsFrom([["u1", "merge"], ["u2", "close"], ["u3", "merge"], ["u4", "close"]]); + const report = scoreBenchmarkProposals({ subjectId: SUBJECT, groundTruth, proposals }); + + const predictions: BenchmarkActionKind[] = ["merge", "close", "merge", "close"]; + const realized: BenchmarkActionKind[] = ["merge", "merge", "close", "close"]; + const direct = scoreBacktest( + "merge", + realized.map((actual, index) => ({ + ruleId: "merge", + targetKey: String(index), + outcome: actual, + label: actual === "merge" ? ("reversed" as const) : ("confirmed" as const), + firedAt: FROZEN, + decidedAt: groundTruth.horizonEnd, + metadata: { index }, + })), + (backtestCase) => (predictions[(backtestCase.metadata as { index: number }).index] === "merge" ? "reversed" : "confirmed"), + ); + assert.deepEqual(report.perAction.merge, direct); + // And the counts are the ones a human would write down by hand: TP=u1, FP=u3, TN=u4, FN=u2. + assert.deepEqual( + { tp: direct.truePositive, fp: direct.falsePositive, tn: direct.trueNegative, fn: direct.falseNegative }, + { tp: 1, fp: 1, tn: 1, fn: 1 }, + ); +}); + +test("scores every action one-vs-rest, each report self-labeled by its action", () => { + const groundTruth = truthFrom([["u1", "merge"], ["u2", "close"], ["u3", "request_changes"], ["u4", "label"], ["u5", "hold"]]); + const report = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth, + proposals: proposalsFrom([["u1", "merge"], ["u2", "close"], ["u3", "request_changes"], ["u4", "label"], ["u5", "hold"]]), + }); + for (const action of SCORED_ACTIONS) { + assert.equal(report.perAction[action].ruleId, action); + assert.equal(report.perAction[action].caseCount, 5); + assert.equal(report.perAction[action].precision, 1); + assert.equal(report.perAction[action].recall, 1); + } + assert.deepEqual(report.macro, { precision: 1, recall: 1, actionsScored: 5 }); + assert.deepEqual(report.micro, { precision: 1, recall: 1, accuracy: 1 }); +}); + +test("REGRESSION: macro exposes the answer-merge-to-everything agent that micro flatters", () => { + // Eight merges, two closes — the skew the module header warns about. The agent answers "merge" always. + const pairs: Array<[string, BenchmarkActionKind]> = [ + ["u1", "merge"], ["u2", "merge"], ["u3", "merge"], ["u4", "merge"], + ["u5", "merge"], ["u6", "merge"], ["u7", "merge"], ["u8", "merge"], + ["u9", "close"], ["u10", "close"], + ]; + const report = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth: truthFrom(pairs), + proposals: proposalsFrom(pairs.map(([id]) => [id, "merge"] as [string, BenchmarkActionKind])), + }); + // Micro (=accuracy) says 80% — respectable-looking for an agent that never made a decision. + assert.equal(report.micro.accuracy, 0.8); + // Macro sees the floor-level `close` recall (0 of 2 caught) and drags the headline down accordingly. + assert.equal(report.perAction.close.recall, 0); + assert.ok(report.macro.recall !== null && report.macro.recall < 0.6, `macro recall ${report.macro.recall}`); +}); + +test("an action with no realized instances is EXCLUDED from the macro mean, never counted as 0", () => { + // Only merges are realized, and the agent gets them all right. `close`/`label`/... have no positives, so + // their recall is null and must leave the average rather than dragging a perfect agent to 0.2. + const report = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth: truthFrom([["u1", "merge"], ["u2", "merge"]]), + proposals: proposalsFrom([["u1", "merge"], ["u2", "merge"]]), + }); + assert.equal(report.perAction.close.recall, null); + assert.equal(report.macro.recall, 1); + // ...and the count says the headline is an average over ONE action, not five — the claim is legible. + const scoredRecallActions = SCORED_ACTIONS.filter((action) => report.perAction[action].recall !== null); + assert.deepEqual(scoredRecallActions, ["merge"]); +}); + +test("coverage: abstentions lower coverage and never enter the confusion matrix as errors", () => { + const report = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth: truthFrom([["u1", "merge"], ["u2", "close"], ["u3", "merge"], ["u4", "close"]]), + proposals: proposalsFrom([["u1", "merge"], ["u2", "abstain"], ["u3", "merge"]]), + }); + // u2 declared an abstention; u4 was simply never answered — the SAME act, scored the same way. + assert.deepEqual(report.coverage, { + decided: 2, + abstained: 2, + coverage: 0.5, + unresolvedExcluded: 0, + unscorableProposals: 0, + }); + // Two decided cases only: an abstention is absent from every count, so precision stays perfect. + assert.equal(report.perAction.merge.caseCount, 2); + assert.equal(report.perAction.merge.precision, 1); + assert.equal(report.perAction.close.falseNegative, 0); +}); + +test("unresolved ground truth leaves the denominator before scoring begins, and is reported", () => { + const report = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth: truthFrom([["u1", "merge"], ["u2", null], ["u3", null]]), + // The agent answered the unresolved units too; they are excluded regardless of what it said. + proposals: proposalsFrom([["u1", "merge"], ["u2", "close"], ["u3", "merge"]]), + }); + assert.equal(report.coverage.decided, 1); + assert.equal(report.coverage.unresolvedExcluded, 2); + // A proposal for an excluded unit is unscorable, not free credit and not an error. + assert.equal(report.coverage.unscorableProposals, 2); + assert.equal(report.perAction.merge.caseCount, 1); +}); + +test("a proposal for a work unit outside the snapshot is counted as unscorable, never scored", () => { + const report = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth: truthFrom([["u1", "merge"]]), + proposals: proposalsFrom([["u1", "merge"], ["not-in-snapshot", "merge"], ["also-not", "close"]]), + }); + assert.equal(report.coverage.unscorableProposals, 2); + assert.equal(report.perAction.merge.caseCount, 1); +}); + +test("a resubmitted proposal is a correction: the LAST one for a work unit wins", () => { + const report = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth: truthFrom([["u1", "merge"]]), + proposals: proposalsFrom([["u1", "close"], ["u1", "merge"]]), + }); + assert.equal(report.perAction.merge.truePositive, 1); + assert.equal(report.coverage.decided, 1); +}); + +test("an agent that faced nothing reports NULL coverage, never 0", () => { + const report = scoreBenchmarkProposals({ subjectId: SUBJECT, groundTruth: truthFrom([]), proposals: [] }); + assert.deepEqual(report.coverage, { decided: 0, abstained: 0, coverage: null, unresolvedExcluded: 0, unscorableProposals: 0 }); + assert.deepEqual(report.macro, { precision: null, recall: null, actionsScored: 0 }); + assert.deepEqual(report.micro, { precision: null, recall: null, accuracy: null }); +}); + +test("the report carries the snapshot and subject it commits to", () => { + const report = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth: truthFrom([["u1", "merge"]]), + proposals: proposalsFrom([["u1", "merge"]]), + }); + assert.equal(report.schemaVersion, 1); + assert.equal(report.snapshotRef, SNAPSHOT); + assert.equal(report.subjectId, SUBJECT); +}); + +test("REGRESSION: the Pareto floor holds across actions — gaining on merge while losing on close is REGRESSED", () => { + const groundTruth = truthFrom([ + ["u1", "merge"], ["u2", "merge"], ["u3", "merge"], + ["u4", "close"], ["u5", "close"], ["u6", "close"], + ]); + // Baseline catches 1 of 3 merges (recall 0.33) and all 3 closes (recall 1). The candidate buys perfect + // merge recall by spending close recall: 3 of 3 merges, 1 of 3 closes. Precision stays 1 on both actions + // in both reports, so recall alone moves and the trade is unambiguous. + const baseline = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth, + proposals: proposalsFrom([["u1", "merge"], ["u2", "hold"], ["u3", "hold"], ["u4", "close"], ["u5", "close"], ["u6", "close"]]), + }); + const candidate = scoreBenchmarkProposals({ + subjectId: SUBJECT, + groundTruth, + proposals: proposalsFrom([["u1", "merge"], ["u2", "merge"], ["u3", "merge"], ["u4", "close"], ["u5", "hold"], ["u6", "hold"]]), + }); + assert.equal(baseline.perAction.merge.recall, 0.3333333333333333); + assert.equal(candidate.perAction.merge.recall, 1); + assert.equal(baseline.perAction.close.recall, 1); + assert.equal(candidate.perAction.close.recall, 0.3333333333333333); + const comparison = compareBenchmarkScores(baseline, candidate); + assert.ok(comparison.improvedActions.includes("merge"), "merge should have improved"); + assert.ok(comparison.regressedActions.includes("close"), "close should have regressed"); + // A trade is not a win — the entire point of the floor. + assert.equal(comparison.verdict, "regressed"); +}); + +test("comparison verdicts: improved when only gains, unchanged when identical, per-action detail preserved", () => { + const groundTruth = truthFrom([["u1", "merge"], ["u2", "merge"]]); + const worse = scoreBenchmarkProposals({ subjectId: SUBJECT, groundTruth, proposals: proposalsFrom([["u1", "merge"], ["u2", "close"]]) }); + const better = scoreBenchmarkProposals({ subjectId: SUBJECT, groundTruth, proposals: proposalsFrom([["u1", "merge"], ["u2", "merge"]]) }); + assert.equal(compareBenchmarkScores(worse, better).verdict, "improved"); + assert.equal(compareBenchmarkScores(better, better).verdict, "unchanged"); + const detail = compareBenchmarkScores(worse, better); + assert.equal(detail.perAction.merge.verdict, "improved"); + for (const action of SCORED_ACTIONS) assert.equal(detail.perAction[action].ruleId, action); +}); + +test("comparing two DIFFERENT subjects throws — that is a caller bug, not a valid comparison", () => { + const groundTruth = truthFrom([["u1", "merge"]]); + const mine = scoreBenchmarkProposals({ subjectId: "agent-a", groundTruth, proposals: proposalsFrom([["u1", "merge"]]) }); + const theirs: BenchmarkScoreReport = { ...mine, subjectId: "agent-b" }; + assert.throws(() => compareBenchmarkScores(mine, theirs), /different subjects/); +});