From d051556b10c65f1860b0b0ac6cb4ec3e48c4c1fd Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:32:18 -0700 Subject: [PATCH] feat(benchmark): reversal-aware realized-history ground truth (#9261) The labels every candidate proposal is scored against: for each work unit in a snapshot, what the maintainer ACTUALLY did within the prediction horizon, emitted in #9260's exact action vocabulary so proposal and outcome compare with no translation layer that could drift. Three rules the surrounding system already believes, now enforced here: - REVERSAL-AWARE SETTLEMENT. A merge later reverted is not a clean merge. The label is the settled state at horizon end with the reversal recorded alongside -- the treatment that makes public-rule-precision.ts's precision honest. The vocabulary is the established reversal_reopened/reverted/superseded set, never a benchmark-local re-invention, so benchmark and internal backtest ground truth cannot disagree about the same event. A reversal counts only inside the horizon AND strictly after the action it overturns; the earliest one wins. - UNRESOLVED IS EXPLICIT AND EXCLUDED. No action inside the horizon is a first-class `unresolved`, never silently a correct abstention (which would reward declining exactly what nobody decided) nor an incorrect prediction (which would punish the horizon being short). It leaves the denominator, and the unresolved RATE is published so a climbing rate reads as "the horizon is wrong" rather than as a scoring artifact. - STABILITY. Every decision is a pure function of events dated inside the window, so a later event -- including a later reversal -- cannot move a settled label. Asserted directly: the same input plus a year of post-horizon history yields byte-identical output. Closes #9261 --- .../src/calibration/benchmark-ground-truth.ts | 203 +++++++++++++++++ packages/loopover-engine/src/index.ts | 1 + .../test/benchmark-ground-truth.test.ts | 210 ++++++++++++++++++ 3 files changed, 414 insertions(+) create mode 100644 packages/loopover-engine/src/calibration/benchmark-ground-truth.ts create mode 100644 packages/loopover-engine/test/benchmark-ground-truth.test.ts diff --git a/packages/loopover-engine/src/calibration/benchmark-ground-truth.ts b/packages/loopover-engine/src/calibration/benchmark-ground-truth.ts new file mode 100644 index 0000000000..1a1a395661 --- /dev/null +++ b/packages/loopover-engine/src/calibration/benchmark-ground-truth.ts @@ -0,0 +1,203 @@ +// Realized-history ground truth for the frozen-repo benchmark (#9261, harness #9216, epic #8534). +// +// For each work unit in a snapshot: what the maintainer ACTUALLY did within the prediction horizon. These +// are the labels every candidate proposal (#9260) is scored against, so they are emitted in that module's +// exact action vocabulary — proposal and outcome are directly comparable, with no translation layer that +// could drift. +// +// Three rules this module exists to enforce, each of which the surrounding system already believes and which +// a benchmark must not quietly contradict: +// +// REVERSAL-AWARE SETTLEMENT. A merge later reverted is not a clean merge. The realized outcome is the +// SETTLED state at horizon end, with the reversal recorded alongside — the same treatment that makes +// `public-rule-precision.ts`'s precision honest (it subtracts reversals from confirmations rather than +// counting the original decision). The reversal vocabulary is the established one +// (`reversal_reopened` / `reversal_reverted` / `reversal_superseded`), not a new benchmark-local set, so +// benchmark ground truth and internal backtest ground truth can never disagree about the same event. +// +// UNRESOLVED IS EXPLICIT AND EXCLUDED. A work unit with no maintainer action inside the horizon is +// `unresolved`. It is NOT silently counted as a correct abstention (which would reward an agent for +// declining exactly the cases nobody decided) nor as an incorrect prediction (which would punish an agent +// for the horizon being too short). It leaves the scoring denominator entirely, and the unresolved RATE is +// published so a climbing rate is legible as "the horizon is wrong" rather than as a scoring artifact. +// +// STABILITY. Ground truth for a given (snapshot, horizon) must not depend on WHEN extraction runs, once +// the horizon has elapsed. Every decision here is a pure function of events dated inside the window, so a +// later event — including a later reversal — cannot retroactively change a settled label. That is what +// makes a leaderboard reproducible instead of drifting under its own scores. +// +// Pure core (this module); the IO wrapper that reads events from D1 lives engine-side of the harness, the +// same split every sibling calibration module uses. + +import type { BenchmarkActionKind, BenchmarkCloseReasonClass } from "./benchmark-proposal.js"; + +/** The established reversal vocabulary — deliberately imported as literals from the same set the audit-event + * table and `public-rule-precision.ts` already use, never a benchmark-local re-invention. */ +export type RealizedReversalKind = "reversal_reopened" | "reversal_reverted" | "reversal_superseded"; + +/** One realized maintainer event inside (or outside) a horizon, already normalized by the IO wrapper. */ +export type RealizedMaintainerEvent = { + workUnitId: string; + /** Same closed vocabulary the agent proposes in — comparison needs no translation. */ + action: BenchmarkActionKind; + /** ISO-8601. Events at exactly the horizon end are INSIDE the window (inclusive), matching how the + * corpus builder's own windowing treats a boundary event as belonging to the window it closes. */ + occurredAt: string; + /** Present only for a `close`; the class realized history settled on. */ + reasonClass?: BenchmarkCloseReasonClass | undefined; + /** Present only for a `label`. */ + labels?: readonly string[] | undefined; +}; + +/** A reversal of a previously-realized action, in the established vocabulary. */ +export type RealizedReversalEvent = { + workUnitId: string; + kind: RealizedReversalKind; + occurredAt: string; +}; + +/** The settled label for ONE work unit. `unresolved` carries no action by construction — the type makes + * "unresolved but somehow also a merge" unrepresentable rather than merely untested. */ +export type BenchmarkGroundTruth = + | { workUnitId: string; outcome: "unresolved" } + | { + workUnitId: string; + outcome: "settled"; + action: BenchmarkActionKind; + reasonClass?: BenchmarkCloseReasonClass | undefined; + labels?: readonly string[] | undefined; + settledAt: string; + /** The reversal that overturned this action inside the horizon, if any. A reversed action is still + * the realized action — it is simply not a CONFIRMED one, exactly as `public-rule-precision.ts` + * treats a reversed decision: recorded, and subtracted from the confirmed count. */ + reversal: { kind: RealizedReversalKind; occurredAt: string } | null; + }; + +export type BenchmarkGroundTruthSet = { + schemaVersion: 1; + snapshotRef: string; + horizonDays: number; + frozenAt: string; + /** The horizon's inclusive end — `frozenAt + horizonDays`, computed once and published so a consumer + * never has to re-derive (and possibly re-derive differently) the window a label was settled in. */ + horizonEnd: string; + truths: BenchmarkGroundTruth[]; + /** Denominator discipline, published rather than left implicit: `scoreable` is what a scorer divides by; + * `unresolved` is excluded from it, and a climbing `unresolvedRate` is the signal the horizon is wrong. */ + coverage: { + workUnits: number; + scoreable: number; + unresolved: number; + /** `unresolved / workUnits`, 3dp. `null` for an empty work-unit set — never 0, which would read as + * "a fully resolved benchmark" (the same null-not-zero discipline as PublicRulePrecisionRow). */ + unresolvedRate: number | null; + }; +}; + +const MS_PER_DAY = 24 * 60 * 60 * 1000; + +/** The inclusive horizon end for a snapshot. Exported so the harness, the scorer, and any third-party + * verifier compute the SAME instant rather than three slightly different ones. */ +export function benchmarkHorizonEnd(frozenAt: string, horizonDays: number): string { + return new Date(Date.parse(frozenAt) + horizonDays * MS_PER_DAY).toISOString(); +} + +function withinHorizon(occurredAt: string, frozenAtMs: number, horizonEndMs: number): boolean { + const at = Date.parse(occurredAt); + return Number.isFinite(at) && at >= frozenAtMs && at <= horizonEndMs; +} + +/** + * Derive the settled, reversal-aware ground-truth set for one snapshot's work units. + * + * PURE: no clock, no IO, no randomness. Everything is decided from events dated inside the window, which is + * what makes the result stable regardless of when extraction runs (asserted as an invariant in the tests). + * + * `workUnitIds` is the authoritative roster from the snapshot — a work unit with no qualifying event is + * emitted as `unresolved` rather than omitted, so the unresolved rate has an honest denominator and a + * consumer can tell "nobody acted" apart from "the extractor forgot about it". + */ +export function deriveBenchmarkGroundTruth(input: { + snapshotRef: string; + frozenAt: string; + horizonDays: number; + workUnitIds: readonly string[]; + events: readonly RealizedMaintainerEvent[]; + reversals?: readonly RealizedReversalEvent[] | undefined; +}): BenchmarkGroundTruthSet { + const frozenAtMs = Date.parse(input.frozenAt); + const horizonEnd = benchmarkHorizonEnd(input.frozenAt, input.horizonDays); + const horizonEndMs = Date.parse(horizonEnd); + + // The SETTLED state is the LAST qualifying action in the window, not the first: a maintainer who labels, + // then requests changes, then merges has settled on the merge. Ties on identical timestamps keep the + // earlier-listed event, so a caller's stable input order yields a stable label. + const settled = new Map(); + for (const event of input.events) { + if (!withinHorizon(event.occurredAt, frozenAtMs, horizonEndMs)) continue; + const current = settled.get(event.workUnitId); + if (!current || Date.parse(event.occurredAt) > Date.parse(current.occurredAt)) settled.set(event.workUnitId, event); + } + + // A reversal counts only when it lands inside the same horizon AND strictly after the action it overturns + // — a reversal predating the settled action reversed something else, and counting it would mislabel an + // action nobody has yet overturned. The EARLIEST such reversal is recorded: the first overturning is the + // one that makes the action non-confirmed. + const reversalFor = (workUnitId: string, settledAt: string): { kind: RealizedReversalKind; occurredAt: string } | null => { + let best: RealizedReversalEvent | undefined; + for (const reversal of input.reversals ?? []) { + if (reversal.workUnitId !== workUnitId) continue; + if (!withinHorizon(reversal.occurredAt, frozenAtMs, horizonEndMs)) continue; + if (Date.parse(reversal.occurredAt) <= Date.parse(settledAt)) continue; + if (!best || Date.parse(reversal.occurredAt) < Date.parse(best.occurredAt)) best = reversal; + } + return best ? { kind: best.kind, occurredAt: best.occurredAt } : null; + }; + + const truths: BenchmarkGroundTruth[] = []; + let unresolved = 0; + for (const workUnitId of input.workUnitIds) { + const event = settled.get(workUnitId); + if (!event) { + unresolved += 1; + truths.push({ workUnitId, outcome: "unresolved" }); + continue; + } + truths.push({ + workUnitId, + outcome: "settled", + action: event.action, + // Spreads, not literal `undefined` values: an absent parameter must be OMITTED (the same + // exactOptionalPropertyTypes discipline every sibling module follows), so a `merge` truth cannot + // carry a present-but-undefined `reasonClass` that a strict comparison would trip over. + ...(event.action === "close" && event.reasonClass !== undefined ? { reasonClass: event.reasonClass } : {}), + ...(event.action === "label" && event.labels !== undefined ? { labels: [...event.labels] } : {}), + settledAt: event.occurredAt, + reversal: reversalFor(workUnitId, event.occurredAt), + }); + } + + const workUnits = input.workUnitIds.length; + return { + schemaVersion: 1, + snapshotRef: input.snapshotRef, + horizonDays: input.horizonDays, + frozenAt: input.frozenAt, + horizonEnd, + truths, + coverage: { + workUnits, + scoreable: workUnits - unresolved, + unresolved, + unresolvedRate: workUnits === 0 ? null : Math.round((unresolved / workUnits) * 1000) / 1000, + }, + }; +} + +/** The work units a scorer may divide by — `unresolved` never enters the denominator (#9261 requirement 4). + * Exported as the single definition of that rule so no scorer can re-implement it slightly differently. */ +export function scoreableGroundTruths( + set: BenchmarkGroundTruthSet, +): Array> { + return set.truths.filter((truth): truth is Extract => truth.outcome === "settled"); +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 881cd371b2..0ddea29fba 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -184,6 +184,7 @@ export * from "./calibration/reliability-curve.js"; export * from "./calibration/attestation-envelope.js"; export * from "./calibration/attester.js"; export * from "./calibration/benchmark-proposal.js"; +export * from "./calibration/benchmark-ground-truth.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/benchmark-ground-truth.test.ts b/packages/loopover-engine/test/benchmark-ground-truth.test.ts new file mode 100644 index 0000000000..ccbef3c905 --- /dev/null +++ b/packages/loopover-engine/test/benchmark-ground-truth.test.ts @@ -0,0 +1,210 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { benchmarkHorizonEnd, deriveBenchmarkGroundTruth, scoreableGroundTruths, type BenchmarkGroundTruth } from "../dist/index.js"; + +// #9261 (harness #9216, epic #8534): the labels every proposal is scored against. The three properties these +// tests exist to pin are the three the surrounding system already believes and a benchmark must not +// contradict: reversal-aware settlement, unresolved excluded from the denominator, and stability of a +// settled label regardless of when extraction runs. + +const FROZEN = "2026-07-01T00:00:00.000Z"; +const BASE = { snapshotRef: "a1".repeat(32), frozenAt: FROZEN, horizonDays: 14 }; + +/** Narrow one truth to the settled variant, failing the test loudly if it is not — `noUncheckedIndexedAccess` + * makes a destructured element possibly-undefined, which blocks the discriminant narrowing on its own. */ +function settledAt(set: { truths: readonly BenchmarkGroundTruth[] }, index = 0): Extract { + const truth = set.truths[index]; + assert.ok(truth && truth.outcome === "settled", `truths[${index}] is not settled`); + return truth; +} + +/** `frozenAt + days`, as an ISO string — the tests' own arithmetic, independent of the module's. */ +function at(days: number, hours = 0): string { + return new Date(Date.parse(FROZEN) + days * 86_400_000 + hours * 3_600_000).toISOString(); +} + +test("benchmarkHorizonEnd is the inclusive frozenAt + horizonDays instant", () => { + assert.equal(benchmarkHorizonEnd(FROZEN, 14), "2026-07-15T00:00:00.000Z"); + assert.equal(benchmarkHorizonEnd(FROZEN, 1), "2026-07-02T00:00:00.000Z"); +}); + +test("settles on the LAST qualifying action in the window, carrying only that action's own parameters", () => { + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["o/r#1", "o/r#2", "o/r#3"], + events: [ + // #1: labeled, then changes requested, then merged — the merge is the settled state. + { workUnitId: "o/r#1", action: "label", occurredAt: at(1), labels: ["bug"] }, + { workUnitId: "o/r#1", action: "request_changes", occurredAt: at(2) }, + { workUnitId: "o/r#1", action: "merge", occurredAt: at(3) }, + { workUnitId: "o/r#2", action: "close", occurredAt: at(4), reasonClass: "duplicate" }, + { workUnitId: "o/r#3", action: "label", occurredAt: at(5), labels: ["docs", "good-first-issue"] }, + ], + }); + const [first, second, third] = set.truths; + assert.deepEqual(first, { workUnitId: "o/r#1", outcome: "settled", action: "merge", settledAt: at(3), reversal: null }); + // A merge carries NO reasonClass/labels keys at all — not present-but-undefined. + assert.equal("reasonClass" in (first as object), false); + assert.equal("labels" in (first as object), false); + assert.deepEqual(second, { workUnitId: "o/r#2", outcome: "settled", action: "close", reasonClass: "duplicate", settledAt: at(4), reversal: null }); + assert.deepEqual(third, { workUnitId: "o/r#3", outcome: "settled", action: "label", labels: ["docs", "good-first-issue"], settledAt: at(5), reversal: null }); +}); + +test("REGRESSION: a merge later REVERTED inside the horizon is not a clean merge — the reversal is recorded", () => { + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["o/r#1"], + events: [{ workUnitId: "o/r#1", action: "merge", occurredAt: at(2) }], + reversals: [{ workUnitId: "o/r#1", kind: "reversal_reverted", occurredAt: at(5) }], + }); + const truth = settledAt(set); + // The realized ACTION is still the merge — it is simply not a confirmed one, exactly as + // public-rule-precision.ts records a reversed decision rather than erasing it. + assert.equal(truth.action, "merge"); + assert.deepEqual(truth.reversal, { kind: "reversal_reverted", occurredAt: at(5) }); + // A reversed unit is still SCOREABLE — the reversal changes confirmation, not whether it counts. + assert.equal(set.coverage.scoreable, 1); +}); + +test("the established reversal vocabulary is accepted verbatim, and the EARLIEST overturning wins", () => { + for (const kind of ["reversal_reopened", "reversal_reverted", "reversal_superseded"] as const) { + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["o/r#1"], + events: [{ workUnitId: "o/r#1", action: "close", occurredAt: at(1), reasonClass: "stale" }], + reversals: [ + { workUnitId: "o/r#1", kind: "reversal_superseded", occurredAt: at(9) }, + { workUnitId: "o/r#1", kind, occurredAt: at(4) }, + ], + }); + assert.deepEqual(settledAt(set).reversal, { kind, occurredAt: at(4) }); + } +}); + +test("INVARIANT: a reversal is ignored unless it lands inside the horizon AND strictly after the settled action", () => { + const events = [{ workUnitId: "o/r#1", action: "merge" as const, occurredAt: at(5) }]; + const cases = [ + ["before the settled action", at(3)], + ["at the same instant as the settled action", at(5)], + ["after the horizon end", at(30)], + ["before the snapshot was frozen", at(-2)], + ] as const; + for (const [why, occurredAt] of cases) { + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["o/r#1"], + events, + reversals: [{ workUnitId: "o/r#1", kind: "reversal_reverted", occurredAt }], + }); + assert.equal(settledAt(set).reversal, null, why); + } + // A reversal for a DIFFERENT work unit never attaches. + const other = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["o/r#1"], + events, + reversals: [{ workUnitId: "o/r#2", kind: "reversal_reverted", occurredAt: at(7) }], + }); + assert.equal(settledAt(other).reversal, null); +}); + +test("REGRESSION: a work unit with no action in the horizon is `unresolved` and leaves the denominator", () => { + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["acted", "silent", "too-early", "too-late"], + events: [ + { workUnitId: "acted", action: "merge", occurredAt: at(3) }, + // Outside the window in both directions: an action before the freeze is history the agent could + // already see, and one after the horizon is beyond the question that was asked. + { workUnitId: "too-early", action: "merge", occurredAt: at(-1) }, + { workUnitId: "too-late", action: "merge", occurredAt: at(20) }, + ], + }); + assert.deepEqual( + set.truths.map((truth) => [truth.workUnitId, truth.outcome]), + [["acted", "settled"], ["silent", "unresolved"], ["too-early", "unresolved"], ["too-late", "unresolved"]], + ); + // Excluded from the denominator — never counted as a correct abstention or an incorrect prediction. + assert.deepEqual(set.coverage, { workUnits: 4, scoreable: 1, unresolved: 3, unresolvedRate: 0.75 }); + assert.deepEqual(scoreableGroundTruths(set).map((truth) => truth.workUnitId), ["acted"]); +}); + +test("boundary events are INSIDE the window at both ends — inclusive, matching the published horizonEnd", () => { + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["at-freeze", "at-end"], + events: [ + { workUnitId: "at-freeze", action: "hold", occurredAt: FROZEN }, + { workUnitId: "at-end", action: "merge", occurredAt: benchmarkHorizonEnd(FROZEN, 14) }, + ], + }); + assert.equal(set.coverage.unresolved, 0); + assert.equal(set.horizonEnd, benchmarkHorizonEnd(FROZEN, 14)); +}); + +test("INVARIANT: ground truth is stable regardless of WHEN extraction runs, once the horizon has elapsed", () => { + // The same (snapshot, horizon) with the same in-window events must yield byte-identical output no matter + // how much later history has accumulated — a leaderboard that drifts under its own scores is not one. + const input = { + ...BASE, + workUnitIds: ["o/r#1", "o/r#2"], + events: [ + { workUnitId: "o/r#1", action: "merge" as const, occurredAt: at(3) }, + { workUnitId: "o/r#2", action: "close" as const, occurredAt: at(6), reasonClass: "spam" as const }, + ], + reversals: [{ workUnitId: "o/r#1", kind: "reversal_reverted" as const, occurredAt: at(8) }], + }; + const early = deriveBenchmarkGroundTruth(input); + // Extraction re-run much later, with a year of further history appended: none of it is in the window, + // so none of it may move a settled label. + const late = deriveBenchmarkGroundTruth({ + ...input, + events: [...input.events, { workUnitId: "o/r#1", action: "merge" as const, occurredAt: at(400) }], + reversals: [...input.reversals, { workUnitId: "o/r#2", kind: "reversal_reopened" as const, occurredAt: at(400) }], + }); + assert.deepEqual(late, early); +}); + +test("an unparseable timestamp is out-of-window, not a crash and not a silent settle", () => { + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["o/r#1"], + events: [{ workUnitId: "o/r#1", action: "merge", occurredAt: "not a date" }], + }); + assert.equal(set.truths[0]?.outcome, "unresolved"); +}); + +test("identical timestamps keep the earlier-listed event, so a stable input order yields a stable label", () => { + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["o/r#1"], + events: [ + { workUnitId: "o/r#1", action: "merge", occurredAt: at(2) }, + { workUnitId: "o/r#1", action: "close", occurredAt: at(2), reasonClass: "defective" }, + ], + }); + assert.equal(settledAt(set).action, "merge"); +}); + +test("an empty work-unit roster reports a NULL unresolved rate, never 0 (which would read as fully resolved)", () => { + const set = deriveBenchmarkGroundTruth({ ...BASE, workUnitIds: [], events: [] }); + assert.deepEqual(set.coverage, { workUnits: 0, scoreable: 0, unresolved: 0, unresolvedRate: null }); + assert.deepEqual(scoreableGroundTruths(set), []); +}); + +test("close/label parameters are dropped when the settled action is not the one they belong to", () => { + // A malformed upstream event carrying a reasonClass on a merge must not leak it into the label. + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["o/r#1", "o/r#2"], + events: [ + { workUnitId: "o/r#1", action: "merge", occurredAt: at(1), reasonClass: "spam", labels: ["x"] }, + // A close whose reasonClass is genuinely absent stays absent rather than being invented. + { workUnitId: "o/r#2", action: "close", occurredAt: at(1) }, + ], + }); + assert.equal("reasonClass" in (set.truths[0] as object), false); + assert.equal("labels" in (set.truths[0] as object), false); + assert.equal("reasonClass" in (set.truths[1] as object), false); +});