diff --git a/packages/loopover-engine/src/calibration/benchmark-ground-truth.ts b/packages/loopover-engine/src/calibration/benchmark-ground-truth.ts new file mode 100644 index 000000000..1a1a39566 --- /dev/null +++ b/packages/loopover-engine/src/calibration/benchmark-ground-truth.ts @@ -0,0 +1,203 @@ +// Realized-history ground truth for the frozen-repo benchmark (#9261, harness #9216, epic #8534). +// +// For each work unit in a snapshot: what the maintainer ACTUALLY did within the prediction horizon. These +// are the labels every candidate proposal (#9260) is scored against, so they are emitted in that module's +// exact action vocabulary — proposal and outcome are directly comparable, with no translation layer that +// could drift. +// +// Three rules this module exists to enforce, each of which the surrounding system already believes and which +// a benchmark must not quietly contradict: +// +// REVERSAL-AWARE SETTLEMENT. A merge later reverted is not a clean merge. The realized outcome is the +// SETTLED state at horizon end, with the reversal recorded alongside — the same treatment that makes +// `public-rule-precision.ts`'s precision honest (it subtracts reversals from confirmations rather than +// counting the original decision). The reversal vocabulary is the established one +// (`reversal_reopened` / `reversal_reverted` / `reversal_superseded`), not a new benchmark-local set, so +// benchmark ground truth and internal backtest ground truth can never disagree about the same event. +// +// UNRESOLVED IS EXPLICIT AND EXCLUDED. A work unit with no maintainer action inside the horizon is +// `unresolved`. It is NOT silently counted as a correct abstention (which would reward an agent for +// declining exactly the cases nobody decided) nor as an incorrect prediction (which would punish an agent +// for the horizon being too short). It leaves the scoring denominator entirely, and the unresolved RATE is +// published so a climbing rate is legible as "the horizon is wrong" rather than as a scoring artifact. +// +// STABILITY. Ground truth for a given (snapshot, horizon) must not depend on WHEN extraction runs, once +// the horizon has elapsed. Every decision here is a pure function of events dated inside the window, so a +// later event — including a later reversal — cannot retroactively change a settled label. That is what +// makes a leaderboard reproducible instead of drifting under its own scores. +// +// Pure core (this module); the IO wrapper that reads events from D1 lives engine-side of the harness, the +// same split every sibling calibration module uses. + +import type { BenchmarkActionKind, BenchmarkCloseReasonClass } from "./benchmark-proposal.js"; + +/** The established reversal vocabulary — deliberately imported as literals from the same set the audit-event + * table and `public-rule-precision.ts` already use, never a benchmark-local re-invention. */ +export type RealizedReversalKind = "reversal_reopened" | "reversal_reverted" | "reversal_superseded"; + +/** One realized maintainer event inside (or outside) a horizon, already normalized by the IO wrapper. */ +export type RealizedMaintainerEvent = { + workUnitId: string; + /** Same closed vocabulary the agent proposes in — comparison needs no translation. */ + action: BenchmarkActionKind; + /** ISO-8601. Events at exactly the horizon end are INSIDE the window (inclusive), matching how the + * corpus builder's own windowing treats a boundary event as belonging to the window it closes. */ + occurredAt: string; + /** Present only for a `close`; the class realized history settled on. */ + reasonClass?: BenchmarkCloseReasonClass | undefined; + /** Present only for a `label`. */ + labels?: readonly string[] | undefined; +}; + +/** A reversal of a previously-realized action, in the established vocabulary. */ +export type RealizedReversalEvent = { + workUnitId: string; + kind: RealizedReversalKind; + occurredAt: string; +}; + +/** The settled label for ONE work unit. `unresolved` carries no action by construction — the type makes + * "unresolved but somehow also a merge" unrepresentable rather than merely untested. */ +export type BenchmarkGroundTruth = + | { workUnitId: string; outcome: "unresolved" } + | { + workUnitId: string; + outcome: "settled"; + action: BenchmarkActionKind; + reasonClass?: BenchmarkCloseReasonClass | undefined; + labels?: readonly string[] | undefined; + settledAt: string; + /** The reversal that overturned this action inside the horizon, if any. A reversed action is still + * the realized action — it is simply not a CONFIRMED one, exactly as `public-rule-precision.ts` + * treats a reversed decision: recorded, and subtracted from the confirmed count. */ + reversal: { kind: RealizedReversalKind; occurredAt: string } | null; + }; + +export type BenchmarkGroundTruthSet = { + schemaVersion: 1; + snapshotRef: string; + horizonDays: number; + frozenAt: string; + /** The horizon's inclusive end — `frozenAt + horizonDays`, computed once and published so a consumer + * never has to re-derive (and possibly re-derive differently) the window a label was settled in. */ + horizonEnd: string; + truths: BenchmarkGroundTruth[]; + /** Denominator discipline, published rather than left implicit: `scoreable` is what a scorer divides by; + * `unresolved` is excluded from it, and a climbing `unresolvedRate` is the signal the horizon is wrong. */ + coverage: { + workUnits: number; + scoreable: number; + unresolved: number; + /** `unresolved / workUnits`, 3dp. `null` for an empty work-unit set — never 0, which would read as + * "a fully resolved benchmark" (the same null-not-zero discipline as PublicRulePrecisionRow). */ + unresolvedRate: number | null; + }; +}; + +const MS_PER_DAY = 24 * 60 * 60 * 1000; + +/** The inclusive horizon end for a snapshot. Exported so the harness, the scorer, and any third-party + * verifier compute the SAME instant rather than three slightly different ones. */ +export function benchmarkHorizonEnd(frozenAt: string, horizonDays: number): string { + return new Date(Date.parse(frozenAt) + horizonDays * MS_PER_DAY).toISOString(); +} + +function withinHorizon(occurredAt: string, frozenAtMs: number, horizonEndMs: number): boolean { + const at = Date.parse(occurredAt); + return Number.isFinite(at) && at >= frozenAtMs && at <= horizonEndMs; +} + +/** + * Derive the settled, reversal-aware ground-truth set for one snapshot's work units. + * + * PURE: no clock, no IO, no randomness. Everything is decided from events dated inside the window, which is + * what makes the result stable regardless of when extraction runs (asserted as an invariant in the tests). + * + * `workUnitIds` is the authoritative roster from the snapshot — a work unit with no qualifying event is + * emitted as `unresolved` rather than omitted, so the unresolved rate has an honest denominator and a + * consumer can tell "nobody acted" apart from "the extractor forgot about it". + */ +export function deriveBenchmarkGroundTruth(input: { + snapshotRef: string; + frozenAt: string; + horizonDays: number; + workUnitIds: readonly string[]; + events: readonly RealizedMaintainerEvent[]; + reversals?: readonly RealizedReversalEvent[] | undefined; +}): BenchmarkGroundTruthSet { + const frozenAtMs = Date.parse(input.frozenAt); + const horizonEnd = benchmarkHorizonEnd(input.frozenAt, input.horizonDays); + const horizonEndMs = Date.parse(horizonEnd); + + // The SETTLED state is the LAST qualifying action in the window, not the first: a maintainer who labels, + // then requests changes, then merges has settled on the merge. Ties on identical timestamps keep the + // earlier-listed event, so a caller's stable input order yields a stable label. + const settled = new Map(); + for (const event of input.events) { + if (!withinHorizon(event.occurredAt, frozenAtMs, horizonEndMs)) continue; + const current = settled.get(event.workUnitId); + if (!current || Date.parse(event.occurredAt) > Date.parse(current.occurredAt)) settled.set(event.workUnitId, event); + } + + // A reversal counts only when it lands inside the same horizon AND strictly after the action it overturns + // — a reversal predating the settled action reversed something else, and counting it would mislabel an + // action nobody has yet overturned. The EARLIEST such reversal is recorded: the first overturning is the + // one that makes the action non-confirmed. + const reversalFor = (workUnitId: string, settledAt: string): { kind: RealizedReversalKind; occurredAt: string } | null => { + let best: RealizedReversalEvent | undefined; + for (const reversal of input.reversals ?? []) { + if (reversal.workUnitId !== workUnitId) continue; + if (!withinHorizon(reversal.occurredAt, frozenAtMs, horizonEndMs)) continue; + if (Date.parse(reversal.occurredAt) <= Date.parse(settledAt)) continue; + if (!best || Date.parse(reversal.occurredAt) < Date.parse(best.occurredAt)) best = reversal; + } + return best ? { kind: best.kind, occurredAt: best.occurredAt } : null; + }; + + const truths: BenchmarkGroundTruth[] = []; + let unresolved = 0; + for (const workUnitId of input.workUnitIds) { + const event = settled.get(workUnitId); + if (!event) { + unresolved += 1; + truths.push({ workUnitId, outcome: "unresolved" }); + continue; + } + truths.push({ + workUnitId, + outcome: "settled", + action: event.action, + // Spreads, not literal `undefined` values: an absent parameter must be OMITTED (the same + // exactOptionalPropertyTypes discipline every sibling module follows), so a `merge` truth cannot + // carry a present-but-undefined `reasonClass` that a strict comparison would trip over. + ...(event.action === "close" && event.reasonClass !== undefined ? { reasonClass: event.reasonClass } : {}), + ...(event.action === "label" && event.labels !== undefined ? { labels: [...event.labels] } : {}), + settledAt: event.occurredAt, + reversal: reversalFor(workUnitId, event.occurredAt), + }); + } + + const workUnits = input.workUnitIds.length; + return { + schemaVersion: 1, + snapshotRef: input.snapshotRef, + horizonDays: input.horizonDays, + frozenAt: input.frozenAt, + horizonEnd, + truths, + coverage: { + workUnits, + scoreable: workUnits - unresolved, + unresolved, + unresolvedRate: workUnits === 0 ? null : Math.round((unresolved / workUnits) * 1000) / 1000, + }, + }; +} + +/** The work units a scorer may divide by — `unresolved` never enters the denominator (#9261 requirement 4). + * Exported as the single definition of that rule so no scorer can re-implement it slightly differently. */ +export function scoreableGroundTruths( + set: BenchmarkGroundTruthSet, +): Array> { + return set.truths.filter((truth): truth is Extract => truth.outcome === "settled"); +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 881cd371b..0ddea29fb 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -184,6 +184,7 @@ export * from "./calibration/reliability-curve.js"; export * from "./calibration/attestation-envelope.js"; export * from "./calibration/attester.js"; export * from "./calibration/benchmark-proposal.js"; +export * from "./calibration/benchmark-ground-truth.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/benchmark-ground-truth.test.ts b/packages/loopover-engine/test/benchmark-ground-truth.test.ts new file mode 100644 index 000000000..ccbef3c90 --- /dev/null +++ b/packages/loopover-engine/test/benchmark-ground-truth.test.ts @@ -0,0 +1,210 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { benchmarkHorizonEnd, deriveBenchmarkGroundTruth, scoreableGroundTruths, type BenchmarkGroundTruth } from "../dist/index.js"; + +// #9261 (harness #9216, epic #8534): the labels every proposal is scored against. The three properties these +// tests exist to pin are the three the surrounding system already believes and a benchmark must not +// contradict: reversal-aware settlement, unresolved excluded from the denominator, and stability of a +// settled label regardless of when extraction runs. + +const FROZEN = "2026-07-01T00:00:00.000Z"; +const BASE = { snapshotRef: "a1".repeat(32), frozenAt: FROZEN, horizonDays: 14 }; + +/** Narrow one truth to the settled variant, failing the test loudly if it is not — `noUncheckedIndexedAccess` + * makes a destructured element possibly-undefined, which blocks the discriminant narrowing on its own. */ +function settledAt(set: { truths: readonly BenchmarkGroundTruth[] }, index = 0): Extract { + const truth = set.truths[index]; + assert.ok(truth && truth.outcome === "settled", `truths[${index}] is not settled`); + return truth; +} + +/** `frozenAt + days`, as an ISO string — the tests' own arithmetic, independent of the module's. */ +function at(days: number, hours = 0): string { + return new Date(Date.parse(FROZEN) + days * 86_400_000 + hours * 3_600_000).toISOString(); +} + +test("benchmarkHorizonEnd is the inclusive frozenAt + horizonDays instant", () => { + assert.equal(benchmarkHorizonEnd(FROZEN, 14), "2026-07-15T00:00:00.000Z"); + assert.equal(benchmarkHorizonEnd(FROZEN, 1), "2026-07-02T00:00:00.000Z"); +}); + +test("settles on the LAST qualifying action in the window, carrying only that action's own parameters", () => { + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["o/r#1", "o/r#2", "o/r#3"], + events: [ + // #1: labeled, then changes requested, then merged — the merge is the settled state. + { workUnitId: "o/r#1", action: "label", occurredAt: at(1), labels: ["bug"] }, + { workUnitId: "o/r#1", action: "request_changes", occurredAt: at(2) }, + { workUnitId: "o/r#1", action: "merge", occurredAt: at(3) }, + { workUnitId: "o/r#2", action: "close", occurredAt: at(4), reasonClass: "duplicate" }, + { workUnitId: "o/r#3", action: "label", occurredAt: at(5), labels: ["docs", "good-first-issue"] }, + ], + }); + const [first, second, third] = set.truths; + assert.deepEqual(first, { workUnitId: "o/r#1", outcome: "settled", action: "merge", settledAt: at(3), reversal: null }); + // A merge carries NO reasonClass/labels keys at all — not present-but-undefined. + assert.equal("reasonClass" in (first as object), false); + assert.equal("labels" in (first as object), false); + assert.deepEqual(second, { workUnitId: "o/r#2", outcome: "settled", action: "close", reasonClass: "duplicate", settledAt: at(4), reversal: null }); + assert.deepEqual(third, { workUnitId: "o/r#3", outcome: "settled", action: "label", labels: ["docs", "good-first-issue"], settledAt: at(5), reversal: null }); +}); + +test("REGRESSION: a merge later REVERTED inside the horizon is not a clean merge — the reversal is recorded", () => { + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["o/r#1"], + events: [{ workUnitId: "o/r#1", action: "merge", occurredAt: at(2) }], + reversals: [{ workUnitId: "o/r#1", kind: "reversal_reverted", occurredAt: at(5) }], + }); + const truth = settledAt(set); + // The realized ACTION is still the merge — it is simply not a confirmed one, exactly as + // public-rule-precision.ts records a reversed decision rather than erasing it. + assert.equal(truth.action, "merge"); + assert.deepEqual(truth.reversal, { kind: "reversal_reverted", occurredAt: at(5) }); + // A reversed unit is still SCOREABLE — the reversal changes confirmation, not whether it counts. + assert.equal(set.coverage.scoreable, 1); +}); + +test("the established reversal vocabulary is accepted verbatim, and the EARLIEST overturning wins", () => { + for (const kind of ["reversal_reopened", "reversal_reverted", "reversal_superseded"] as const) { + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["o/r#1"], + events: [{ workUnitId: "o/r#1", action: "close", occurredAt: at(1), reasonClass: "stale" }], + reversals: [ + { workUnitId: "o/r#1", kind: "reversal_superseded", occurredAt: at(9) }, + { workUnitId: "o/r#1", kind, occurredAt: at(4) }, + ], + }); + assert.deepEqual(settledAt(set).reversal, { kind, occurredAt: at(4) }); + } +}); + +test("INVARIANT: a reversal is ignored unless it lands inside the horizon AND strictly after the settled action", () => { + const events = [{ workUnitId: "o/r#1", action: "merge" as const, occurredAt: at(5) }]; + const cases = [ + ["before the settled action", at(3)], + ["at the same instant as the settled action", at(5)], + ["after the horizon end", at(30)], + ["before the snapshot was frozen", at(-2)], + ] as const; + for (const [why, occurredAt] of cases) { + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["o/r#1"], + events, + reversals: [{ workUnitId: "o/r#1", kind: "reversal_reverted", occurredAt }], + }); + assert.equal(settledAt(set).reversal, null, why); + } + // A reversal for a DIFFERENT work unit never attaches. + const other = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["o/r#1"], + events, + reversals: [{ workUnitId: "o/r#2", kind: "reversal_reverted", occurredAt: at(7) }], + }); + assert.equal(settledAt(other).reversal, null); +}); + +test("REGRESSION: a work unit with no action in the horizon is `unresolved` and leaves the denominator", () => { + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["acted", "silent", "too-early", "too-late"], + events: [ + { workUnitId: "acted", action: "merge", occurredAt: at(3) }, + // Outside the window in both directions: an action before the freeze is history the agent could + // already see, and one after the horizon is beyond the question that was asked. + { workUnitId: "too-early", action: "merge", occurredAt: at(-1) }, + { workUnitId: "too-late", action: "merge", occurredAt: at(20) }, + ], + }); + assert.deepEqual( + set.truths.map((truth) => [truth.workUnitId, truth.outcome]), + [["acted", "settled"], ["silent", "unresolved"], ["too-early", "unresolved"], ["too-late", "unresolved"]], + ); + // Excluded from the denominator — never counted as a correct abstention or an incorrect prediction. + assert.deepEqual(set.coverage, { workUnits: 4, scoreable: 1, unresolved: 3, unresolvedRate: 0.75 }); + assert.deepEqual(scoreableGroundTruths(set).map((truth) => truth.workUnitId), ["acted"]); +}); + +test("boundary events are INSIDE the window at both ends — inclusive, matching the published horizonEnd", () => { + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["at-freeze", "at-end"], + events: [ + { workUnitId: "at-freeze", action: "hold", occurredAt: FROZEN }, + { workUnitId: "at-end", action: "merge", occurredAt: benchmarkHorizonEnd(FROZEN, 14) }, + ], + }); + assert.equal(set.coverage.unresolved, 0); + assert.equal(set.horizonEnd, benchmarkHorizonEnd(FROZEN, 14)); +}); + +test("INVARIANT: ground truth is stable regardless of WHEN extraction runs, once the horizon has elapsed", () => { + // The same (snapshot, horizon) with the same in-window events must yield byte-identical output no matter + // how much later history has accumulated — a leaderboard that drifts under its own scores is not one. + const input = { + ...BASE, + workUnitIds: ["o/r#1", "o/r#2"], + events: [ + { workUnitId: "o/r#1", action: "merge" as const, occurredAt: at(3) }, + { workUnitId: "o/r#2", action: "close" as const, occurredAt: at(6), reasonClass: "spam" as const }, + ], + reversals: [{ workUnitId: "o/r#1", kind: "reversal_reverted" as const, occurredAt: at(8) }], + }; + const early = deriveBenchmarkGroundTruth(input); + // Extraction re-run much later, with a year of further history appended: none of it is in the window, + // so none of it may move a settled label. + const late = deriveBenchmarkGroundTruth({ + ...input, + events: [...input.events, { workUnitId: "o/r#1", action: "merge" as const, occurredAt: at(400) }], + reversals: [...input.reversals, { workUnitId: "o/r#2", kind: "reversal_reopened" as const, occurredAt: at(400) }], + }); + assert.deepEqual(late, early); +}); + +test("an unparseable timestamp is out-of-window, not a crash and not a silent settle", () => { + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["o/r#1"], + events: [{ workUnitId: "o/r#1", action: "merge", occurredAt: "not a date" }], + }); + assert.equal(set.truths[0]?.outcome, "unresolved"); +}); + +test("identical timestamps keep the earlier-listed event, so a stable input order yields a stable label", () => { + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["o/r#1"], + events: [ + { workUnitId: "o/r#1", action: "merge", occurredAt: at(2) }, + { workUnitId: "o/r#1", action: "close", occurredAt: at(2), reasonClass: "defective" }, + ], + }); + assert.equal(settledAt(set).action, "merge"); +}); + +test("an empty work-unit roster reports a NULL unresolved rate, never 0 (which would read as fully resolved)", () => { + const set = deriveBenchmarkGroundTruth({ ...BASE, workUnitIds: [], events: [] }); + assert.deepEqual(set.coverage, { workUnits: 0, scoreable: 0, unresolved: 0, unresolvedRate: null }); + assert.deepEqual(scoreableGroundTruths(set), []); +}); + +test("close/label parameters are dropped when the settled action is not the one they belong to", () => { + // A malformed upstream event carrying a reasonClass on a merge must not leak it into the label. + const set = deriveBenchmarkGroundTruth({ + ...BASE, + workUnitIds: ["o/r#1", "o/r#2"], + events: [ + { workUnitId: "o/r#1", action: "merge", occurredAt: at(1), reasonClass: "spam", labels: ["x"] }, + // A close whose reasonClass is genuinely absent stays absent rather than being invented. + { workUnitId: "o/r#2", action: "close", occurredAt: at(1) }, + ], + }); + assert.equal("reasonClass" in (set.truths[0] as object), false); + assert.equal("labels" in (set.truths[0] as object), false); + assert.equal("reasonClass" in (set.truths[1] as object), false); +});