diff --git a/packages/loopover-engine/src/calibration/benchmark-anti-overfit.ts b/packages/loopover-engine/src/calibration/benchmark-anti-overfit.ts new file mode 100644 index 000000000..f60e2f6ea --- /dev/null +++ b/packages/loopover-engine/src/calibration/benchmark-anti-overfit.ts @@ -0,0 +1,195 @@ +// Anti-overfit controls for the public benchmark (#9263, harness #9216, epic #8534). +// +// A public leaderboard creates a direct incentive to overfit, and an overfit agent is worse than useless: it +// looks best exactly when it generalizes worst. The internal backtest gate already enforces this discipline; +// this module applies the same controls to EXTERNAL submissions, plus the ones only an adversarial public +// benchmark needs. Every knob below is one an adversary optimizes against, so each carries its reasoning +// rather than a bare value. +// +// ── DECISION 1: SPLIT GRANULARITY IS THE REPO, NOT THE WORK UNIT ────────────────────────────────────── +// The split reuses `splitBacktestCorpus` verbatim (requirement 1 — no second split mechanism), but the key +// it splits on is the REPO, so every work unit belonging to a repo lands in the same slice. Work-unit-level +// splitting leaks badly: two PRs in the same repo share a maintainer, a review culture, a CI setup and often +// the same files, so an agent that saw repo A's visible PRs has effectively seen the answer pattern for repo +// A's held-out PRs. Splitting per repo is what makes "held out" mean "generalizes to a repo it has never +// seen" — which is the property actually worth measuring, and the one #9216 requirement 3 names. +// +// ── DECISION 2: HELD-OUT SCORES ARE NOT PUBLISHED PER SUBMISSION ───────────────────────────────────── +// A leaderboard that reports a held-out score on every attempt converts the held-out set into a visible set +// within a few dozen submissions: each score is an oracle query, and differencing successive scores recovers +// per-unit membership. Held-out results publish on a fixed cadence or at evaluation close, never on demand; +// `heldOutPublicationDecision` is the single place that rule is decided, and #9265's emitter is where it is +// enforced, so a leaderboard cannot leak membership by accident. +// +// ── DECISION 3: SUBMISSION CAP PER (AGENT, WINDOW) ─────────────────────────────────────────────────── +// Unbounded resubmission against a fixed corpus is gradient descent on the test set by brute force — with +// enough attempts a random-search agent tops the board having learned nothing. The cap is per (agent, +// benchmark window) rather than global or per-day: a per-day cap just spreads the same brute force over more +// days, and a global cap would punish an agent that keeps competing across rotations. +// +// ── DECISION 4: ROTATION RETIRES A WINDOW, IT DOES NOT EXTEND IT ───────────────────────────────────── +// A benchmark public for a year is not a test set any more — it is training data with extra steps. A window +// has a fixed evaluation close; after it, the snapshot is RETIRED and a fresh one takes over. Retired +// windows stay published for historical comparison but stop being the scoring basis, which is exactly the +// distinction that keeps an old leaderboard honest instead of quietly authoritative. +// +// Same purity contract as the rest of this module family: no IO, no randomness, no wall-clock reads — the +// caller supplies `now` wherever a time comparison is needed, so every decision here is reproducible. + +import type { BacktestCase } from "./backtest-corpus.js"; +import { splitBacktestCorpus } from "./backtest-split.js"; + +/** Default share of REPOS (not work units — see decision 1) withheld from the visible slice. */ +export const DEFAULT_HELD_OUT_FRACTION = 0.25; + +/** Default submissions permitted per (agent, benchmark window). See decision 3 for why the cap is scoped + * this way; the value itself is deliberately generous enough for honest iteration and far too small for + * brute-force search over a fixed corpus. */ +export const DEFAULT_SUBMISSION_CAP = 20; + +export type BenchmarkSplitPolicy = { + /** Unguessable without knowing it; identical across runs. Never published while a window is open. */ + splitSeed: string; + heldOutFraction: number; +}; + +export type BenchmarkWorkUnitRef = { + workUnitId: string; + /** `owner/repo` — the split key (decision 1). */ + repoFullName: string; +}; + +/** + * Partition work units into visible and held-out slices at REPO granularity, through the shared + * `splitBacktestCorpus` primitive. + * + * The reuse is literal: one synthetic case per distinct repo, with the repo in the `targetKey` slot, so the + * assignment digest is byte-identically the one the internal backtest split already computes. A repo's + * assignment therefore depends only on (seed, benchmarkId, repo) — never on corpus size or ordering — so a + * benchmark that grows over time never reshuffles which repos were already held out. + */ +export function splitBenchmarkWorkUnits( + benchmarkId: string, + workUnits: readonly BenchmarkWorkUnitRef[], + policy: BenchmarkSplitPolicy, +): { visible: BenchmarkWorkUnitRef[]; heldOut: BenchmarkWorkUnitRef[]; heldOutRepoCount: number; visibleRepoCount: number } { + const repos = [...new Set(workUnits.map((unit) => unit.repoFullName))]; + const repoCases: BacktestCase[] = repos.map((repoFullName) => ({ + ruleId: benchmarkId, + targetKey: repoFullName, + outcome: "repo", + label: "confirmed", + firedAt: "", + decidedAt: "", + })); + const split = splitBacktestCorpus(repoCases, policy.heldOutFraction, policy.splitSeed); + const heldOutRepos = new Set(split.heldOut.map((repoCase) => repoCase.targetKey)); + const visible: BenchmarkWorkUnitRef[] = []; + const heldOut: BenchmarkWorkUnitRef[] = []; + for (const unit of workUnits) { + if (heldOutRepos.has(unit.repoFullName)) heldOut.push(unit); + else visible.push(unit); + } + return { visible, heldOut, heldOutRepoCount: heldOutRepos.size, visibleRepoCount: repos.length - heldOutRepos.size }; +} + +export type BenchmarkWindow = { + benchmarkId: string; + snapshotRef: string; + opensAt: string; + /** Inclusive evaluation close. After this the window is RETIRED (decision 4), not extended. */ + closesAt: string; + /** Held-out scores may also publish periodically before close; 0/absent means "at close only". */ + heldOutPublishEveryDays?: number | undefined; +}; + +export type BenchmarkWindowState = "pending" | "open" | "retired"; + +/** Where `now` sits relative to a window. A retired window stays readable for historical comparison but is + * no longer a scoring basis — the caller enforces that by refusing submissions (below). */ +export function benchmarkWindowState(window: BenchmarkWindow, now: string): BenchmarkWindowState { + const at = Date.parse(now); + if (at < Date.parse(window.opensAt)) return "pending"; + return at > Date.parse(window.closesAt) ? "retired" : "open"; +} + +export type SubmissionDecision = + | { accepted: true; submissionIndex: number; remaining: number } + | { accepted: false; reason: "window_not_open" | "window_retired" | "cap_reached"; remaining: number }; + +/** + * Decide whether one more submission from this agent is admissible. + * + * PURE and total: it never throws for ordinarily-invalid input, and the refusal reason is NAMED so a + * submitter learns which control stopped them (a cap they can wait out, versus a window that will never + * reopen) rather than seeing an opaque rejection. + */ +export function decideSubmission(input: { + window: BenchmarkWindow; + now: string; + /** Submissions this agent has already made against THIS window. */ + priorSubmissions: number; + cap?: number | undefined; +}): SubmissionDecision { + const cap = input.cap ?? DEFAULT_SUBMISSION_CAP; + const remaining = Math.max(0, cap - input.priorSubmissions); + const state = benchmarkWindowState(input.window, input.now); + if (state === "pending") return { accepted: false, reason: "window_not_open", remaining }; + if (state === "retired") return { accepted: false, reason: "window_retired", remaining }; + if (remaining <= 0) return { accepted: false, reason: "cap_reached", remaining: 0 }; + return { accepted: true, submissionIndex: input.priorSubmissions + 1, remaining: remaining - 1 }; +} + +export type HeldOutPublicationDecision = + | { publish: true; reason: "evaluation_closed" | "scheduled_cadence" } + | { publish: false; reason: "window_open_between_cadences" | "window_not_open" }; + +/** + * Decide whether held-out scores may be published right now (decision 2). + * + * At or after close, always — the window is over and there is nothing left to leak into. Before close, only + * on the configured cadence boundary, counted in whole periods since `opensAt`, so the publication instants + * are a fixed public schedule rather than something a submitter can trigger by submitting. + */ +export function heldOutPublicationDecision(window: BenchmarkWindow, now: string): HeldOutPublicationDecision { + const state = benchmarkWindowState(window, now); + if (state === "pending") return { publish: false, reason: "window_not_open" }; + if (state === "retired") return { publish: true, reason: "evaluation_closed" }; + const everyDays = window.heldOutPublishEveryDays ?? 0; + if (everyDays <= 0) return { publish: false, reason: "window_open_between_cadences" }; + const elapsedMs = Date.parse(now) - Date.parse(window.opensAt); + const periodMs = everyDays * 24 * 60 * 60 * 1000; + // A boundary instant publishes; anything strictly between boundaries does not. + return elapsedMs > 0 && elapsedMs % periodMs === 0 + ? { publish: true, reason: "scheduled_cadence" } + : { publish: false, reason: "window_open_between_cadences" }; +} + +/** What a leaderboard is allowed to show for one submission while its window is open. */ +export type PublishableScores = { + visible: T; + /** Present ONLY when {@link heldOutPublicationDecision} allowed it. `null` is the honest "not published + * yet", and carries no count, no fraction and no per-unit membership — anything derived from the + * held-out slice is an oracle query, so the shape itself refuses to answer one. */ + heldOut: T | null; + heldOutPublication: HeldOutPublicationDecision; +}; + +/** + * Gate a scored pair through the publication policy. + * + * The held-out score is DROPPED, not merely hidden behind a flag: a caller cannot forget to check the flag + * and serialize a field that is present in memory. That is the difference between a policy and a habit. + */ +export function gateHeldOutPublication( + window: BenchmarkWindow, + now: string, + scores: { visible: T; heldOut: T }, +): PublishableScores { + const decision = heldOutPublicationDecision(window, now); + return { + visible: scores.visible, + heldOut: decision.publish ? scores.heldOut : null, + heldOutPublication: decision, + }; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 5989875d6..d9fb9903d 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -186,6 +186,7 @@ export * from "./calibration/attester.js"; export * from "./calibration/benchmark-proposal.js"; export * from "./calibration/benchmark-ground-truth.js"; export * from "./calibration/benchmark-score.js"; +export * from "./calibration/benchmark-anti-overfit.js"; export { GOVERNOR_LEDGER_EVENT_TYPES, normalizeGovernorLedgerEvent, diff --git a/packages/loopover-engine/test/benchmark-anti-overfit.test.ts b/packages/loopover-engine/test/benchmark-anti-overfit.test.ts new file mode 100644 index 000000000..9a582af87 --- /dev/null +++ b/packages/loopover-engine/test/benchmark-anti-overfit.test.ts @@ -0,0 +1,199 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + benchmarkWindowState, + decideSubmission, + DEFAULT_HELD_OUT_FRACTION, + DEFAULT_SUBMISSION_CAP, + gateHeldOutPublication, + heldOutPublicationDecision, + splitBacktestCorpus, + splitBenchmarkWorkUnits, + type BenchmarkWindow, + type BenchmarkWorkUnitRef, +} from "../dist/index.js"; + +// #9263 (harness #9216, epic #8534): the controls that make a leaderboard position mean "generalizes" +// rather than "has submitted the most times". Each of the four recorded decisions in the module header has +// its enforcing test here, and the load-bearing one is that held-out MEMBERSHIP is not inferable from +// anything the published shape can carry. + +const POLICY = { splitSeed: "seed-2026q3", heldOutFraction: DEFAULT_HELD_OUT_FRACTION }; + +/** Work units across many repos, several units per repo — the shape decision 1 is about. */ +function unitsAcrossRepos(repoCount: number, perRepo = 3): BenchmarkWorkUnitRef[] { + const units: BenchmarkWorkUnitRef[] = []; + for (let repo = 0; repo < repoCount; repo += 1) { + for (let index = 0; index < perRepo; index += 1) { + units.push({ workUnitId: `owner/repo${repo}#${index}`, repoFullName: `owner/repo${repo}` }); + } + } + return units; +} + +const WINDOW: BenchmarkWindow = { + benchmarkId: "bench-2026q3", + snapshotRef: "a1".repeat(32), + opensAt: "2026-07-01T00:00:00.000Z", + closesAt: "2026-09-30T00:00:00.000Z", +}; + +test("DECISION 1: the split is at REPO granularity — a repo's units never straddle the boundary", () => { + const units = unitsAcrossRepos(24); + const split = splitBenchmarkWorkUnits("bench-2026q3", units, POLICY); + const heldOutRepos = new Set(split.heldOut.map((unit) => unit.repoFullName)); + const visibleRepos = new Set(split.visible.map((unit) => unit.repoFullName)); + // The two repo sets are disjoint: no repo contributes to both slices, which is the whole property. + for (const repo of heldOutRepos) assert.equal(visibleRepos.has(repo), false, `${repo} straddles the split`); + assert.equal(heldOutRepos.size + visibleRepos.size, 24); + assert.equal(split.heldOutRepoCount, heldOutRepos.size); + assert.equal(split.visibleRepoCount, visibleRepos.size); + // Every unit is accounted for exactly once. + assert.equal(split.visible.length + split.heldOut.length, units.length); +}); + +test("DECISION 1: the assignment IS the shared primitive's, not a lookalike", () => { + // Recomputing through splitBacktestCorpus directly must name the same held-out repos — a second split + // mechanism is exactly what requirement 1 forbids, so the equality is asserted rather than assumed. + const units = unitsAcrossRepos(16); + const repos = [...new Set(units.map((unit) => unit.repoFullName))]; + const direct = splitBacktestCorpus( + repos.map((repoFullName) => ({ + ruleId: "bench-2026q3", + targetKey: repoFullName, + outcome: "repo", + label: "confirmed" as const, + firedAt: "", + decidedAt: "", + })), + POLICY.heldOutFraction, + POLICY.splitSeed, + ); + const split = splitBenchmarkWorkUnits("bench-2026q3", units, POLICY); + assert.deepEqual( + [...new Set(split.heldOut.map((unit) => unit.repoFullName))].sort(), + direct.heldOut.map((repoCase) => repoCase.targetKey).sort(), + ); +}); + +test("PROPERTY: assignment is deterministic, seed-dependent, and stable as the corpus grows", () => { + const small = unitsAcrossRepos(12); + const grown = [...small, ...unitsAcrossRepos(20).slice(36)]; + const a = splitBenchmarkWorkUnits("b", small, POLICY); + const b = splitBenchmarkWorkUnits("b", small, POLICY); + assert.deepEqual(a, b, "identical inputs must give identical output"); + // A different seed reassigns; without the seed, membership is unguessable. + const other = splitBenchmarkWorkUnits("b", small, { ...POLICY, splitSeed: "different" }); + assert.notDeepEqual( + [...new Set(other.heldOut.map((unit) => unit.repoFullName))].sort(), + [...new Set(a.heldOut.map((unit) => unit.repoFullName))].sort(), + ); + // Growth never reshuffles an already-assigned repo — the property backtest-split.ts exists to guarantee. + const grownSplit = splitBenchmarkWorkUnits("b", grown, POLICY); + const grownHeldOutRepos = new Set(grownSplit.heldOut.map((unit) => unit.repoFullName)); + for (const unit of a.heldOut) assert.ok(grownHeldOutRepos.has(unit.repoFullName), `${unit.repoFullName} moved`); +}); + +test("the split degenerates cleanly at both extremes and on an empty corpus", () => { + const units = unitsAcrossRepos(8); + assert.equal(splitBenchmarkWorkUnits("b", units, { ...POLICY, heldOutFraction: 0 }).heldOut.length, 0); + assert.equal(splitBenchmarkWorkUnits("b", units, { ...POLICY, heldOutFraction: 1 }).visible.length, 0); + const empty = splitBenchmarkWorkUnits("b", [], POLICY); + assert.deepEqual(empty, { visible: [], heldOut: [], heldOutRepoCount: 0, visibleRepoCount: 0 }); + // An out-of-range fraction fails closed through the primitive's own guard, rather than splitting nothing. + assert.throws(() => splitBenchmarkWorkUnits("b", units, { ...POLICY, heldOutFraction: 1.5 }), /invalid_held_out_fraction/); +}); + +test("benchmarkWindowState: pending before opening, open through the inclusive close, retired after", () => { + assert.equal(benchmarkWindowState(WINDOW, "2026-06-30T23:59:59.000Z"), "pending"); + assert.equal(benchmarkWindowState(WINDOW, WINDOW.opensAt), "open"); + assert.equal(benchmarkWindowState(WINDOW, "2026-08-01T00:00:00.000Z"), "open"); + assert.equal(benchmarkWindowState(WINDOW, WINDOW.closesAt), "open"); + assert.equal(benchmarkWindowState(WINDOW, "2026-10-01T00:00:00.000Z"), "retired"); +}); + +test("DECISION 3: the submission cap is enforced per (agent, window) and names its refusal", () => { + const now = "2026-08-01T00:00:00.000Z"; + const first = decideSubmission({ window: WINDOW, now, priorSubmissions: 0 }); + assert.deepEqual(first, { accepted: true, submissionIndex: 1, remaining: DEFAULT_SUBMISSION_CAP - 1 }); + const last = decideSubmission({ window: WINDOW, now, priorSubmissions: DEFAULT_SUBMISSION_CAP - 1 }); + assert.deepEqual(last, { accepted: true, submissionIndex: DEFAULT_SUBMISSION_CAP, remaining: 0 }); + // Brute force stops here — the refusal says WHICH control fired. + assert.deepEqual(decideSubmission({ window: WINDOW, now, priorSubmissions: DEFAULT_SUBMISSION_CAP }), { + accepted: false, + reason: "cap_reached", + remaining: 0, + }); + // An over-count (a replayed or double-recorded submission) still refuses, and never reports negative room. + assert.deepEqual(decideSubmission({ window: WINDOW, now, priorSubmissions: 999 }), { accepted: false, reason: "cap_reached", remaining: 0 }); + // An explicit cap overrides the default in both directions. + assert.equal(decideSubmission({ window: WINDOW, now, priorSubmissions: 2, cap: 3 }).accepted, true); + assert.equal(decideSubmission({ window: WINDOW, now, priorSubmissions: 3, cap: 3 }).accepted, false); +}); + +test("DECISION 4: a retired window refuses submissions; so does one that has not opened", () => { + assert.deepEqual(decideSubmission({ window: WINDOW, now: "2026-10-02T00:00:00.000Z", priorSubmissions: 0 }), { + accepted: false, + reason: "window_retired", + remaining: DEFAULT_SUBMISSION_CAP, + }); + assert.deepEqual(decideSubmission({ window: WINDOW, now: "2026-06-01T00:00:00.000Z", priorSubmissions: 0 }), { + accepted: false, + reason: "window_not_open", + remaining: DEFAULT_SUBMISSION_CAP, + }); + // A retired window refuses even with room left — rotation is not extended by an unused quota. + assert.equal(decideSubmission({ window: WINDOW, now: "2026-12-01T00:00:00.000Z", priorSubmissions: 1 }).accepted, false); +}); + +test("DECISION 2: held-out publishes at close, never on demand while the window is open", () => { + assert.deepEqual(heldOutPublicationDecision(WINDOW, "2026-08-15T00:00:00.000Z"), { + publish: false, + reason: "window_open_between_cadences", + }); + assert.deepEqual(heldOutPublicationDecision(WINDOW, "2026-10-05T00:00:00.000Z"), { publish: true, reason: "evaluation_closed" }); + assert.deepEqual(heldOutPublicationDecision(WINDOW, "2026-06-01T00:00:00.000Z"), { publish: false, reason: "window_not_open" }); +}); + +test("DECISION 2: a configured cadence publishes on the fixed schedule only, not on submission", () => { + const cadenced: BenchmarkWindow = { ...WINDOW, heldOutPublishEveryDays: 30 }; + // Exactly 30 and 60 days after opening: scheduled boundaries. + assert.deepEqual(heldOutPublicationDecision(cadenced, "2026-07-31T00:00:00.000Z"), { publish: true, reason: "scheduled_cadence" }); + assert.deepEqual(heldOutPublicationDecision(cadenced, "2026-08-30T00:00:00.000Z"), { publish: true, reason: "scheduled_cadence" }); + // A day either side of a boundary — and the opening instant itself — publish nothing. + for (const between of ["2026-07-30T00:00:00.000Z", "2026-08-01T00:00:00.000Z", WINDOW.opensAt]) { + assert.equal(heldOutPublicationDecision(cadenced, between).publish, false, between); + } +}); + +test("REGRESSION: a gated payload DROPS the held-out score rather than flagging it, so it cannot be serialized by mistake", () => { + const scores = { visible: { macro: 0.7 }, heldOut: { macro: 0.4 } }; + const open = gateHeldOutPublication(WINDOW, "2026-08-15T00:00:00.000Z", scores); + assert.equal(open.heldOut, null); + // The held-out value is absent from the payload ENTIRELY -- no count, no fraction, nothing an observer + // could difference across submissions to recover per-unit membership. + assert.equal(JSON.stringify(open).includes("0.4"), false); + assert.deepEqual(open.visible, { macro: 0.7 }); + // After close there is nothing left to leak into, so the real score is published. + const closed = gateHeldOutPublication(WINDOW, "2026-10-05T00:00:00.000Z", scores); + assert.deepEqual(closed.heldOut, { macro: 0.4 }); + assert.deepEqual(closed.heldOutPublication, { publish: true, reason: "evaluation_closed" }); +}); + +test("REGRESSION: nothing in a published open-window payload reveals held-out MEMBERSHIP", () => { + // The adversary's question is "is repo X held out?". The published shape must not answer it, directly or + // by differencing: it carries the visible slice and a policy decision, and no held-out identifiers. + const units = unitsAcrossRepos(12); + const split = splitBenchmarkWorkUnits("bench-2026q3", units, POLICY); + const published = gateHeldOutPublication(WINDOW, "2026-08-15T00:00:00.000Z", { + visible: { units: split.visible.map((unit) => unit.workUnitId) }, + heldOut: { units: split.heldOut.map((unit) => unit.workUnitId) }, + }); + const serialized = JSON.stringify(published); + for (const unit of split.heldOut) { + assert.equal(serialized.includes(unit.workUnitId), false, `${unit.workUnitId} leaked`); + assert.equal(serialized.includes(unit.repoFullName), false, `${unit.repoFullName} leaked`); + } + assert.ok(split.heldOut.length > 0, "fixture must actually hold something out"); +});