|
| 1 | +// Benchmark leaderboard artifacts as spec-conformant EvalScoreRecords (#9265, harness #9216, epic #8534). |
| 2 | +// |
| 3 | +// #9216 requirement 5 is explicit: leaderboard artifacts are EXACTLY the eval-interface's consumable score |
| 4 | +// records — no parallel format. So this module emits `EvalScoreRecord`s with `workUnit.kind === |
| 5 | +// "benchmark_run"` through the SAME `finalizeRecord`/digest path the `outcome_confirmed_precision` records |
| 6 | +// already use, and the leaderboard view is derived from those records and nothing else. |
| 7 | +// |
| 8 | +// That last point is the structural one: there is no second query path from the scorer to the board. A |
| 9 | +// number on the leaderboard is, by construction, a field of a record a third party can fetch, re-derive and |
| 10 | +// digest-verify — so the published board and the consumable feed cannot disagree, because they are the same |
| 11 | +// data read twice. |
| 12 | +// |
| 13 | +// Two policies are enforced HERE rather than left to a caller's discipline: |
| 14 | +// |
| 15 | +// HELD-OUT CADENCE (#9263 decision 2). The emitter is where publication cadence is enforced, because it |
| 16 | +// is the last point before a number becomes public. A held-out record is simply not emitted while the |
| 17 | +// window is open — not emitted-and-filtered, not emitted-with-a-flag. A leaderboard therefore cannot leak |
| 18 | +// held-out membership by forgetting to check something. |
| 19 | +// |
| 20 | +// TRUST TIER FROM THE ACTUAL EXECUTION (#9265 requirement 3). `attested` requires a real attestation |
| 21 | +// envelope to be handed in; absent one the tier is `reproducible`, never optimistically upgraded. The |
| 22 | +// record type's own discriminated union makes "attested without an envelope" unrepresentable, so this is |
| 23 | +// a compile-time guarantee rather than a runtime check that could be skipped. |
| 24 | +import { |
| 25 | + EVAL_SCORE_RECORD_SCHEMA_VERSION, |
| 26 | + canonicalJson, |
| 27 | + contentDigest, |
| 28 | + type EvalScoreRecord, |
| 29 | + type EvalScoreRecordTrust, |
| 30 | +} from "./eval-score-records"; |
| 31 | +import type { BenchmarkScoreReport, BenchmarkWindow } from "@loopover/engine"; |
| 32 | +import { heldOutPublicationDecision } from "@loopover/engine"; |
| 33 | + |
| 34 | +/** Semantics version for the `benchmark_run` work-unit kind (#9215 §5) — independent of any package |
| 35 | + * version. Bump ONLY when what the numbers MEAN changes (the aggregation choice, the coverage rule, the |
| 36 | + * ground-truth settlement rules), never for an unrelated refactor. */ |
| 37 | +export const BENCHMARK_RUN_SCORING_RULE_VERSION = "benchmark-run-v1"; |
| 38 | + |
| 39 | +/** Which slice a record scores. Carried in the leaderboard view (not the record) because the record's own |
| 40 | + * commitments already pin the split via `splitSeed`/`heldOutFraction`, and adding a top-level field would |
| 41 | + * violate #9265 requirement 1's "no extra top-level fields". */ |
| 42 | +export type BenchmarkSlice = "visible" | "held_out"; |
| 43 | + |
| 44 | +export type BenchmarkRecordInput = { |
| 45 | + subjectId: string; |
| 46 | + report: BenchmarkScoreReport; |
| 47 | + window: BenchmarkWindow; |
| 48 | + /** #9259's content-addressed snapshot checksum — the commitment that lets a third party re-derive. */ |
| 49 | + snapshotChecksum: string; |
| 50 | + splitSeed: string; |
| 51 | + heldOutFraction: number; |
| 52 | + /** Real envelope evidence, or absent. Absent ⇒ `reproducible`; never asserted optimistically. */ |
| 53 | + attestation?: { envelopeDigest: string; measurement: string; runId: string } | undefined; |
| 54 | + issuedAt: string; |
| 55 | +}; |
| 56 | + |
| 57 | +function trustFrom(attestation: BenchmarkRecordInput["attestation"]): EvalScoreRecordTrust { |
| 58 | + return attestation ? { tier: "attested", attestation } : { tier: "reproducible" }; |
| 59 | +} |
| 60 | + |
| 61 | +/** |
| 62 | + * Build ONE spec-conformant `EvalScoreRecord` for a benchmark run. |
| 63 | + * |
| 64 | + * The score block maps the multi-class report onto #9215's fixed fields without inventing any: `precision` |
| 65 | + * and `recall` are the MACRO numbers (#9262's recorded headline decision — the number that answers the |
| 66 | + * benchmark's actual question), `decided`/`abstained`/`coverage` come straight from the report's coverage |
| 67 | + * block, and `confirmed` is the count of decided units the agent got right. Nothing benchmark-specific is |
| 68 | + * bolted onto the record; a consumer that already reads `outcome_confirmed_precision` records reads these |
| 69 | + * with the same code. |
| 70 | + */ |
| 71 | +export async function buildBenchmarkEvalScoreRecord(input: BenchmarkRecordInput): Promise<EvalScoreRecord> { |
| 72 | + const { report } = input; |
| 73 | + // Micro (== accuracy under single-label scoring) times the decided count recovers the number correct, |
| 74 | + // which is exactly what `confirmed` means for this work-unit kind: decided units that matched reality. |
| 75 | + const confirmed = report.micro.accuracy === null ? 0 : Math.round(report.micro.accuracy * report.coverage.decided); |
| 76 | + const withoutDigest: Omit<EvalScoreRecord, "recordDigest"> = { |
| 77 | + schemaVersion: EVAL_SCORE_RECORD_SCHEMA_VERSION, |
| 78 | + subject: { kind: "agent", id: input.subjectId }, |
| 79 | + workUnit: { kind: "benchmark_run", benchmarkId: input.window.benchmarkId, snapshotRef: report.snapshotRef }, |
| 80 | + score: { |
| 81 | + decided: report.coverage.decided, |
| 82 | + confirmed, |
| 83 | + precision: report.macro.precision, |
| 84 | + recall: report.macro.recall, |
| 85 | + coverage: report.coverage.coverage, |
| 86 | + abstained: report.coverage.abstained, |
| 87 | + }, |
| 88 | + commitments: { |
| 89 | + corpusChecksum: input.snapshotChecksum, |
| 90 | + scoringRuleVersion: BENCHMARK_RUN_SCORING_RULE_VERSION, |
| 91 | + windowStart: input.window.opensAt, |
| 92 | + windowEnd: input.window.closesAt, |
| 93 | + splitSeed: input.splitSeed, |
| 94 | + heldOutFraction: input.heldOutFraction, |
| 95 | + }, |
| 96 | + trust: trustFrom(input.attestation), |
| 97 | + issuedAt: input.issuedAt, |
| 98 | + }; |
| 99 | + return { ...withoutDigest, recordDigest: await contentDigest(withoutDigest) }; |
| 100 | +} |
| 101 | + |
| 102 | +/** |
| 103 | + * Emit the publishable records for one submission, enforcing #9263's held-out cadence at this boundary. |
| 104 | + * |
| 105 | + * The visible-slice record always emits. The held-out record emits ONLY when the cadence allows it, and |
| 106 | + * when it does not, it is never built at all — so there is no in-memory held-out record for a downstream |
| 107 | + * bug to serialize. `heldOutPublication` is returned so a caller can explain the absence to a submitter |
| 108 | + * ("published at evaluation close") rather than leaving a silent gap that reads like a missing score. |
| 109 | + */ |
| 110 | +export async function buildBenchmarkEvalScoreRecords(input: { |
| 111 | + visible: BenchmarkRecordInput; |
| 112 | + heldOut: BenchmarkRecordInput; |
| 113 | + now: string; |
| 114 | +}): Promise<{ |
| 115 | + records: Array<{ slice: BenchmarkSlice; record: EvalScoreRecord }>; |
| 116 | + heldOutPublication: ReturnType<typeof heldOutPublicationDecision>; |
| 117 | +}> { |
| 118 | + const decision = heldOutPublicationDecision(input.visible.window, input.now); |
| 119 | + const records: Array<{ slice: BenchmarkSlice; record: EvalScoreRecord }> = [ |
| 120 | + { slice: "visible", record: await buildBenchmarkEvalScoreRecord(input.visible) }, |
| 121 | + ]; |
| 122 | + if (decision.publish) { |
| 123 | + records.push({ slice: "held_out", record: await buildBenchmarkEvalScoreRecord(input.heldOut) }); |
| 124 | + } |
| 125 | + return { records, heldOutPublication: decision }; |
| 126 | +} |
| 127 | + |
| 128 | +export type LeaderboardRow = { |
| 129 | + subjectId: string; |
| 130 | + benchmarkId: string; |
| 131 | + snapshotRef: string; |
| 132 | + /** The macro headline the board ranks on (#9262). `null` ranks LAST — an unmeasurable score is not a |
| 133 | + * zero score, but it is also not a rankable one, and silently sorting it as 0 would invent a claim. */ |
| 134 | + precision: number | null; |
| 135 | + recall: number | null; |
| 136 | + coverage: number | null; |
| 137 | + decided: number; |
| 138 | + abstained: number; |
| 139 | + trustTier: EvalScoreRecord["trust"]["tier"]; |
| 140 | + issuedAt: string; |
| 141 | + recordDigest: string; |
| 142 | +}; |
| 143 | + |
| 144 | +/** |
| 145 | + * Derive the leaderboard from records ONLY — there is deliberately no separate query path, so the board and |
| 146 | + * the consumable feed cannot disagree. |
| 147 | + * |
| 148 | + * Ranking: macro precision descending, `null` last; ties broken by coverage descending (an agent that |
| 149 | + * answered more of the benchmark at the same precision is doing more work), then by `issuedAt` ascending so |
| 150 | + * the earlier claim to a position keeps it. Immutability is respected rather than worked around: when a |
| 151 | + * subject has several records for the same benchmark, the LATEST `issuedAt` wins, because a corrected score |
| 152 | + * is a new record, never a mutation. |
| 153 | + */ |
| 154 | +export function deriveBenchmarkLeaderboard(records: readonly EvalScoreRecord[], benchmarkId: string): LeaderboardRow[] { |
| 155 | + const latestBySubject = new Map<string, EvalScoreRecord>(); |
| 156 | + for (const record of records) { |
| 157 | + if (record.workUnit.kind !== "benchmark_run" || record.workUnit.benchmarkId !== benchmarkId) continue; |
| 158 | + const current = latestBySubject.get(record.subject.id); |
| 159 | + if (!current || Date.parse(record.issuedAt) > Date.parse(current.issuedAt)) latestBySubject.set(record.subject.id, record); |
| 160 | + } |
| 161 | + const rows: LeaderboardRow[] = []; |
| 162 | + for (const record of latestBySubject.values()) { |
| 163 | + /* v8 ignore next -- filtered to benchmark_run above; the guard only narrows the union for the compiler. */ |
| 164 | + if (record.workUnit.kind !== "benchmark_run") continue; |
| 165 | + rows.push({ |
| 166 | + subjectId: record.subject.id, |
| 167 | + benchmarkId: record.workUnit.benchmarkId, |
| 168 | + snapshotRef: record.workUnit.snapshotRef, |
| 169 | + precision: record.score.precision, |
| 170 | + recall: record.score.recall, |
| 171 | + coverage: record.score.coverage, |
| 172 | + decided: record.score.decided, |
| 173 | + abstained: record.score.abstained, |
| 174 | + trustTier: record.trust.tier, |
| 175 | + issuedAt: record.issuedAt, |
| 176 | + recordDigest: record.recordDigest, |
| 177 | + }); |
| 178 | + } |
| 179 | + return rows.sort((a, b) => { |
| 180 | + if (a.precision !== b.precision) { |
| 181 | + if (a.precision === null) return 1; |
| 182 | + if (b.precision === null) return -1; |
| 183 | + return b.precision - a.precision; |
| 184 | + } |
| 185 | + const aCoverage = a.coverage ?? -1; |
| 186 | + const bCoverage = b.coverage ?? -1; |
| 187 | + if (aCoverage !== bCoverage) return bCoverage - aCoverage; |
| 188 | + return Date.parse(a.issuedAt) - Date.parse(b.issuedAt); |
| 189 | + }); |
| 190 | +} |
| 191 | + |
| 192 | +export { canonicalJson }; |
0 commit comments