From 57a7f4c3613810918161fd93c4e2c75a9ee7bfd0 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 09:23:09 +0000 Subject: [PATCH 01/18] feat: add eval work item identity helpers --- evals/lib/types.ts | 30 +++++++++++++++ test/unit/evals/workItem.test.ts | 66 ++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 test/unit/evals/workItem.test.ts diff --git a/evals/lib/types.ts b/evals/lib/types.ts index 73154cad..14ec1328 100644 --- a/evals/lib/types.ts +++ b/evals/lib/types.ts @@ -1,3 +1,4 @@ +import { invariant } from '../../src/util/assert.js'; import type { EventRecord } from '../../src/protocol/schemas.js'; import type { ArtifactKind } from '../../src/tools/review-bundle.js'; import type { @@ -14,6 +15,35 @@ export type SkillCondition = 'none' | 'self-load' | 'preloaded' | 'stale'; /** Eval lane identifier. */ export type EvalLane = 'prompt' | 'execution' | 'dogfood'; +/** Stable identity for one eval work item across lanes and trials. */ +export interface EvalWorkItemIdentity { + lane: EvalLane; + caseId: string; + condition: SkillCondition; + trial: number; +} + +/** Build a stable string key for a work item identity. */ +export function buildWorkItemKey(identity: EvalWorkItemIdentity): string { + invariant(identity.caseId.length > 0, 'identity.caseId must be a non-empty string'); + invariant( + Number.isInteger(identity.trial) && identity.trial > 0, + 'identity.trial must be a positive integer', + ); + + return `${identity.lane}:${identity.caseId}:${identity.condition}:${identity.trial}`; +} + +/** Assert that a list of work item identities contains no duplicates. */ +export function assertUniqueWorkItems(items: EvalWorkItemIdentity[]): void { + const seen = new Set(); + for (const item of items) { + const key = buildWorkItemKey(item); + invariant(!seen.has(key), `Duplicate work item identity: ${key}`); + seen.add(key); + } +} + /** Deterministic verifier kind. */ export type VerifierKind = | 'snapshot' diff --git a/test/unit/evals/workItem.test.ts b/test/unit/evals/workItem.test.ts new file mode 100644 index 00000000..3c23909e --- /dev/null +++ b/test/unit/evals/workItem.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from 'vitest'; + +import { + assertUniqueWorkItems, + buildWorkItemKey, +} from '../../../evals/lib/types.js'; +import type { EvalWorkItemIdentity } from '../../../evals/lib/types.js'; + +function createWorkItemIdentity( + overrides: Partial = {}, +): EvalWorkItemIdentity { + return { + lane: 'prompt', + caseId: 'case-1', + condition: 'none', + trial: 1, + ...overrides, + }; +} + +describe('buildWorkItemKey', () => { + it('produces the expected stable string format', () => { + const identity = createWorkItemIdentity({ + lane: 'dogfood', + caseId: 'bug-repro-7', + condition: 'preloaded', + trial: 3, + }); + + expect(buildWorkItemKey(identity)).toBe('dogfood:bug-repro-7:preloaded:3'); + }); +}); + +describe('assertUniqueWorkItems', () => { + it('passes for distinct items', () => { + const items = [ + createWorkItemIdentity(), + createWorkItemIdentity({ condition: 'self-load' }), + createWorkItemIdentity({ lane: 'execution', caseId: 'case-2' }), + ]; + + expect(() => assertUniqueWorkItems(items)).not.toThrow(); + }); + + it('treats different trials for the same case and condition as distinct', () => { + const items = [ + createWorkItemIdentity(), + createWorkItemIdentity({ trial: 2 }), + ]; + + expect(() => assertUniqueWorkItems(items)).not.toThrow(); + }); + + it('throws for duplicate items', () => { + const duplicate = createWorkItemIdentity({ + lane: 'execution', + caseId: 'case-7', + condition: 'stale', + trial: 1, + }); + + expect(() => assertUniqueWorkItems([duplicate, { ...duplicate }])).toThrow( + 'Duplicate work item identity: execution:case-7:stale:1', + ); + }); +}); From e5851c61233855d17acac0c9ed39c116a4d25244 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 09:24:45 +0000 Subject: [PATCH 02/18] fix: clean up eval runner temp directories --- evals/dogfood/runner.ts | 447 ++++++++++++++++++++------------------ evals/execution/runner.ts | 322 ++++++++++++++------------- 2 files changed, 398 insertions(+), 371 deletions(-) diff --git a/evals/dogfood/runner.ts b/evals/dogfood/runner.ts index 3bf4d0f5..b87df0bf 100644 --- a/evals/dogfood/runner.ts +++ b/evals/dogfood/runner.ts @@ -1,4 +1,4 @@ -import { mkdir, readFile, stat } from 'node:fs/promises'; +import { mkdir, readFile, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; @@ -593,224 +593,237 @@ export async function runDogfoodLane( await mkdir(outputDir, { recursive: true }); await mkdir(homeDir, { recursive: true }); - const systemPromptContext = await buildSystemPromptContext(condition); - const requestCase = DogfoodEvalCaseSchema.parse({ - ...evalCase, - prompt: buildPrompt( - { ...evalCase, conditions: [condition] }, - requestedBundlePath, - systemPromptContext, - ), - bundlePath: requestedBundlePath, - conditions: [condition], - }) as DogfoodEvalCase; - try { - const agentResult = await provider.invokeAgentMode({ - runId: metadata.runId, - providerId: provider.id, - condition, - trial: 1, - ...(requestedModelId === undefined - ? {} - : { modelId: requestedModelId }), - cwd: repoRoot, - homeDir, - outputDir, - env: { - AGENT_TTY_HOME: homeDir, - EVAL_OUTPUT_DIR: outputDir, - EVAL_REQUESTED_BUNDLE_DIR: requestedBundlePath, - EVAL_FIXTURE: evalCase.fixture ?? '', - ...systemPromptContext.env, - }, - evalCase: requestCase, - }); - - const transcript = await resolveTranscriptText(agentResult); - const bundlePath = await resolveBundlePath( - agentResult.bundlePath, - requestedBundlePath, - ); - const reportText = await resolveReportText( - bundlePath, - agentResult.normalized.finalText, - ); - const dogfoodScore = await scoreDogfoodRun( - bundlePath ?? requestedBundlePath, - reportText, - transcript, - evalCase, - ); - const reportRequirementScore = scoreReportRequirements( - reportText ?? '', - evalCase.reportRequirements, - ); - const reportSections = evalCase.reportRequirements - .map((requirement) => requirement.section) - .filter( - (section): section is string => - typeof section === 'string' && section.trim().length > 0, - ); - const baseReportCompleteness = scoreReportCompleteness( - reportText ?? '', - reportSections, - ); - const reportCompleteness = { - ...baseReportCompleteness, - sectionsExpected: reportRequirementScore.details.length, - sectionsFound: reportRequirementScore.details.filter( - (detail) => detail.found, - ).length, - score: clampUnitInterval( - (baseReportCompleteness.score + reportRequirementScore.score) / 2, + const systemPromptContext = await buildSystemPromptContext(condition); + const requestCase = DogfoodEvalCaseSchema.parse({ + ...evalCase, + prompt: buildPrompt( + { ...evalCase, conditions: [condition] }, + requestedBundlePath, + systemPromptContext, ), - details: reportRequirementScore.details.map((detail) => ({ - section: detail.section, - found: detail.found, - required: - evalCase.reportRequirements.find( - (requirement) => - (requirement.section ?? requirement.id) === detail.section, - )?.required ?? true, - })), - missingSections: reportRequirementScore.details - .filter((detail) => !detail.found) - .map((detail) => detail.section), - }; - const bundleCompleteness = await scoreBundleCompleteness( - bundlePath ?? requestedBundlePath, - evalCase.validationProfile, - ); - const evidenceQuality = await scoreEvidenceQuality( - bundlePath ?? requestedBundlePath, - ); - const workflowChecks = - requestCase.workflowChecks.length === 0 - ? [] - : checkWorkflow(transcript, requestCase.workflowChecks); - const scannableTranscript = buildScannableTranscript( - agentResult.normalized, - ); - const antiPatternFindings = detectAntiPatterns( - scannableTranscript, - requestCase.antiPatterns, - ); - const artifactManifestPath = - await resolveArtifactManifestPath(bundlePath); - const blockingAntiPattern = antiPatternFindings.some( - (finding) => finding.severity === 'error', - ); - const missingRequiredReportSection = - reportRequirementScore.details.some((detail) => !detail.found); - const missingRequiredWorkflow = workflowChecks.some( - (check) => !check.passed, - ); - const ok = - agentResult.ok && - !blockingAntiPattern && - !missingRequiredReportSection && - !missingRequiredWorkflow && - dogfoodScore.overallScore >= 0.6; - const completedAt = agentResult.completedAt; - const result: EvalResult = EvalResultSchema.parse({ - runId: metadata.runId, - providerId: provider.id, - ...(agentResult.runtime.version === undefined - ? {} - : { providerVersion: agentResult.runtime.version }), - ...(requestedModelId === undefined - ? {} - : { modelId: requestedModelId }), - lane: 'dogfood', - caseId: evalCase.id, - category: evalCase.category, - condition, - expectedSkill: evalCase.expectedSkill, - trial: 1, - ok, - score: { - total: dogfoodScore.overallScore, - maxPossible: 1, - items: buildDogfoodBreakdownItems(dogfoodScore), - }, - workflowChecks, - antiPatternFindings, - bundleCompleteness, - reportCompleteness, - evidenceQuality, - ...(agentResult.transcriptPath === undefined - ? {} - : { transcriptPath: agentResult.transcriptPath }), - ...(bundlePath === undefined ? {} : { bundlePath }), - ...(artifactManifestPath === undefined - ? {} - : { artifactManifestPath }), - ...(agentResult.eventLogPath === undefined - ? {} - : { eventLogPath: agentResult.eventLogPath }), - normalizedOutput: agentResult.normalized, - ...(agentResult.errorClass === undefined - ? {} - : { errorClass: agentResult.errorClass }), - ...(agentResult.errorMessage === undefined - ? {} - : { errorMessage: agentResult.errorMessage }), - startedAt: agentResult.startedAt, - completedAt, - durationMs: agentResult.durationMs, - }) as EvalResult; - - results.push(result); - } catch (error) { - const completedAt = new Date().toISOString(); - const errorMessage = - error instanceof Error ? error.message : String(error); - const errorClass = - error instanceof Error && error.name.length > 0 - ? error.name - : 'Error'; - const result: EvalResult = EvalResultSchema.parse({ - runId: metadata.runId, - providerId: provider.id, - ...(detectedRuntime.version === undefined - ? {} - : { providerVersion: detectedRuntime.version }), - ...(requestedModelId === undefined - ? {} - : { modelId: requestedModelId }), - lane: 'dogfood', - caseId: evalCase.id, - category: evalCase.category, - condition, - expectedSkill: evalCase.expectedSkill, - trial: 1, - ok: false, - score: { - total: 0, - maxPossible: 1, - items: buildDogfoodBreakdownItems({ - bundleCompleteness: 0, - reportCompleteness: 0, - evidenceQuality: 0, - taxonomyUsage: 0, - reproducibility: 0, - }), - }, - workflowChecks: [], - antiPatternFindings: [], - normalizedOutput: EMPTY_NORMALIZED_OUTPUT, - errorClass, - errorMessage, - startedAt, - completedAt, - durationMs: Math.max( - 0, - Date.parse(completedAt) - Date.parse(startedAt), - ), - }) as EvalResult; - - results.push(result); + bundlePath: requestedBundlePath, + conditions: [condition], + }) as DogfoodEvalCase; + + try { + const agentResult = await provider.invokeAgentMode({ + runId: metadata.runId, + providerId: provider.id, + condition, + trial: 1, + ...(requestedModelId === undefined + ? {} + : { modelId: requestedModelId }), + cwd: repoRoot, + homeDir, + outputDir, + env: { + AGENT_TTY_HOME: homeDir, + EVAL_OUTPUT_DIR: outputDir, + EVAL_REQUESTED_BUNDLE_DIR: requestedBundlePath, + EVAL_FIXTURE: evalCase.fixture ?? '', + ...systemPromptContext.env, + }, + evalCase: requestCase, + }); + + const transcript = await resolveTranscriptText(agentResult); + const bundlePath = await resolveBundlePath( + agentResult.bundlePath, + requestedBundlePath, + ); + const reportText = await resolveReportText( + bundlePath, + agentResult.normalized.finalText, + ); + const dogfoodScore = await scoreDogfoodRun( + bundlePath ?? requestedBundlePath, + reportText, + transcript, + evalCase, + ); + const reportRequirementScore = scoreReportRequirements( + reportText ?? '', + evalCase.reportRequirements, + ); + const reportSections = evalCase.reportRequirements + .map((requirement) => requirement.section) + .filter( + (section): section is string => + typeof section === 'string' && section.trim().length > 0, + ); + const baseReportCompleteness = scoreReportCompleteness( + reportText ?? '', + reportSections, + ); + const reportCompleteness = { + ...baseReportCompleteness, + sectionsExpected: reportRequirementScore.details.length, + sectionsFound: reportRequirementScore.details.filter( + (detail) => detail.found, + ).length, + score: clampUnitInterval( + (baseReportCompleteness.score + reportRequirementScore.score) / 2, + ), + details: reportRequirementScore.details.map((detail) => ({ + section: detail.section, + found: detail.found, + required: + evalCase.reportRequirements.find( + (requirement) => + (requirement.section ?? requirement.id) === detail.section, + )?.required ?? true, + })), + missingSections: reportRequirementScore.details + .filter((detail) => !detail.found) + .map((detail) => detail.section), + }; + const bundleCompleteness = await scoreBundleCompleteness( + bundlePath ?? requestedBundlePath, + evalCase.validationProfile, + ); + const evidenceQuality = await scoreEvidenceQuality( + bundlePath ?? requestedBundlePath, + ); + const workflowChecks = + requestCase.workflowChecks.length === 0 + ? [] + : checkWorkflow(transcript, requestCase.workflowChecks); + const scannableTranscript = buildScannableTranscript( + agentResult.normalized, + ); + const antiPatternFindings = detectAntiPatterns( + scannableTranscript, + requestCase.antiPatterns, + ); + const artifactManifestPath = + await resolveArtifactManifestPath(bundlePath); + const blockingAntiPattern = antiPatternFindings.some( + (finding) => finding.severity === 'error', + ); + const missingRequiredReportSection = + reportRequirementScore.details.some((detail) => !detail.found); + const missingRequiredWorkflow = workflowChecks.some( + (check) => !check.passed, + ); + const ok = + agentResult.ok && + !blockingAntiPattern && + !missingRequiredReportSection && + !missingRequiredWorkflow && + dogfoodScore.overallScore >= 0.6; + const completedAt = agentResult.completedAt; + const result: EvalResult = EvalResultSchema.parse({ + runId: metadata.runId, + providerId: provider.id, + ...(agentResult.runtime.version === undefined + ? {} + : { providerVersion: agentResult.runtime.version }), + ...(requestedModelId === undefined + ? {} + : { modelId: requestedModelId }), + lane: 'dogfood', + caseId: evalCase.id, + category: evalCase.category, + condition, + expectedSkill: evalCase.expectedSkill, + trial: 1, + ok, + score: { + total: dogfoodScore.overallScore, + maxPossible: 1, + items: buildDogfoodBreakdownItems(dogfoodScore), + }, + workflowChecks, + antiPatternFindings, + bundleCompleteness, + reportCompleteness, + evidenceQuality, + ...(agentResult.transcriptPath === undefined + ? {} + : { transcriptPath: agentResult.transcriptPath }), + ...(bundlePath === undefined ? {} : { bundlePath }), + ...(artifactManifestPath === undefined + ? {} + : { artifactManifestPath }), + ...(agentResult.eventLogPath === undefined + ? {} + : { eventLogPath: agentResult.eventLogPath }), + normalizedOutput: agentResult.normalized, + ...(agentResult.errorClass === undefined + ? {} + : { errorClass: agentResult.errorClass }), + ...(agentResult.errorMessage === undefined + ? {} + : { errorMessage: agentResult.errorMessage }), + startedAt: agentResult.startedAt, + completedAt, + durationMs: agentResult.durationMs, + }) as EvalResult; + + results.push(result); + } catch (error) { + const completedAt = new Date().toISOString(); + const errorMessage = + error instanceof Error ? error.message : String(error); + const errorClass = + error instanceof Error && error.name.length > 0 + ? error.name + : 'Error'; + const result: EvalResult = EvalResultSchema.parse({ + runId: metadata.runId, + providerId: provider.id, + ...(detectedRuntime.version === undefined + ? {} + : { providerVersion: detectedRuntime.version }), + ...(requestedModelId === undefined + ? {} + : { modelId: requestedModelId }), + lane: 'dogfood', + caseId: evalCase.id, + category: evalCase.category, + condition, + expectedSkill: evalCase.expectedSkill, + trial: 1, + ok: false, + score: { + total: 0, + maxPossible: 1, + items: buildDogfoodBreakdownItems({ + bundleCompleteness: 0, + reportCompleteness: 0, + evidenceQuality: 0, + taxonomyUsage: 0, + reproducibility: 0, + }), + }, + workflowChecks: [], + antiPatternFindings: [], + normalizedOutput: EMPTY_NORMALIZED_OUTPUT, + errorClass, + errorMessage, + startedAt, + completedAt, + durationMs: Math.max( + 0, + Date.parse(completedAt) - Date.parse(startedAt), + ), + }) as EvalResult; + + results.push(result); + } + } finally { + try { + await rm(homeDir, { recursive: true, force: true }); + } catch { + // Best-effort cleanup; preserve the eval result when temp-home cleanup fails. + } + try { + await rm(outputDir, { recursive: true, force: true }); + } catch { + // Best-effort cleanup; preserve the eval result when temp-dir cleanup fails. + } } } } diff --git a/evals/execution/runner.ts b/evals/execution/runner.ts index b144ada9..e5c7acd1 100644 --- a/evals/execution/runner.ts +++ b/evals/execution/runner.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readdir, readFile, stat, writeFile } from 'node:fs/promises'; +import { mkdtemp, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; @@ -8,7 +8,7 @@ import { countAgentTtyCalls, detectAntiPatterns, } from '../lib/antiPatterns.js'; -import { createIsolatedEvalHome } from '../lib/cliHarness.js'; +import { cleanupEvalHome, createIsolatedEvalHome } from '../lib/cliHarness.js'; import { EvalResultSchema } from '../lib/schemas.js'; import { checkWorkflow } from '../lib/scoring.js'; import type { @@ -805,167 +805,181 @@ async function runSingleExecutionCase( ): Promise { const homeDir = await createIsolatedEvalHome(); const outputDir = await mkdtemp(join(tmpdir(), RUNNER_OUTPUT_PREFIX)); - const request = await createEvalRequest( - provider, - metadata, - evalCase, - condition, - homeDir, - outputDir, - runtime, - ); - const invocationStartedAt = new Date().toISOString(); - const invocationStartedMs = Date.now(); - - if (runtime !== undefined && !runtime.available) { - const transcript = `Provider ${provider.id} is unavailable.`; - const normalizedOutput = createFallbackNormalizedOutput(transcript); - const scannableTranscript = buildScannableTranscript(normalizedOutput); - const persisted = await persistRunnerArtifacts( - outputDir, - transcript, - '', - '', - ); - return buildResult( - metadata, - provider, - evalCase, - condition, - runtime, - normalizedOutput, - false, - invocationStartedAt, - invocationStartedAt, - 0, - checkWorkflow(transcript, evalCase.workflowChecks), - detectAntiPatterns(scannableTranscript, evalCase.antiPatterns), - buildVerifierFailures( - evalCase, - 'Provider is unavailable; verifier did not run.', - ), - buildArtifactFailures( - evalCase, - 'Provider is unavailable; artifact verification did not run.', - ), - persisted.transcriptPath, - persisted.stdoutPath, - persisted.stderrPath, - undefined, - undefined, - 'provider-unavailable', - 'Provider detect() reported unavailable.', - ); - } try { - const providerResult = await provider.invokeAgentMode(request); - const transcript = await buildTranscript(providerResult); - const persisted = await persistRunnerArtifacts( - outputDir, - transcript, - providerResult.rawStdout, - providerResult.rawStderr, - ); - const artifacts = await collectArtifacts([ - outputDir, - ...(providerResult.bundlePath === undefined - ? [] - : [providerResult.bundlePath]), - ...(providerResult.transcriptPath === undefined - ? [] - : [providerResult.transcriptPath]), - ...(providerResult.eventLogPath === undefined - ? [] - : [providerResult.eventLogPath]), - ]); - const verifierContext = { - home: homeDir, - sessionId: providerResult.sessionId ?? '', - transcript, - artifacts, - }; - const verifierResults = await Promise.all( - evalCase.verifiers.map(async (spec) => ({ - spec, - result: await verify(spec, verifierContext), - })), - ); - const artifactResults = await evaluateArtifactRequirements( - evalCase.artifactRequirements, - artifacts, - ); - const scannableTranscript = buildScannableTranscript( - providerResult.normalized, - ); - return buildResult( - metadata, + const request = await createEvalRequest( provider, - evalCase, - condition, - providerResult.runtime, - providerResult.normalized, - providerResult.ok, - providerResult.startedAt, - providerResult.completedAt, - providerResult.durationMs, - checkWorkflow(transcript, evalCase.workflowChecks), - detectAntiPatterns(scannableTranscript, evalCase.antiPatterns), - verifierResults, - artifactResults, - providerResult.transcriptPath ?? persisted.transcriptPath, - persisted.stdoutPath, - persisted.stderrPath, - providerResult.bundlePath, - providerResult.eventLogPath, - providerResult.errorClass, - providerResult.errorMessage, - ); - } catch (error) { - const completedAt = new Date().toISOString(); - const durationMs = Math.max(0, Date.now() - invocationStartedMs); - const errorMessage = error instanceof Error ? error.message : String(error); - const transcript = - error instanceof Error && error.stack !== undefined - ? `${error.name}: ${errorMessage}\n${error.stack}` - : errorMessage; - const normalizedOutput = createFallbackNormalizedOutput(transcript); - const scannableTranscript = buildScannableTranscript(normalizedOutput); - const persisted = await persistRunnerArtifacts( - outputDir, - transcript, - '', - errorMessage, - ); - return buildResult( metadata, - provider, evalCase, condition, + homeDir, + outputDir, runtime, - normalizedOutput, - false, - invocationStartedAt, - completedAt, - durationMs, - checkWorkflow(transcript, evalCase.workflowChecks), - detectAntiPatterns(scannableTranscript, evalCase.antiPatterns), - buildVerifierFailures( + ); + const invocationStartedAt = new Date().toISOString(); + const invocationStartedMs = Date.now(); + + if (runtime !== undefined && !runtime.available) { + const transcript = `Provider ${provider.id} is unavailable.`; + const normalizedOutput = createFallbackNormalizedOutput(transcript); + const scannableTranscript = buildScannableTranscript(normalizedOutput); + const persisted = await persistRunnerArtifacts( + outputDir, + transcript, + '', + '', + ); + return buildResult( + metadata, + provider, evalCase, - 'Provider invocation failed before verifier execution.', - ), - buildArtifactFailures( + condition, + runtime, + normalizedOutput, + false, + invocationStartedAt, + invocationStartedAt, + 0, + checkWorkflow(transcript, evalCase.workflowChecks), + detectAntiPatterns(scannableTranscript, evalCase.antiPatterns), + buildVerifierFailures( + evalCase, + 'Provider is unavailable; verifier did not run.', + ), + buildArtifactFailures( + evalCase, + 'Provider is unavailable; artifact verification did not run.', + ), + persisted.transcriptPath, + persisted.stdoutPath, + persisted.stderrPath, + undefined, + undefined, + 'provider-unavailable', + 'Provider detect() reported unavailable.', + ); + } + + try { + const providerResult = await provider.invokeAgentMode(request); + const transcript = await buildTranscript(providerResult); + const persisted = await persistRunnerArtifacts( + outputDir, + transcript, + providerResult.rawStdout, + providerResult.rawStderr, + ); + const artifacts = await collectArtifacts([ + outputDir, + ...(providerResult.bundlePath === undefined + ? [] + : [providerResult.bundlePath]), + ...(providerResult.transcriptPath === undefined + ? [] + : [providerResult.transcriptPath]), + ...(providerResult.eventLogPath === undefined + ? [] + : [providerResult.eventLogPath]), + ]); + const verifierContext = { + home: homeDir, + sessionId: providerResult.sessionId ?? '', + transcript, + artifacts, + }; + const verifierResults = await Promise.all( + evalCase.verifiers.map(async (spec) => ({ + spec, + result: await verify(spec, verifierContext), + })), + ); + const artifactResults = await evaluateArtifactRequirements( + evalCase.artifactRequirements, + artifacts, + ); + const scannableTranscript = buildScannableTranscript( + providerResult.normalized, + ); + return buildResult( + metadata, + provider, evalCase, - 'Provider invocation failed before artifact verification.', - ), - persisted.transcriptPath, - persisted.stdoutPath, - persisted.stderrPath, - undefined, - undefined, - error instanceof Error ? error.name : 'ProviderInvocationError', - errorMessage, - ); + condition, + providerResult.runtime, + providerResult.normalized, + providerResult.ok, + providerResult.startedAt, + providerResult.completedAt, + providerResult.durationMs, + checkWorkflow(transcript, evalCase.workflowChecks), + detectAntiPatterns(scannableTranscript, evalCase.antiPatterns), + verifierResults, + artifactResults, + providerResult.transcriptPath ?? persisted.transcriptPath, + persisted.stdoutPath, + persisted.stderrPath, + providerResult.bundlePath, + providerResult.eventLogPath, + providerResult.errorClass, + providerResult.errorMessage, + ); + } catch (error) { + const completedAt = new Date().toISOString(); + const durationMs = Math.max(0, Date.now() - invocationStartedMs); + const errorMessage = error instanceof Error ? error.message : String(error); + const transcript = + error instanceof Error && error.stack !== undefined + ? `${error.name}: ${errorMessage}\n${error.stack}` + : errorMessage; + const normalizedOutput = createFallbackNormalizedOutput(transcript); + const scannableTranscript = buildScannableTranscript(normalizedOutput); + const persisted = await persistRunnerArtifacts( + outputDir, + transcript, + '', + errorMessage, + ); + return buildResult( + metadata, + provider, + evalCase, + condition, + runtime, + normalizedOutput, + false, + invocationStartedAt, + completedAt, + durationMs, + checkWorkflow(transcript, evalCase.workflowChecks), + detectAntiPatterns(scannableTranscript, evalCase.antiPatterns), + buildVerifierFailures( + evalCase, + 'Provider invocation failed before verifier execution.', + ), + buildArtifactFailures( + evalCase, + 'Provider invocation failed before artifact verification.', + ), + persisted.transcriptPath, + persisted.stdoutPath, + persisted.stderrPath, + undefined, + undefined, + error instanceof Error ? error.name : 'ProviderInvocationError', + errorMessage, + ); + } + } finally { + try { + await cleanupEvalHome(homeDir); + } catch { + // Best-effort cleanup; preserve the eval result when temp-home cleanup fails. + } + try { + await rm(outputDir, { recursive: true, force: true }); + } catch { + // Best-effort cleanup; preserve the eval result when temp-dir cleanup fails. + } } } From 860a2df4cbf25ad75c108fe8303c617208468f93 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 09:55:32 +0000 Subject: [PATCH 03/18] feat(evals): add shared bounded work-item scheduler --- evals/lib/scheduler.ts | 66 ++++++++++ test/unit/evals/scheduler.test.ts | 212 ++++++++++++++++++++++++++++++ 2 files changed, 278 insertions(+) create mode 100644 evals/lib/scheduler.ts create mode 100644 test/unit/evals/scheduler.test.ts diff --git a/evals/lib/scheduler.ts b/evals/lib/scheduler.ts new file mode 100644 index 00000000..906be21c --- /dev/null +++ b/evals/lib/scheduler.ts @@ -0,0 +1,66 @@ +import { invariant } from '../../src/util/assert.js'; + +export interface SchedulerOptions { + concurrency: number; + logLine?: (line: string) => void; +} + +export type ScheduledWorkItem = { key: string }; + +export type SettledResult = + | { item: T; status: 'fulfilled'; value: R } + | { item: T; status: 'rejected'; reason: unknown }; + +export async function runScheduled( + items: readonly T[], + executor: (item: T) => Promise, + options: SchedulerOptions, +): Promise[]> { + invariant(Array.isArray(items), 'items must be an array'); + invariant( + Number.isInteger(options.concurrency) && options.concurrency > 0, + 'options.concurrency must be a positive integer', + ); + + if (items.length === 0) { + return []; + } + + const settlements: Array | undefined> = new Array(items.length); + const workerCount = Math.min(options.concurrency, items.length); + const logLine = options.logLine; + let nextIndex = 0; + + async function worker(): Promise { + while (true) { + const index = nextIndex; + if (index >= items.length) { + return; + } + nextIndex += 1; + + const item = items[index]; + invariant(item !== undefined, `Missing scheduled item at index ${index}`); + + logLine?.(`[${item.key}] start`); + try { + const value = await executor(item); + settlements[index] = { item, status: 'fulfilled', value }; + logLine?.(`[${item.key}] ok`); + } catch (reason) { + settlements[index] = { item, status: 'rejected', reason }; + logLine?.(`[${item.key}] failed: ${String(reason)}`); + } + } + } + + await Promise.all(Array.from({ length: workerCount }, () => worker())); + + return settlements.map((settlement, index) => { + invariant( + settlement !== undefined, + `Missing scheduler settlement for item at index ${index}`, + ); + return settlement; + }); +} diff --git a/test/unit/evals/scheduler.test.ts b/test/unit/evals/scheduler.test.ts new file mode 100644 index 00000000..a9586814 --- /dev/null +++ b/test/unit/evals/scheduler.test.ts @@ -0,0 +1,212 @@ +import { describe, expect, it } from 'vitest'; + +import { runScheduled } from '../../../evals/lib/scheduler.js'; +import type { SettledResult } from '../../../evals/lib/scheduler.js'; + +interface TestItem { + key: string; + delayMs?: number; +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function expectRejectedError( + settlement: SettledResult | undefined, + message: string, +): void { + expect(settlement).toBeDefined(); + if (settlement === undefined) { + throw new Error('Expected settlement'); + } + + expect(settlement.status).toBe('rejected'); + if (settlement.status !== 'rejected') { + throw new Error('Expected rejected settlement'); + } + + expect(settlement.reason).toBeInstanceOf(Error); + if (!(settlement.reason instanceof Error)) { + throw new Error('Expected Error rejection reason'); + } + + expect(settlement.reason.message).toBe(message); +} + +describe('runScheduled', () => { + it('respects max concurrency', async () => { + const items = Array.from({ length: 6 }, (_, index) => ({ key: `item-${index}` })); + let inFlight = 0; + let maxInFlight = 0; + + const results = await runScheduled( + items, + async (item) => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + await delay(20); + inFlight -= 1; + return item.key; + }, + { concurrency: 2 }, + ); + + expect(results).toHaveLength(items.length); + expect(inFlight).toBe(0); + expect(maxInFlight).toBe(2); + }); + + it('returns an empty array for empty items', async () => { + let called = false; + + const results = await runScheduled( + [], + async () => { + called = true; + return 'unused'; + }, + { concurrency: 1 }, + ); + + expect(results).toEqual([]); + expect(called).toBe(false); + }); + + it('captures synchronous throws as rejected settlements', async () => { + const items: TestItem[] = [{ key: 'sync-throw' }]; + + const results = await runScheduled( + items, + (_item): Promise => { + throw new Error('sync boom'); + }, + { concurrency: 1 }, + ); + + expectRejectedError(results[0], 'sync boom'); + expect(results[0]).toMatchObject({ item: items[0], status: 'rejected' }); + }); + + it('captures async rejects as rejected settlements', async () => { + const items: TestItem[] = [{ key: 'async-reject' }]; + + const results = await runScheduled( + items, + () => Promise.reject(new Error('async boom')), + { concurrency: 1 }, + ); + + expectRejectedError(results[0], 'async boom'); + expect(results[0]).toMatchObject({ item: items[0], status: 'rejected' }); + }); + + it('continues running successful siblings after failures', async () => { + const items: TestItem[] = [ + { key: 'success-1', delayMs: 15 }, + { key: 'fail-1', delayMs: 5 }, + { key: 'success-2', delayMs: 1 }, + { key: 'fail-2', delayMs: 10 }, + ]; + + const results = await runScheduled( + items, + async (item) => { + await delay(item.delayMs ?? 0); + if (item.key.startsWith('fail')) { + throw new Error(`${item.key} failed`); + } + return `${item.key} ok`; + }, + { concurrency: 2 }, + ); + + expect(results).toHaveLength(items.length); + expect(results.map((settlement) => settlement.status)).toEqual([ + 'fulfilled', + 'rejected', + 'fulfilled', + 'rejected', + ]); + + const [first, second, third, fourth] = results; + expect(first).toMatchObject({ + item: items[0], + status: 'fulfilled', + value: 'success-1 ok', + }); + expect(second).toMatchObject({ item: items[1], status: 'rejected' }); + expect(third).toMatchObject({ + item: items[2], + status: 'fulfilled', + value: 'success-2 ok', + }); + expect(fourth).toMatchObject({ item: items[3], status: 'rejected' }); + }); + + it('preserves input order in returned settlements', async () => { + const items: TestItem[] = [ + { key: 'item-0', delayMs: 30 }, + { key: 'item-1', delayMs: 20 }, + { key: 'item-2', delayMs: 10 }, + { key: 'item-3', delayMs: 0 }, + ]; + + const results = await runScheduled( + items, + async (item) => { + await delay(item.delayMs ?? 0); + return item.key; + }, + { concurrency: items.length }, + ); + + expect(results.map((settlement) => settlement.item.key)).toEqual( + items.map((item) => item.key), + ); + expect( + results.map((settlement) => + settlement.status === 'fulfilled' ? settlement.value : 'rejected', + ), + ).toEqual(items.map((item) => item.key)); + }); + + it('logs lifecycle lines with the work-item key prefix', async () => { + const items: TestItem[] = [{ key: 'alpha' }, { key: 'beta' }]; + const lines: string[] = []; + + await runScheduled( + items, + async (item) => { + if (item.key === 'beta') { + throw new Error('beta failed'); + } + return `${item.key} ok`; + }, + { + concurrency: 1, + logLine: (line) => lines.push(line), + }, + ); + + expect(lines).toEqual([ + '[alpha] start', + '[alpha] ok', + '[beta] start', + '[beta] failed: Error: beta failed', + ]); + for (const item of items) { + expect(lines.some((line) => line.startsWith(`[${item.key}]`))).toBe(true); + } + }); + + it('asserts on invalid concurrency values', async () => { + const items: TestItem[] = [{ key: 'item-0' }]; + + for (const concurrency of [0, -1, 1.5]) { + await expect( + runScheduled(items, async (item) => item.key, { concurrency }), + ).rejects.toThrow('options.concurrency must be a positive integer'); + } + }); +}); From f2d02f7bd185e58a8e8a0e672c4a3077a37252fa Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 10:04:26 +0000 Subject: [PATCH 04/18] refactor(evals): split prompt lane into enumerate + execute + schedule --- evals/prompt/runner.ts | 248 +++++++++++++++++++++++++++++++++-------- 1 file changed, 202 insertions(+), 46 deletions(-) diff --git a/evals/prompt/runner.ts b/evals/prompt/runner.ts index 0e272960..43fb97ac 100644 --- a/evals/prompt/runner.ts +++ b/evals/prompt/runner.ts @@ -3,6 +3,7 @@ import { readFile } from 'node:fs/promises'; import { assertString, invariant } from '../../src/util/assert.js'; import { detectAntiPatterns } from '../lib/antiPatterns.js'; import { SKILL_CONDITIONS } from '../lib/matrix.js'; +import { runScheduled } from '../lib/scheduler.js'; import { EvalResultSchema, PromptEvalCaseSchema, @@ -10,9 +11,12 @@ import { RunMetadataSchema, } from '../lib/schemas.js'; import { scorePromptCase } from '../lib/scoring.js'; +import type { ScheduledWorkItem, SettledResult } from '../lib/scheduler.js'; +import { assertUniqueWorkItems, buildWorkItemKey } from '../lib/types.js'; import type { AntiPatternFinding, EvalResult, + EvalWorkItemIdentity, PromptCaseScore, PromptEvalCase, ProviderPromptRequest, @@ -49,6 +53,26 @@ interface LoadedSkillPrompts { let loadedSkillPromptsPromise: Promise | undefined; +type PromptWorkItemOptions = { + conditions?: SkillCondition[]; + caseFilter?: string[]; +}; + +type PromptLaneOptions = PromptWorkItemOptions & { + concurrency?: number; +}; + +type PromptWorkItem = EvalWorkItemIdentity & + ScheduledWorkItem & { + evalCase: PromptEvalCase; + systemPrompt: string; + }; + +type RejectedPromptWorkItemSettlement = Extract< + SettledResult, + { status: 'rejected' } +>; + function clonePromptCase(evalCase: PromptEvalCase): PromptEvalCase { return PromptEvalCaseSchema.parse({ ...evalCase, @@ -128,6 +152,45 @@ function resolveRequestedCases( }); } +function validatePromptRunMetadata(metadata: RunMetadata): RunMetadata { + const parsedMetadata = RunMetadataSchema.parse(metadata) as RunMetadata; + invariant( + parsedMetadata.lanes.includes('prompt'), + 'Run metadata must include the prompt lane', + ); + invariant( + parsedMetadata.totalTrials > 0, + 'Run metadata totalTrials must be positive for prompt-lane runs', + ); + return parsedMetadata; +} + +function assertPromptProviderMetadata( + provider: EvalProvider, + metadata: RunMetadata, +): void { + invariant( + metadata.providers.includes(provider.id), + `Run metadata providers must include ${provider.id}`, + ); +} + +function assertPromptWorkItem(workItem: PromptWorkItem): void { + invariant(workItem.lane === 'prompt', 'Prompt work item lane must be prompt'); + invariant( + workItem.caseId === workItem.evalCase.id, + `Prompt work item case id must match eval case id for ${workItem.key}`, + ); + invariant( + workItem.systemPrompt.length > 0, + `Prompt work item system prompt must not be empty for ${workItem.key}`, + ); + invariant( + workItem.key === buildWorkItemKey(workItem), + `Prompt work item key must match identity for ${workItem.caseId}`, + ); +} + function buildFallbackNormalizedOutput( errorMessage?: string, ): EvalResult['normalizedOutput'] { @@ -304,6 +367,54 @@ function buildErrorEvalResult( }) as EvalResult; } +function buildRejectedPromptWorkItemEvalResult( + provider: EvalProvider, + metadata: RunMetadata, + settlement: RejectedPromptWorkItemSettlement, +): EvalResult { + const errorMessage = + settlement.reason instanceof Error + ? settlement.reason.message + : String(settlement.reason); + const message = `Unexpected scheduler rejection for ${settlement.item.key}: ${errorMessage}`; + const errorClass = + settlement.reason instanceof Error ? settlement.reason.name : 'Error'; + const promptScore: PromptCaseScore = scorePromptCase( + '', + settlement.item.evalCase, + ); + const antiPatternFindings = mergeAntiPatternFindings(detectAntiPatterns('')); + const startedAt = new Date().toISOString(); + const completedAt = new Date().toISOString(); + const durationMs = Math.max( + 0, + Date.parse(completedAt) - Date.parse(startedAt), + ); + + return EvalResultSchema.parse({ + runId: metadata.runId, + providerId: provider.id, + modelId: metadata.models[0], + lane: settlement.item.lane, + caseId: settlement.item.caseId, + category: settlement.item.evalCase.category, + condition: settlement.item.condition, + expectedSkill: settlement.item.evalCase.expectedSkill, + trial: settlement.item.trial, + ok: false, + score: promptScore.breakdown, + promptScore, + workflowChecks: promptScore.workflowChecks, + antiPatternFindings, + normalizedOutput: buildFallbackNormalizedOutput(message), + errorClass, + errorMessage: message, + startedAt, + completedAt, + durationMs, + }) as EvalResult; +} + async function readSkillFile(relativePath: string): Promise { const content = await readFile( new URL(relativePath, import.meta.url), @@ -377,33 +488,18 @@ export function getAllPromptCases(): PromptEvalCase[] { return allCases; } -export async function runPromptLane( - provider: EvalProvider, +async function enumeratePromptWorkItemsFromMetadata( metadata: RunMetadata, - options?: { conditions?: SkillCondition[]; caseFilter?: string[] }, -): Promise { - const parsedMetadata = RunMetadataSchema.parse(metadata) as RunMetadata; - invariant( - parsedMetadata.lanes.includes('prompt'), - 'Run metadata must include the prompt lane', - ); - invariant( - parsedMetadata.providers.includes(provider.id), - `Run metadata providers must include ${provider.id}`, - ); - invariant( - parsedMetadata.totalTrials > 0, - 'Run metadata totalTrials must be positive for prompt-lane runs', - ); - + options?: PromptWorkItemOptions, +): Promise { const allCases = getAllPromptCases(); const requestedCases = resolveRequestedCases(allCases, options?.caseFilter); const requestedConditions = resolveRequestedConditions( - parsedMetadata, + metadata, options?.conditions, ); const loadedSkillPrompts = await loadSkillPrompts(); - const results: EvalResult[] = []; + const items: PromptWorkItem[] = []; for (const evalCase of requestedCases) { for (const condition of requestedConditions) { @@ -414,40 +510,100 @@ export async function runPromptLane( for ( let trialIndex = 0; - trialIndex < parsedMetadata.totalTrials; + trialIndex < metadata.totalTrials; trialIndex += 1 ) { const trial = trialIndex + 1; - const request = buildPromptRequest( - provider, - parsedMetadata, - evalCase, + const identity: EvalWorkItemIdentity = { + lane: 'prompt', + caseId: evalCase.id, condition, trial, + }; + items.push({ + ...identity, + key: buildWorkItemKey(identity), + evalCase, systemPrompt, - ); - const startedAt = new Date().toISOString(); - const startedAtMs = Date.now(); - - try { - const promptResult = await provider.invokePlanMode(request); - results.push(buildEvalResult(request, promptResult)); - } catch (error: unknown) { - const completedAt = new Date().toISOString(); - const durationMs = Math.max(0, Date.now() - startedAtMs); - results.push( - buildErrorEvalResult( - request, - error, - startedAt, - completedAt, - durationMs, - ), - ); - } + }); } } } - return results; + assertUniqueWorkItems(items); + return items; +} + +export async function enumeratePromptWorkItems( + metadata: RunMetadata, + options?: PromptWorkItemOptions, +): Promise { + const parsedMetadata = validatePromptRunMetadata(metadata); + return enumeratePromptWorkItemsFromMetadata(parsedMetadata, options); +} + +export async function executePromptWorkItem( + provider: EvalProvider, + metadata: RunMetadata, + workItem: PromptWorkItem, +): Promise { + assertPromptProviderMetadata(provider, metadata); + assertPromptWorkItem(workItem); + + const request = buildPromptRequest( + provider, + metadata, + workItem.evalCase, + workItem.condition, + workItem.trial, + workItem.systemPrompt, + ); + const startedAt = new Date().toISOString(); + const startedAtMs = Date.now(); + + try { + const promptResult = await provider.invokePlanMode(request); + return buildEvalResult(request, promptResult); + } catch (error: unknown) { + const completedAt = new Date().toISOString(); + const durationMs = Math.max(0, Date.now() - startedAtMs); + return buildErrorEvalResult( + request, + error, + startedAt, + completedAt, + durationMs, + ); + } +} + +export async function runPromptLane( + provider: EvalProvider, + metadata: RunMetadata, + options?: PromptLaneOptions, +): Promise { + const parsedMetadata = validatePromptRunMetadata(metadata); + assertPromptProviderMetadata(provider, parsedMetadata); + + const items = await enumeratePromptWorkItemsFromMetadata( + parsedMetadata, + options, + ); + const settlements = await runScheduled( + items, + async (item) => executePromptWorkItem(provider, parsedMetadata, item), + { + concurrency: options?.concurrency ?? 1, + }, + ); + + return settlements.map((settlement) => + settlement.status === 'fulfilled' + ? settlement.value + : buildRejectedPromptWorkItemEvalResult( + provider, + parsedMetadata, + settlement, + ), + ); } From 2e77534a66349538e88a318734cd7ed81f52827b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 10:01:10 +0000 Subject: [PATCH 05/18] refactor(evals): split execution lane into enumerate + execute + schedule --- evals/execution/runner.ts | 174 +++++++++++++++++++++++++++++++------- 1 file changed, 144 insertions(+), 30 deletions(-) diff --git a/evals/execution/runner.ts b/evals/execution/runner.ts index e5c7acd1..c1debc34 100644 --- a/evals/execution/runner.ts +++ b/evals/execution/runner.ts @@ -10,11 +10,15 @@ import { } from '../lib/antiPatterns.js'; import { cleanupEvalHome, createIsolatedEvalHome } from '../lib/cliHarness.js'; import { EvalResultSchema } from '../lib/schemas.js'; +import { runScheduled } from '../lib/scheduler.js'; import { checkWorkflow } from '../lib/scoring.js'; +import { assertUniqueWorkItems, buildWorkItemKey } from '../lib/types.js'; +import type { ScheduledWorkItem } from '../lib/scheduler.js'; import type { AntiPatternFinding, ArtifactRequirement, EvalResult, + EvalWorkItemIdentity, ExecutionEvalCase, NormalizedProviderOutput, ProviderRuntimeInfo, @@ -60,6 +64,11 @@ const CASE_CATEGORY_EXPECTATIONS = { const RUNNER_OUTPUT_PREFIX = 'agent-tty-execution-eval-'; const DEFAULT_TRIAL = 1; +type ExecutionWorkItem = EvalWorkItemIdentity & + ScheduledWorkItem & { + evalCase: ExecutionEvalCase; + }; + type EvaluatedVerifier = { spec: VerifierSpec; result: VerifierResult; @@ -665,6 +674,76 @@ function normalizeRequestedConditions( return new Set(conditions); } +function buildExecutionWorkItem( + evalCase: ExecutionEvalCase, + condition: SkillCondition, +): ExecutionWorkItem { + const identity: EvalWorkItemIdentity = { + lane: 'execution', + caseId: evalCase.id, + condition, + trial: DEFAULT_TRIAL, + }; + + return { + ...identity, + key: buildWorkItemKey(identity), + evalCase, + }; +} + +function buildRejectedExecutionWorkItemResult( + provider: EvalProvider, + metadata: RunMetadata, + workItem: ExecutionWorkItem, + runtime: ProviderRuntimeInfo | undefined, + reason: unknown, +): EvalResult { + const errorClass = + reason instanceof Error && reason.name.length > 0 + ? reason.name + : 'ExecutionWorkItemError'; + const rawErrorMessage = + reason instanceof Error && reason.message.length > 0 + ? reason.message + : safeStringify(reason); + const errorMessage = + rawErrorMessage.length > 0 + ? rawErrorMessage + : 'Unknown execution work item failure'; + const timestamp = new Date().toISOString(); + const modelId = resolveModelId(metadata, runtime); + + return EvalResultSchema.parse({ + runId: metadata.runId, + providerId: provider.id, + ...(runtime?.version === undefined + ? {} + : { providerVersion: runtime.version }), + ...(modelId === undefined ? {} : { modelId }), + lane: 'execution', + caseId: workItem.evalCase.id, + category: workItem.evalCase.category, + condition: workItem.condition, + expectedSkill: workItem.evalCase.expectedSkill, + trial: workItem.trial, + ok: false, + score: { + total: 0, + maxPossible: 0, + items: [], + }, + workflowChecks: [], + antiPatternFindings: [], + normalizedOutput: createFallbackNormalizedOutput(''), + errorClass, + errorMessage, + startedAt: timestamp, + completedAt: timestamp, + durationMs: 0, + }) as EvalResult; +} + function resolveModelId( metadata: RunMetadata, runtime: ProviderRuntimeInfo | undefined, @@ -989,18 +1068,13 @@ export function getAllExecutionCases(): ExecutionEvalCase[] { return [...EXECUTION_CASES]; } -/** - * Run the execution eval lane for one provider across the selected case and - * condition matrix. - */ -export async function runExecutionLane( - provider: EvalProvider, - metadata: RunMetadata, - options: { conditions?: SkillCondition[]; caseFilter?: string[] } = {}, -): Promise { - const requestedConditions = normalizeRequestedConditions(options.conditions); +export function enumerateExecutionWorkItems(options?: { + conditions?: SkillCondition[]; + caseFilter?: string[]; +}): ExecutionWorkItem[] { + const requestedConditions = normalizeRequestedConditions(options?.conditions); const requestedCaseIds = - options.caseFilter === undefined || options.caseFilter.length === 0 + options?.caseFilter === undefined || options.caseFilter.length === 0 ? undefined : new Set(options.caseFilter); const availableCases = getAllExecutionCases(); @@ -1017,30 +1091,70 @@ export async function runExecutionLane( } } - const selectedCases = availableCases.filter( - (evalCase) => - requestedCaseIds === undefined || requestedCaseIds.has(evalCase.id), + const items = availableCases.flatMap((evalCase) => { + if (requestedCaseIds !== undefined && !requestedCaseIds.has(evalCase.id)) { + return []; + } + + return evalCase.conditions + .filter( + (condition) => + requestedConditions === undefined || requestedConditions.has(condition), + ) + .map((condition) => buildExecutionWorkItem(evalCase, condition)); + }); + + assertUniqueWorkItems(items); + return items; +} + +export async function executeExecutionWorkItem( + provider: EvalProvider, + metadata: RunMetadata, + workItem: ExecutionWorkItem, + runtime: ProviderRuntimeInfo | undefined, +): Promise { + return runSingleExecutionCase( + provider, + metadata, + workItem.evalCase, + workItem.condition, + runtime, ); +} + +/** + * Run the execution eval lane for one provider across the selected case and + * condition matrix. + */ +export async function runExecutionLane( + provider: EvalProvider, + metadata: RunMetadata, + options: { + conditions?: SkillCondition[]; + caseFilter?: string[]; + concurrency?: number; + } = {}, +): Promise { + const items = enumerateExecutionWorkItems(options); const runtime = await detectRuntime(provider); - const results: EvalResult[] = []; + const settlements = await runScheduled( + items, + (item) => executeExecutionWorkItem(provider, metadata, item, runtime), + { + concurrency: options.concurrency ?? 1, + }, + ); - for (const evalCase of selectedCases) { - const selectedConditions = evalCase.conditions.filter( - (condition) => - requestedConditions === undefined || requestedConditions.has(condition), - ); - for (const condition of selectedConditions) { - results.push( - await runSingleExecutionCase( + return settlements.map((settlement) => + settlement.status === 'fulfilled' + ? settlement.value + : buildRejectedExecutionWorkItemResult( provider, metadata, - evalCase, - condition, + settlement.item, runtime, + settlement.reason, ), - ); - } - } - - return results; + ); } From 754a6fc7b667ba5d1942f21351f211ddc24986d0 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 10:02:53 +0000 Subject: [PATCH 06/18] refactor(evals): split dogfood lane into enumerate + execute + schedule --- evals/dogfood/runner.ts | 638 +++++++++++++++++++++++----------------- 1 file changed, 375 insertions(+), 263 deletions(-) diff --git a/evals/dogfood/runner.ts b/evals/dogfood/runner.ts index b87df0bf..e1bbb004 100644 --- a/evals/dogfood/runner.ts +++ b/evals/dogfood/runner.ts @@ -13,13 +13,17 @@ import { scoreEvidenceQuality, scoreReportCompleteness, } from '../lib/bundleScoring.js'; -import { SKILL_CONDITIONS } from '../lib/matrix.js'; import { fixtureCommand } from '../lib/cliHarness.js'; +import { SKILL_CONDITIONS } from '../lib/matrix.js'; import { DogfoodEvalCaseSchema, EvalResultSchema } from '../lib/schemas.js'; import { checkWorkflow } from '../lib/scoring.js'; +import { runScheduled } from '../lib/scheduler.js'; +import { assertUniqueWorkItems, buildWorkItemKey } from '../lib/types.js'; +import type { ScheduledWorkItem } from '../lib/scheduler.js'; import type { DogfoodEvalCase, EvalResult, + EvalWorkItemIdentity, NormalizedProviderOutput, ProviderRuntimeInfo, RunMetadata, @@ -63,12 +67,23 @@ const NO_CAPABILITIES: ProviderRuntimeInfo['capabilities'] = { supportsTranscriptCapture: false, }; +type DogfoodWorkItem = EvalWorkItemIdentity & + ScheduledWorkItem & { + evalCase: DogfoodEvalCase; + }; + interface LoadedDogfoodSkillPrompts { bootstrapSkillText: string; canonicalAgentTtySkillText: string; canonicalDogfoodSkillText: string; } +interface DogfoodLaneContext { + detectedRuntime: ProviderRuntimeInfo; + requestedModelId: string | undefined; + repoRoot: string; +} + let loadedDogfoodSkillPromptsPromise: | Promise | undefined; @@ -545,11 +560,90 @@ export function getAllDogfoodCases(): DogfoodEvalCase[] { return buildCaseInventory(); } -export async function runDogfoodLane( - provider: EvalProvider, - metadata: RunMetadata, - options?: { conditions?: SkillCondition[]; caseFilter?: string[] }, -): Promise { +function buildDogfoodWorkItem( + evalCase: DogfoodEvalCase, + condition: SkillCondition, +): DogfoodWorkItem { + const identity: EvalWorkItemIdentity = { + lane: 'dogfood', + caseId: evalCase.id, + condition, + trial: 1, + }; + + return { + ...identity, + key: buildWorkItemKey(identity), + evalCase, + }; +} + +function describeDogfoodError(error: unknown): { + errorClass: string; + errorMessage: string; +} { + return { + errorClass: + error instanceof Error && error.name.length > 0 ? error.name : 'Error', + errorMessage: error instanceof Error ? error.message : String(error), + }; +} + +function buildFailedDogfoodEvalResult(params: { + metadata: RunMetadata; + providerId: string; + providerVersion: string | undefined; + requestedModelId: string | undefined; + workItem: DogfoodWorkItem; + startedAt?: string; + error: unknown; +}): EvalResult { + const completedAt = new Date().toISOString(); + const startedAt = params.startedAt ?? completedAt; + const { errorClass, errorMessage } = describeDogfoodError(params.error); + + return EvalResultSchema.parse({ + runId: params.metadata.runId, + providerId: params.providerId, + ...(params.providerVersion === undefined + ? {} + : { providerVersion: params.providerVersion }), + ...(params.requestedModelId === undefined + ? {} + : { modelId: params.requestedModelId }), + lane: 'dogfood', + caseId: params.workItem.caseId, + category: params.workItem.evalCase.category, + condition: params.workItem.condition, + expectedSkill: params.workItem.evalCase.expectedSkill, + trial: 1, + ok: false, + score: { + total: 0, + maxPossible: 1, + items: buildDogfoodBreakdownItems({ + bundleCompleteness: 0, + reportCompleteness: 0, + evidenceQuality: 0, + taxonomyUsage: 0, + reproducibility: 0, + }), + }, + workflowChecks: [], + antiPatternFindings: [], + normalizedOutput: EMPTY_NORMALIZED_OUTPUT, + errorClass, + errorMessage, + startedAt, + completedAt, + durationMs: Math.max(0, Date.parse(completedAt) - Date.parse(startedAt)), + }) as EvalResult; +} + +export function enumerateDogfoodWorkItems(options?: { + conditions?: SkillCondition[]; + caseFilter?: string[]; +}): DogfoodWorkItem[] { const selectedConditions = validateConditionList(options?.conditions); const selectedCaseIds = validateCaseFilter(options?.caseFilter); const allCases = getAllDogfoodCases(); @@ -557,6 +651,258 @@ export async function runDogfoodLane( selectedCaseIds === undefined ? allCases : allCases.filter((evalCase) => selectedCaseIds.includes(evalCase.id)); + const items: DogfoodWorkItem[] = []; + + for (const evalCase of cases) { + const caseConditions = + selectedConditions === undefined + ? evalCase.conditions + : evalCase.conditions.filter((condition) => + selectedConditions.includes(condition), + ); + + for (const condition of caseConditions) { + items.push(buildDogfoodWorkItem(evalCase, condition)); + } + } + + assertUniqueWorkItems(items); + return items; +} + +export async function executeDogfoodWorkItem( + provider: EvalProvider, + metadata: RunMetadata, + workItem: DogfoodWorkItem, + ctx: DogfoodLaneContext, +): Promise { + invariant( + workItem.lane === 'dogfood', + 'Dogfood work item lane must be dogfood', + ); + invariant( + workItem.evalCase.id === workItem.caseId, + 'Dogfood work item caseId must match evalCase.id', + ); + invariant( + workItem.evalCase.conditions.includes(workItem.condition), + 'Dogfood work item condition must be supported by the eval case', + ); + invariant(workItem.trial === 1, 'Dogfood work item trial must be 1'); + + const startedAt = new Date().toISOString(); + const { outputDir, homeDir, requestedBundlePath } = buildRequestedPaths( + metadata, + provider.id, + workItem.evalCase, + workItem.condition, + ); + + try { + await mkdir(outputDir, { recursive: true }); + await mkdir(homeDir, { recursive: true }); + + const systemPromptContext = await buildSystemPromptContext( + workItem.condition, + ); + const requestCase = DogfoodEvalCaseSchema.parse({ + ...workItem.evalCase, + prompt: buildPrompt( + { ...workItem.evalCase, conditions: [workItem.condition] }, + requestedBundlePath, + systemPromptContext, + ), + bundlePath: requestedBundlePath, + conditions: [workItem.condition], + }) as DogfoodEvalCase; + + const agentResult = await provider.invokeAgentMode({ + runId: metadata.runId, + providerId: provider.id, + condition: workItem.condition, + trial: workItem.trial, + ...(ctx.requestedModelId === undefined + ? {} + : { modelId: ctx.requestedModelId }), + cwd: ctx.repoRoot, + homeDir, + outputDir, + env: { + AGENT_TTY_HOME: homeDir, + EVAL_OUTPUT_DIR: outputDir, + EVAL_REQUESTED_BUNDLE_DIR: requestedBundlePath, + EVAL_FIXTURE: workItem.evalCase.fixture ?? '', + ...systemPromptContext.env, + }, + evalCase: requestCase, + }); + + const transcript = await resolveTranscriptText(agentResult); + const bundlePath = await resolveBundlePath( + agentResult.bundlePath, + requestedBundlePath, + ); + const reportText = await resolveReportText( + bundlePath, + agentResult.normalized.finalText, + ); + const dogfoodScore = await scoreDogfoodRun( + bundlePath ?? requestedBundlePath, + reportText, + transcript, + workItem.evalCase, + ); + const reportRequirementScore = scoreReportRequirements( + reportText ?? '', + workItem.evalCase.reportRequirements, + ); + const reportSections = workItem.evalCase.reportRequirements + .map((requirement) => requirement.section) + .filter( + (section): section is string => + typeof section === 'string' && section.trim().length > 0, + ); + const baseReportCompleteness = scoreReportCompleteness( + reportText ?? '', + reportSections, + ); + const reportCompleteness = { + ...baseReportCompleteness, + sectionsExpected: reportRequirementScore.details.length, + sectionsFound: reportRequirementScore.details.filter( + (detail) => detail.found, + ).length, + score: clampUnitInterval( + (baseReportCompleteness.score + reportRequirementScore.score) / 2, + ), + details: reportRequirementScore.details.map((detail) => ({ + section: detail.section, + found: detail.found, + required: + workItem.evalCase.reportRequirements.find( + (requirement) => + (requirement.section ?? requirement.id) === detail.section, + )?.required ?? true, + })), + missingSections: reportRequirementScore.details + .filter((detail) => !detail.found) + .map((detail) => detail.section), + }; + const bundleCompleteness = await scoreBundleCompleteness( + bundlePath ?? requestedBundlePath, + workItem.evalCase.validationProfile, + ); + const evidenceQuality = await scoreEvidenceQuality( + bundlePath ?? requestedBundlePath, + ); + const workflowChecks = + requestCase.workflowChecks.length === 0 + ? [] + : checkWorkflow(transcript, requestCase.workflowChecks); + const scannableTranscript = buildScannableTranscript( + agentResult.normalized, + ); + const antiPatternFindings = detectAntiPatterns( + scannableTranscript, + requestCase.antiPatterns, + ); + const artifactManifestPath = await resolveArtifactManifestPath(bundlePath); + const blockingAntiPattern = antiPatternFindings.some( + (finding) => finding.severity === 'error', + ); + const missingRequiredReportSection = reportRequirementScore.details.some( + (detail) => !detail.found, + ); + const missingRequiredWorkflow = workflowChecks.some( + (check) => !check.passed, + ); + const ok = + agentResult.ok && + !blockingAntiPattern && + !missingRequiredReportSection && + !missingRequiredWorkflow && + dogfoodScore.overallScore >= 0.6; + const completedAt = agentResult.completedAt; + + return EvalResultSchema.parse({ + runId: metadata.runId, + providerId: provider.id, + ...(agentResult.runtime.version === undefined + ? {} + : { providerVersion: agentResult.runtime.version }), + ...(ctx.requestedModelId === undefined + ? {} + : { modelId: ctx.requestedModelId }), + lane: 'dogfood', + caseId: workItem.caseId, + category: workItem.evalCase.category, + condition: workItem.condition, + expectedSkill: workItem.evalCase.expectedSkill, + trial: workItem.trial, + ok, + score: { + total: dogfoodScore.overallScore, + maxPossible: 1, + items: buildDogfoodBreakdownItems(dogfoodScore), + }, + workflowChecks, + antiPatternFindings, + bundleCompleteness, + reportCompleteness, + evidenceQuality, + ...(agentResult.transcriptPath === undefined + ? {} + : { transcriptPath: agentResult.transcriptPath }), + ...(bundlePath === undefined ? {} : { bundlePath }), + ...(artifactManifestPath === undefined ? {} : { artifactManifestPath }), + ...(agentResult.eventLogPath === undefined + ? {} + : { eventLogPath: agentResult.eventLogPath }), + normalizedOutput: agentResult.normalized, + ...(agentResult.errorClass === undefined + ? {} + : { errorClass: agentResult.errorClass }), + ...(agentResult.errorMessage === undefined + ? {} + : { errorMessage: agentResult.errorMessage }), + startedAt: agentResult.startedAt, + completedAt, + durationMs: agentResult.durationMs, + }) as EvalResult; + } catch (error) { + return buildFailedDogfoodEvalResult({ + metadata, + providerId: provider.id, + providerVersion: ctx.detectedRuntime.version, + requestedModelId: ctx.requestedModelId, + workItem, + startedAt, + error, + }); + } finally { + try { + await rm(homeDir, { recursive: true, force: true }); + } catch { + // Best-effort cleanup; preserve the eval result when temp-home cleanup fails. + } + try { + await rm(outputDir, { recursive: true, force: true }); + } catch { + // Best-effort cleanup; preserve the eval result when temp-dir cleanup fails. + } + } +} + +export async function runDogfoodLane( + provider: EvalProvider, + metadata: RunMetadata, + options?: { + conditions?: SkillCondition[]; + caseFilter?: string[]; + concurrency?: number; + }, +): Promise { + const items = enumerateDogfoodWorkItems(options); let detectedRuntime: ProviderRuntimeInfo; try { @@ -569,264 +915,30 @@ export async function runDogfoodLane( ); } - const requestedModelId = - metadata.models[0] ?? detectedRuntime.defaultModelId ?? undefined; - const repoRoot = resolve(metadata.repoRoot); - const results: EvalResult[] = []; - - for (const evalCase of cases) { - const caseConditions = - selectedConditions === undefined - ? evalCase.conditions - : evalCase.conditions.filter((condition) => - selectedConditions.includes(condition), - ); + const ctx: DogfoodLaneContext = { + detectedRuntime, + requestedModelId: + metadata.models[0] ?? detectedRuntime.defaultModelId ?? undefined, + repoRoot: resolve(metadata.repoRoot), + }; + const settlements = await runScheduled( + items, + (item) => executeDogfoodWorkItem(provider, metadata, item, ctx), + { concurrency: options?.concurrency ?? 1 }, + ); - for (const condition of caseConditions) { - const startedAt = new Date().toISOString(); - const { outputDir, homeDir, requestedBundlePath } = buildRequestedPaths( - metadata, - provider.id, - evalCase, - condition, - ); - await mkdir(outputDir, { recursive: true }); - await mkdir(homeDir, { recursive: true }); - - try { - const systemPromptContext = await buildSystemPromptContext(condition); - const requestCase = DogfoodEvalCaseSchema.parse({ - ...evalCase, - prompt: buildPrompt( - { ...evalCase, conditions: [condition] }, - requestedBundlePath, - systemPromptContext, - ), - bundlePath: requestedBundlePath, - conditions: [condition], - }) as DogfoodEvalCase; - - try { - const agentResult = await provider.invokeAgentMode({ - runId: metadata.runId, - providerId: provider.id, - condition, - trial: 1, - ...(requestedModelId === undefined - ? {} - : { modelId: requestedModelId }), - cwd: repoRoot, - homeDir, - outputDir, - env: { - AGENT_TTY_HOME: homeDir, - EVAL_OUTPUT_DIR: outputDir, - EVAL_REQUESTED_BUNDLE_DIR: requestedBundlePath, - EVAL_FIXTURE: evalCase.fixture ?? '', - ...systemPromptContext.env, - }, - evalCase: requestCase, - }); - - const transcript = await resolveTranscriptText(agentResult); - const bundlePath = await resolveBundlePath( - agentResult.bundlePath, - requestedBundlePath, - ); - const reportText = await resolveReportText( - bundlePath, - agentResult.normalized.finalText, - ); - const dogfoodScore = await scoreDogfoodRun( - bundlePath ?? requestedBundlePath, - reportText, - transcript, - evalCase, - ); - const reportRequirementScore = scoreReportRequirements( - reportText ?? '', - evalCase.reportRequirements, - ); - const reportSections = evalCase.reportRequirements - .map((requirement) => requirement.section) - .filter( - (section): section is string => - typeof section === 'string' && section.trim().length > 0, - ); - const baseReportCompleteness = scoreReportCompleteness( - reportText ?? '', - reportSections, - ); - const reportCompleteness = { - ...baseReportCompleteness, - sectionsExpected: reportRequirementScore.details.length, - sectionsFound: reportRequirementScore.details.filter( - (detail) => detail.found, - ).length, - score: clampUnitInterval( - (baseReportCompleteness.score + reportRequirementScore.score) / 2, - ), - details: reportRequirementScore.details.map((detail) => ({ - section: detail.section, - found: detail.found, - required: - evalCase.reportRequirements.find( - (requirement) => - (requirement.section ?? requirement.id) === detail.section, - )?.required ?? true, - })), - missingSections: reportRequirementScore.details - .filter((detail) => !detail.found) - .map((detail) => detail.section), - }; - const bundleCompleteness = await scoreBundleCompleteness( - bundlePath ?? requestedBundlePath, - evalCase.validationProfile, - ); - const evidenceQuality = await scoreEvidenceQuality( - bundlePath ?? requestedBundlePath, - ); - const workflowChecks = - requestCase.workflowChecks.length === 0 - ? [] - : checkWorkflow(transcript, requestCase.workflowChecks); - const scannableTranscript = buildScannableTranscript( - agentResult.normalized, - ); - const antiPatternFindings = detectAntiPatterns( - scannableTranscript, - requestCase.antiPatterns, - ); - const artifactManifestPath = - await resolveArtifactManifestPath(bundlePath); - const blockingAntiPattern = antiPatternFindings.some( - (finding) => finding.severity === 'error', - ); - const missingRequiredReportSection = - reportRequirementScore.details.some((detail) => !detail.found); - const missingRequiredWorkflow = workflowChecks.some( - (check) => !check.passed, - ); - const ok = - agentResult.ok && - !blockingAntiPattern && - !missingRequiredReportSection && - !missingRequiredWorkflow && - dogfoodScore.overallScore >= 0.6; - const completedAt = agentResult.completedAt; - const result: EvalResult = EvalResultSchema.parse({ - runId: metadata.runId, - providerId: provider.id, - ...(agentResult.runtime.version === undefined - ? {} - : { providerVersion: agentResult.runtime.version }), - ...(requestedModelId === undefined - ? {} - : { modelId: requestedModelId }), - lane: 'dogfood', - caseId: evalCase.id, - category: evalCase.category, - condition, - expectedSkill: evalCase.expectedSkill, - trial: 1, - ok, - score: { - total: dogfoodScore.overallScore, - maxPossible: 1, - items: buildDogfoodBreakdownItems(dogfoodScore), - }, - workflowChecks, - antiPatternFindings, - bundleCompleteness, - reportCompleteness, - evidenceQuality, - ...(agentResult.transcriptPath === undefined - ? {} - : { transcriptPath: agentResult.transcriptPath }), - ...(bundlePath === undefined ? {} : { bundlePath }), - ...(artifactManifestPath === undefined - ? {} - : { artifactManifestPath }), - ...(agentResult.eventLogPath === undefined - ? {} - : { eventLogPath: agentResult.eventLogPath }), - normalizedOutput: agentResult.normalized, - ...(agentResult.errorClass === undefined - ? {} - : { errorClass: agentResult.errorClass }), - ...(agentResult.errorMessage === undefined - ? {} - : { errorMessage: agentResult.errorMessage }), - startedAt: agentResult.startedAt, - completedAt, - durationMs: agentResult.durationMs, - }) as EvalResult; - - results.push(result); - } catch (error) { - const completedAt = new Date().toISOString(); - const errorMessage = - error instanceof Error ? error.message : String(error); - const errorClass = - error instanceof Error && error.name.length > 0 - ? error.name - : 'Error'; - const result: EvalResult = EvalResultSchema.parse({ - runId: metadata.runId, - providerId: provider.id, - ...(detectedRuntime.version === undefined - ? {} - : { providerVersion: detectedRuntime.version }), - ...(requestedModelId === undefined - ? {} - : { modelId: requestedModelId }), - lane: 'dogfood', - caseId: evalCase.id, - category: evalCase.category, - condition, - expectedSkill: evalCase.expectedSkill, - trial: 1, - ok: false, - score: { - total: 0, - maxPossible: 1, - items: buildDogfoodBreakdownItems({ - bundleCompleteness: 0, - reportCompleteness: 0, - evidenceQuality: 0, - taxonomyUsage: 0, - reproducibility: 0, - }), - }, - workflowChecks: [], - antiPatternFindings: [], - normalizedOutput: EMPTY_NORMALIZED_OUTPUT, - errorClass, - errorMessage, - startedAt, - completedAt, - durationMs: Math.max( - 0, - Date.parse(completedAt) - Date.parse(startedAt), - ), - }) as EvalResult; - - results.push(result); - } - } finally { - try { - await rm(homeDir, { recursive: true, force: true }); - } catch { - // Best-effort cleanup; preserve the eval result when temp-home cleanup fails. - } - try { - await rm(outputDir, { recursive: true, force: true }); - } catch { - // Best-effort cleanup; preserve the eval result when temp-dir cleanup fails. - } - } + return settlements.map((settlement) => { + if (settlement.status === 'fulfilled') { + return settlement.value; } - } - return results; + return buildFailedDogfoodEvalResult({ + metadata, + providerId: provider.id, + providerVersion: ctx.detectedRuntime.version, + requestedModelId: ctx.requestedModelId, + workItem: settlement.item, + error: settlement.reason, + }); + }); } From 2eb77513bb522307ba7d05494915a5576777bb50 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 10:09:06 +0000 Subject: [PATCH 07/18] feat(evals): add --concurrency CLI option and align result ordering --- evals/lib/reporting.ts | 4 ++-- evals/run.ts | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/evals/lib/reporting.ts b/evals/lib/reporting.ts index 188d9299..5c110248 100644 --- a/evals/lib/reporting.ts +++ b/evals/lib/reporting.ts @@ -772,11 +772,11 @@ function summarizeAntiPatterns(results: EvalResult[]): AntiPatternSummaryRow[] { function sortResults(results: EvalResult[]): EvalResult[] { return [...results].sort( (left, right) => - compareStrings(left.providerId, right.providerId) || compareLane(left.lane, right.lane) || - compareCondition(left.condition, right.condition) || compareStrings(left.caseId, right.caseId) || + compareCondition(left.condition, right.condition) || left.trial - right.trial || + compareStrings(left.providerId, right.providerId) || compareStrings(left.runId, right.runId) || compareStrings(left.startedAt, right.startedAt) || compareStrings(left.completedAt, right.completedAt), diff --git a/evals/run.ts b/evals/run.ts index ee3ceb54..4084eac2 100644 --- a/evals/run.ts +++ b/evals/run.ts @@ -54,6 +54,7 @@ const HELP_TEXT = [ ' --json Print JSON summary only', ' --verbose Print verbose progress logs to stderr', ' --dry-run List cases that would run without invoking providers', + ' --concurrency Maximum work items to run concurrently per lane. Default: 1', ' --help Show this help text', '', 'Examples:', @@ -75,6 +76,7 @@ interface CliOptions { condition?: string; caseIds: string[]; outputDir?: string; + concurrency?: string; json: boolean; verbose: boolean; dryRun: boolean; @@ -89,6 +91,7 @@ interface ResolvedCliOptions { requestedCondition: string; caseIds: string[]; outputBaseDir: string; + concurrency: number; json: boolean; verbose: boolean; dryRun: boolean; @@ -240,6 +243,21 @@ function parseCliArgs(argumentsList: readonly string[]): CliOptions { options.dryRun = true; continue; } + if (argument === '--concurrency' || argument.startsWith('--concurrency=')) { + const parsed = parseOptionValue( + argument, + '--concurrency', + argumentsList, + index, + ); + invariant( + options.concurrency === undefined, + '--concurrency may only be set once', + ); + options.concurrency = parsed.value; + index = parsed.nextIndex; + continue; + } if (argument === '--provider' || argument.startsWith('--provider=')) { const parsed = parseOptionValue( argument, @@ -407,6 +425,18 @@ function resolveRequestedCaseIds(caseIds: readonly string[]): string[] { return resolvedCaseIds; } +function resolveConcurrency(value: string | undefined): number { + if (value === undefined) { + return 1; + } + const parsed = Number(value); + invariant( + Number.isInteger(parsed) && parsed > 0, + `--concurrency must be a positive integer, got: ${value}`, + ); + return parsed; +} + function resolveOutputBaseDir( repoRoot: string, outputDir: string | undefined, @@ -541,6 +571,7 @@ function buildResolvedOptions( const requestedConditions = resolveRequestedConditions(options.condition); const caseIds = resolveRequestedCaseIds(options.caseIds); const outputBaseDir = resolveOutputBaseDir(repoRoot, options.outputDir); + const concurrency = resolveConcurrency(options.concurrency); const selection = buildCaseSelections( providerId, requestedLanes, @@ -556,6 +587,7 @@ function buildResolvedOptions( requestedCondition: options.condition ?? 'all', caseIds, outputBaseDir, + concurrency, json: options.json, verbose: options.verbose, dryRun: options.dryRun, @@ -868,16 +900,19 @@ async function runLane( return runPromptLane(provider, metadata, { conditions: options.activeConditions, caseFilter, + concurrency: options.concurrency, }); case 'execution': return runExecutionLane(provider, metadata, { conditions: options.activeConditions, caseFilter, + concurrency: options.concurrency, }); case 'dogfood': return runDogfoodLane(provider, metadata, { conditions: options.activeConditions, caseFilter, + concurrency: options.concurrency, }); default: return lane satisfies never; From a921c67867a8ecfc2594910bed16c1a068d116f1 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 10:11:22 +0000 Subject: [PATCH 08/18] test(evals): add lane runner enumeration unit tests --- test/unit/evals/dogfoodRunner.test.ts | 51 +++++ test/unit/evals/executionRunner.test.ts | 51 +++++ test/unit/evals/promptRunner.test.ts | 242 ++++++++++++++++++++++++ 3 files changed, 344 insertions(+) create mode 100644 test/unit/evals/dogfoodRunner.test.ts create mode 100644 test/unit/evals/executionRunner.test.ts create mode 100644 test/unit/evals/promptRunner.test.ts diff --git a/test/unit/evals/dogfoodRunner.test.ts b/test/unit/evals/dogfoodRunner.test.ts new file mode 100644 index 00000000..441baef5 --- /dev/null +++ b/test/unit/evals/dogfoodRunner.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest'; + +import { SKILL_CONDITIONS } from '../../../evals/lib/matrix.js'; +import { enumerateDogfoodWorkItems } from '../../../evals/dogfood/runner.js'; + +const DOGFOOD_CASE_ID = 'exploratory-qa'; + +describe('enumerateDogfoodWorkItems', () => { + it('returns unique dogfood work items with stable keys', () => { + const items = enumerateDogfoodWorkItems(); + const seenKeys = new Set(); + + expect(items.length).toBeGreaterThan(0); + + for (const item of items) { + expect(item.lane).toBe('dogfood'); + expect(item.caseId.length).toBeGreaterThan(0); + expect(SKILL_CONDITIONS).toContain(item.condition); + expect(item.trial).toBe(1); + expect(item.key.length).toBeGreaterThan(0); + expect(item.key).toBe(`dogfood:${item.caseId}:${item.condition}:1`); + expect(seenKeys.has(item.key)).toBe(false); + seenKeys.add(item.key); + } + + expect(seenKeys.size).toBe(items.length); + }); + + it('filters dogfood work items by case id', () => { + const items = enumerateDogfoodWorkItems({ caseFilter: [DOGFOOD_CASE_ID] }); + + expect(items.length).toBeGreaterThan(0); + expect(items.every((item) => item.caseId === DOGFOOD_CASE_ID)).toBe(true); + }); + + it('filters dogfood work items by condition', () => { + const items = enumerateDogfoodWorkItems({ + caseFilter: [DOGFOOD_CASE_ID], + conditions: ['none'], + }); + + expect(items).toHaveLength(1); + expect(items[0]).toMatchObject({ + lane: 'dogfood', + caseId: DOGFOOD_CASE_ID, + condition: 'none', + trial: 1, + key: 'dogfood:exploratory-qa:none:1', + }); + }); +}); diff --git a/test/unit/evals/executionRunner.test.ts b/test/unit/evals/executionRunner.test.ts new file mode 100644 index 00000000..db73274d --- /dev/null +++ b/test/unit/evals/executionRunner.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest'; + +import { SKILL_CONDITIONS } from '../../../evals/lib/matrix.js'; +import { enumerateExecutionWorkItems } from '../../../evals/execution/runner.js'; + +const EXECUTION_CASE_ID = 'hello-prompt'; + +describe('enumerateExecutionWorkItems', () => { + it('returns unique execution work items with stable keys', () => { + const items = enumerateExecutionWorkItems(); + const seenKeys = new Set(); + + expect(items.length).toBeGreaterThan(0); + + for (const item of items) { + expect(item.lane).toBe('execution'); + expect(item.caseId.length).toBeGreaterThan(0); + expect(SKILL_CONDITIONS).toContain(item.condition); + expect(item.trial).toBe(1); + expect(item.key.length).toBeGreaterThan(0); + expect(item.key).toBe(`execution:${item.caseId}:${item.condition}:1`); + expect(seenKeys.has(item.key)).toBe(false); + seenKeys.add(item.key); + } + + expect(seenKeys.size).toBe(items.length); + }); + + it('filters execution work items by case id', () => { + const items = enumerateExecutionWorkItems({ caseFilter: [EXECUTION_CASE_ID] }); + + expect(items.length).toBeGreaterThan(0); + expect(items.every((item) => item.caseId === EXECUTION_CASE_ID)).toBe(true); + }); + + it('filters execution work items by condition', () => { + const items = enumerateExecutionWorkItems({ + caseFilter: [EXECUTION_CASE_ID], + conditions: ['none'], + }); + + expect(items).toHaveLength(1); + expect(items[0]).toMatchObject({ + lane: 'execution', + caseId: EXECUTION_CASE_ID, + condition: 'none', + trial: 1, + key: 'execution:hello-prompt:none:1', + }); + }); +}); diff --git a/test/unit/evals/promptRunner.test.ts b/test/unit/evals/promptRunner.test.ts new file mode 100644 index 00000000..f462b429 --- /dev/null +++ b/test/unit/evals/promptRunner.test.ts @@ -0,0 +1,242 @@ +import { describe, expect, it } from 'vitest'; + +import { SKILL_CONDITIONS } from '../../../evals/lib/matrix.js'; +import { + enumeratePromptWorkItems, + executePromptWorkItem, +} from '../../../evals/prompt/runner.js'; +import type { + NormalizedProviderOutput, + ProviderPromptRequest, + ProviderPromptResult, + ProviderRuntimeInfo, + RunMetadata, + SkillCondition, +} from '../../../evals/lib/types.js'; +import type { EvalProvider } from '../../../evals/providers/base.js'; + +const ALL_CONDITIONS: SkillCondition[] = [...SKILL_CONDITIONS]; +const PROMPT_CASE_FILTER = ['session-creation', 'pure-reasoning']; +const PROMPT_TEST_CASE_ID = 'session-creation'; +const PROMPT_RUNTIME: ProviderRuntimeInfo = { + providerId: 'stub', + available: true, + detectedAt: '2026-01-01T00:00:00.000Z', + version: 'test-provider', + defaultModelId: 'test-model', + capabilities: { + supportsDetect: true, + supportsPlanMode: true, + supportsAgentMode: false, + supportsStreaming: false, + supportsToolCalls: false, + supportsTranscriptCapture: false, + }, + notes: ['unit test runtime'], +}; + +function createRunMetadata(overrides: Partial = {}): RunMetadata { + return { + runId: 'test-run', + createdAt: '2026-01-01T00:00:00.000Z', + repoRoot: '/tmp/test-repo', + providers: + overrides.providers === undefined ? ['stub'] : [...overrides.providers], + models: overrides.models === undefined ? [] : [...overrides.models], + lanes: overrides.lanes === undefined ? ['prompt'] : [...overrides.lanes], + conditions: + overrides.conditions === undefined + ? [...ALL_CONDITIONS] + : [...overrides.conditions], + totalTrials: overrides.totalTrials ?? 2, + notes: overrides.notes === undefined ? [] : [...overrides.notes], + ...(overrides.runId === undefined ? {} : { runId: overrides.runId }), + ...(overrides.createdAt === undefined + ? {} + : { createdAt: overrides.createdAt }), + ...(overrides.repoRoot === undefined ? {} : { repoRoot: overrides.repoRoot }), + }; +} + +function createNormalizedOutput(finalText: string): NormalizedProviderOutput { + return { + finalText, + messages: finalText.length === 0 ? [] : [finalText], + referencedSkills: [], + toolCalls: [], + }; +} + +function createPromptResult( + request: ProviderPromptRequest, + finalText: string, +): ProviderPromptResult { + return { + request, + runtime: PROMPT_RUNTIME, + ok: true, + exitCode: 0, + signal: null, + startedAt: '2026-01-01T00:00:01.000Z', + completedAt: '2026-01-01T00:00:02.000Z', + durationMs: 1000, + rawStdout: finalText, + rawStderr: '', + normalized: createNormalizedOutput(finalText), + }; +} + +describe('enumeratePromptWorkItems', () => { + it('returns unique prompt work items with stable keys', async () => { + const metadata = createRunMetadata(); + const items = await enumeratePromptWorkItems(metadata); + const seenKeys = new Set(); + + expect(items.length).toBeGreaterThan(0); + + for (const item of items) { + expect(item.lane).toBe('prompt'); + expect(item.caseId.length).toBeGreaterThan(0); + expect(SKILL_CONDITIONS).toContain(item.condition); + expect(Number.isInteger(item.trial)).toBe(true); + expect(item.trial).toBeGreaterThanOrEqual(1); + expect(item.key.length).toBeGreaterThan(0); + expect(item.key).toBe( + `prompt:${item.caseId}:${item.condition}:${String(item.trial)}`, + ); + expect(seenKeys.has(item.key)).toBe(false); + seenKeys.add(item.key); + } + + expect(seenKeys.size).toBe(items.length); + }); + + it('filters prompt work items by case id', async () => { + const metadata = createRunMetadata({ totalTrials: 1 }); + const items = await enumeratePromptWorkItems(metadata, { + caseFilter: PROMPT_CASE_FILTER, + }); + + expect(items.length).toBeGreaterThan(0); + expect([...new Set(items.map((item) => item.caseId))]).toEqual( + PROMPT_CASE_FILTER, + ); + for (const item of items) { + expect(PROMPT_CASE_FILTER.includes(item.caseId)).toBe(true); + } + }); + + it('filters prompt work items by condition', async () => { + const requestedConditions: SkillCondition[] = ['none', 'stale']; + const items = await enumeratePromptWorkItems(createRunMetadata({ totalTrials: 1 }), { + caseFilter: [PROMPT_TEST_CASE_ID], + conditions: requestedConditions, + }); + + expect(items).toHaveLength(requestedConditions.length); + expect(items.map((item) => item.condition)).toEqual(requestedConditions); + for (const item of items) { + expect(requestedConditions).toContain(item.condition); + } + }); +}); + +describe('executePromptWorkItem', () => { + it('builds a prompt request and returns the provider result payload', async () => { + const metadata = createRunMetadata({ conditions: ['none'], totalTrials: 1 }); + const [workItem] = await enumeratePromptWorkItems(metadata, { + caseFilter: [PROMPT_TEST_CASE_ID], + conditions: ['none'], + }); + const responseText = 'Use agent-tty create, wait, and snapshot to validate the task.'; + let receivedRequest: ProviderPromptRequest | undefined; + + expect(workItem).toBeDefined(); + if (workItem === undefined) { + throw new Error('Expected a prompt work item'); + } + + const provider = { + id: 'stub', + detect: async () => PROMPT_RUNTIME, + invokePlanMode: async (request: ProviderPromptRequest) => { + receivedRequest = request; + return createPromptResult(request, responseText); + }, + invokeAgentMode: async () => { + throw new Error('invokeAgentMode should not be called in prompt tests'); + }, + parse: (raw: string) => createNormalizedOutput(raw), + } satisfies EvalProvider; + + const result = await executePromptWorkItem(provider, metadata, workItem); + + expect(receivedRequest).toBeDefined(); + expect(receivedRequest).toMatchObject({ + runId: metadata.runId, + providerId: provider.id, + condition: workItem.condition, + trial: workItem.trial, + evalCase: { + id: workItem.caseId, + }, + }); + expect(receivedRequest?.evalCase.context).toContain(workItem.systemPrompt); + expect(result).toMatchObject({ + runId: metadata.runId, + providerId: provider.id, + lane: 'prompt', + caseId: workItem.caseId, + condition: workItem.condition, + trial: workItem.trial, + startedAt: '2026-01-01T00:00:01.000Z', + completedAt: '2026-01-01T00:00:02.000Z', + durationMs: 1000, + }); + expect(result.normalizedOutput.finalText).toBe(responseText); + expect(result.errorClass).toBeUndefined(); + expect(result.errorMessage).toBeUndefined(); + }); + + it('converts provider errors into failed eval results', async () => { + const metadata = createRunMetadata({ conditions: ['none'], totalTrials: 1 }); + const [workItem] = await enumeratePromptWorkItems(metadata, { + caseFilter: [PROMPT_TEST_CASE_ID], + conditions: ['none'], + }); + + expect(workItem).toBeDefined(); + if (workItem === undefined) { + throw new Error('Expected a prompt work item'); + } + + const provider = { + id: 'stub', + detect: async () => PROMPT_RUNTIME, + invokePlanMode: async () => { + throw new Error('prompt boom'); + }, + invokeAgentMode: async () => { + throw new Error('invokeAgentMode should not be called in prompt tests'); + }, + parse: (raw: string) => createNormalizedOutput(raw), + } satisfies EvalProvider; + + const result = await executePromptWorkItem(provider, metadata, workItem); + + expect(result).toMatchObject({ + runId: metadata.runId, + providerId: provider.id, + lane: 'prompt', + caseId: workItem.caseId, + condition: workItem.condition, + trial: workItem.trial, + ok: false, + errorClass: 'Error', + errorMessage: 'prompt boom', + }); + expect(result.normalizedOutput.finalText).toBe(''); + expect(result.normalizedOutput.messages).toEqual(['prompt boom']); + expect(result.durationMs).toBeGreaterThanOrEqual(0); + }); +}); From fe5f3b54cb9497ccb4b1d2cbf78c32b71db39f12 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 10:17:13 +0000 Subject: [PATCH 09/18] style(evals): fix formatting and lint issues --- evals/execution/runner.ts | 15 +++++-- evals/lib/scheduler.ts | 4 +- evals/lib/types.ts | 5 ++- test/unit/evals/executionRunner.test.ts | 4 +- test/unit/evals/promptRunner.test.ts | 54 +++++++++++++++---------- test/unit/evals/scheduler.test.ts | 18 +++++---- 6 files changed, 66 insertions(+), 34 deletions(-) diff --git a/evals/execution/runner.ts b/evals/execution/runner.ts index c1debc34..8beea67c 100644 --- a/evals/execution/runner.ts +++ b/evals/execution/runner.ts @@ -1,4 +1,11 @@ -import { mkdtemp, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { + mkdtemp, + readdir, + readFile, + rm, + stat, + writeFile, +} from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; @@ -1005,7 +1012,8 @@ async function runSingleExecutionCase( } catch (error) { const completedAt = new Date().toISOString(); const durationMs = Math.max(0, Date.now() - invocationStartedMs); - const errorMessage = error instanceof Error ? error.message : String(error); + const errorMessage = + error instanceof Error ? error.message : String(error); const transcript = error instanceof Error && error.stack !== undefined ? `${error.name}: ${errorMessage}\n${error.stack}` @@ -1099,7 +1107,8 @@ export function enumerateExecutionWorkItems(options?: { return evalCase.conditions .filter( (condition) => - requestedConditions === undefined || requestedConditions.has(condition), + requestedConditions === undefined || + requestedConditions.has(condition), ) .map((condition) => buildExecutionWorkItem(evalCase, condition)); }); diff --git a/evals/lib/scheduler.ts b/evals/lib/scheduler.ts index 906be21c..3913e158 100644 --- a/evals/lib/scheduler.ts +++ b/evals/lib/scheduler.ts @@ -26,7 +26,9 @@ export async function runScheduled( return []; } - const settlements: Array | undefined> = new Array(items.length); + const settlements: Array | undefined> = new Array( + items.length, + ); const workerCount = Math.min(options.concurrency, items.length); const logLine = options.logLine; let nextIndex = 0; diff --git a/evals/lib/types.ts b/evals/lib/types.ts index 14ec1328..74681843 100644 --- a/evals/lib/types.ts +++ b/evals/lib/types.ts @@ -25,7 +25,10 @@ export interface EvalWorkItemIdentity { /** Build a stable string key for a work item identity. */ export function buildWorkItemKey(identity: EvalWorkItemIdentity): string { - invariant(identity.caseId.length > 0, 'identity.caseId must be a non-empty string'); + invariant( + identity.caseId.length > 0, + 'identity.caseId must be a non-empty string', + ); invariant( Number.isInteger(identity.trial) && identity.trial > 0, 'identity.trial must be a positive integer', diff --git a/test/unit/evals/executionRunner.test.ts b/test/unit/evals/executionRunner.test.ts index db73274d..b6ea70f5 100644 --- a/test/unit/evals/executionRunner.test.ts +++ b/test/unit/evals/executionRunner.test.ts @@ -27,7 +27,9 @@ describe('enumerateExecutionWorkItems', () => { }); it('filters execution work items by case id', () => { - const items = enumerateExecutionWorkItems({ caseFilter: [EXECUTION_CASE_ID] }); + const items = enumerateExecutionWorkItems({ + caseFilter: [EXECUTION_CASE_ID], + }); expect(items.length).toBeGreaterThan(0); expect(items.every((item) => item.caseId === EXECUTION_CASE_ID)).toBe(true); diff --git a/test/unit/evals/promptRunner.test.ts b/test/unit/evals/promptRunner.test.ts index f462b429..cbfff1e2 100644 --- a/test/unit/evals/promptRunner.test.ts +++ b/test/unit/evals/promptRunner.test.ts @@ -54,7 +54,9 @@ function createRunMetadata(overrides: Partial = {}): RunMetadata { ...(overrides.createdAt === undefined ? {} : { createdAt: overrides.createdAt }), - ...(overrides.repoRoot === undefined ? {} : { repoRoot: overrides.repoRoot }), + ...(overrides.repoRoot === undefined + ? {} + : { repoRoot: overrides.repoRoot }), }; } @@ -128,10 +130,13 @@ describe('enumeratePromptWorkItems', () => { it('filters prompt work items by condition', async () => { const requestedConditions: SkillCondition[] = ['none', 'stale']; - const items = await enumeratePromptWorkItems(createRunMetadata({ totalTrials: 1 }), { - caseFilter: [PROMPT_TEST_CASE_ID], - conditions: requestedConditions, - }); + const items = await enumeratePromptWorkItems( + createRunMetadata({ totalTrials: 1 }), + { + caseFilter: [PROMPT_TEST_CASE_ID], + conditions: requestedConditions, + }, + ); expect(items).toHaveLength(requestedConditions.length); expect(items.map((item) => item.condition)).toEqual(requestedConditions); @@ -143,12 +148,16 @@ describe('enumeratePromptWorkItems', () => { describe('executePromptWorkItem', () => { it('builds a prompt request and returns the provider result payload', async () => { - const metadata = createRunMetadata({ conditions: ['none'], totalTrials: 1 }); + const metadata = createRunMetadata({ + conditions: ['none'], + totalTrials: 1, + }); const [workItem] = await enumeratePromptWorkItems(metadata, { caseFilter: [PROMPT_TEST_CASE_ID], conditions: ['none'], }); - const responseText = 'Use agent-tty create, wait, and snapshot to validate the task.'; + const responseText = + 'Use agent-tty create, wait, and snapshot to validate the task.'; let receivedRequest: ProviderPromptRequest | undefined; expect(workItem).toBeDefined(); @@ -158,14 +167,15 @@ describe('executePromptWorkItem', () => { const provider = { id: 'stub', - detect: async () => PROMPT_RUNTIME, - invokePlanMode: async (request: ProviderPromptRequest) => { + detect: () => Promise.resolve(PROMPT_RUNTIME), + invokePlanMode: (request: ProviderPromptRequest) => { receivedRequest = request; - return createPromptResult(request, responseText); - }, - invokeAgentMode: async () => { - throw new Error('invokeAgentMode should not be called in prompt tests'); + return Promise.resolve(createPromptResult(request, responseText)); }, + invokeAgentMode: () => + Promise.reject( + new Error('invokeAgentMode should not be called in prompt tests'), + ), parse: (raw: string) => createNormalizedOutput(raw), } satisfies EvalProvider; @@ -199,7 +209,10 @@ describe('executePromptWorkItem', () => { }); it('converts provider errors into failed eval results', async () => { - const metadata = createRunMetadata({ conditions: ['none'], totalTrials: 1 }); + const metadata = createRunMetadata({ + conditions: ['none'], + totalTrials: 1, + }); const [workItem] = await enumeratePromptWorkItems(metadata, { caseFilter: [PROMPT_TEST_CASE_ID], conditions: ['none'], @@ -212,13 +225,12 @@ describe('executePromptWorkItem', () => { const provider = { id: 'stub', - detect: async () => PROMPT_RUNTIME, - invokePlanMode: async () => { - throw new Error('prompt boom'); - }, - invokeAgentMode: async () => { - throw new Error('invokeAgentMode should not be called in prompt tests'); - }, + detect: () => Promise.resolve(PROMPT_RUNTIME), + invokePlanMode: () => Promise.reject(new Error('prompt boom')), + invokeAgentMode: () => + Promise.reject( + new Error('invokeAgentMode should not be called in prompt tests'), + ), parse: (raw: string) => createNormalizedOutput(raw), } satisfies EvalProvider; diff --git a/test/unit/evals/scheduler.test.ts b/test/unit/evals/scheduler.test.ts index a9586814..f2731bc6 100644 --- a/test/unit/evals/scheduler.test.ts +++ b/test/unit/evals/scheduler.test.ts @@ -36,7 +36,9 @@ function expectRejectedError( describe('runScheduled', () => { it('respects max concurrency', async () => { - const items = Array.from({ length: 6 }, (_, index) => ({ key: `item-${index}` })); + const items = Array.from({ length: 6 }, (_, index) => ({ + key: `item-${String(index)}`, + })); let inFlight = 0; let maxInFlight = 0; @@ -62,9 +64,9 @@ describe('runScheduled', () => { const results = await runScheduled( [], - async () => { + () => { called = true; - return 'unused'; + return Promise.resolve('unused'); }, { concurrency: 1 }, ); @@ -78,7 +80,7 @@ describe('runScheduled', () => { const results = await runScheduled( items, - (_item): Promise => { + (): Promise => { throw new Error('sync boom'); }, { concurrency: 1 }, @@ -177,11 +179,11 @@ describe('runScheduled', () => { await runScheduled( items, - async (item) => { + (item) => { if (item.key === 'beta') { throw new Error('beta failed'); } - return `${item.key} ok`; + return Promise.resolve(`${item.key} ok`); }, { concurrency: 1, @@ -205,7 +207,9 @@ describe('runScheduled', () => { for (const concurrency of [0, -1, 1.5]) { await expect( - runScheduled(items, async (item) => item.key, { concurrency }), + runScheduled(items, (item) => Promise.resolve(item.key), { + concurrency, + }), ).rejects.toThrow('options.concurrency must be a positive integer'); } }); From 74533c15893a4d9c61c76ab9c8ed0ed64b6d5295 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 10:29:04 +0000 Subject: [PATCH 10/18] test: add eval scheduler parity integration coverage --- .../integration/evals-parallelization.test.ts | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 test/integration/evals-parallelization.test.ts diff --git a/test/integration/evals-parallelization.test.ts b/test/integration/evals-parallelization.test.ts new file mode 100644 index 00000000..da9ec5b9 --- /dev/null +++ b/test/integration/evals-parallelization.test.ts @@ -0,0 +1,161 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, realpathSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import process from 'node:process'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +const DEFAULT_EVAL_TIMEOUT_MS = 30_000; +const TIMING_KEYS = new Set([ + 'runId', + 'createdAt', + 'startedAt', + 'completedAt', + 'durationMs', + 'detectedAt', +]); +const PATH_KEYS = new Set([ + 'repoRoot', + 'cwd', + 'homeDir', + 'outputDir', + 'commandPath', +]); +const VARIABLE_NOTE_PREFIXES = [ + 'output base dir: ', + 'provider command path: ', +] as const; + +type TestedLane = 'prompt' | 'execution'; + +interface EvalRunSummary { + ok: boolean; + providerId: string; + lanes: string[]; + conditions: string[]; + totalResults: number; + jsonReportPath: string; +} + +interface EvalRunOutput { + summary: EvalRunSummary; + report: unknown; +} + +function isVariableNote(value: unknown): value is string { + return ( + typeof value === 'string' && + VARIABLE_NOTE_PREFIXES.some((prefix) => value.startsWith(prefix)) + ); +} + +function isEphemeralKey(key: string): boolean { + return ( + TIMING_KEYS.has(key) || + PATH_KEYS.has(key) || + key.endsWith('Path') || + key.endsWith('Dir') + ); +} + +function normalizeEvalValue(value: unknown): unknown { + if (Array.isArray(value)) { + return value + .map((item) => normalizeEvalValue(item)) + .filter((item) => !isVariableNote(item)); + } + + if (value !== null && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value) + .filter(([key]) => !isEphemeralKey(key)) + .map(([key, item]) => [key, normalizeEvalValue(item)]), + ); + } + + return value; +} + +function runEvalLane( + lane: TestedLane, + concurrency: number, + testRoot: string, +): EvalRunOutput { + const outputDir = join(testRoot, `${lane}-${String(concurrency)}`); + const result = spawnSync( + process.execPath, + [ + '--import', + 'tsx', + './evals/run.ts', + '--provider', + 'stub', + '--lane', + lane, + '--condition', + 'none', + '--concurrency', + String(concurrency), + '--output', + outputDir, + '--json', + ], + { + cwd: process.cwd(), + encoding: 'utf8', + timeout: DEFAULT_EVAL_TIMEOUT_MS, + }, + ); + + expect(result.error).toBeUndefined(); + expect(result.stderr).toBe(''); + expect(result.status).not.toBeNull(); + + const summary = JSON.parse(result.stdout) as EvalRunSummary; + const exitCode = result.status; + if (exitCode === null) { + throw new Error('eval CLI exit code must not be null'); + } + + expect(exitCode).toBe(summary.ok ? 0 : 1); + expect(summary).toMatchObject({ + providerId: 'stub', + lanes: [lane], + conditions: ['none'], + }); + expect(summary.totalResults).toBeGreaterThan(0); + + const report = JSON.parse( + readFileSync(summary.jsonReportPath, 'utf8'), + ) as unknown; + return { summary, report }; +} + +let testRoot = ''; + +describe('eval scheduler parity', { timeout: DEFAULT_EVAL_TIMEOUT_MS }, () => { + beforeEach(() => { + // prettier-ignore + testRoot = realpathSync(mkdtempSync(join(tmpdir(), 'agent-tty-evals-parity-'))); + }); + + afterEach(() => { + rmSync(testRoot, { recursive: true, force: true }); + testRoot = ''; + }); + + for (const lane of ['prompt', 'execution'] as const) { + it(`matches ${lane} lane results between concurrency 1 and 4`, () => { + const serial = runEvalLane(lane, 1, testRoot); + const parallel = runEvalLane(lane, 4, testRoot); + + expect(normalizeEvalValue(serial.summary)).toEqual( + normalizeEvalValue(parallel.summary), + ); + expect(normalizeEvalValue(serial.report)).toEqual( + normalizeEvalValue(parallel.report), + ); + }); + } +}); From e8639041a6826a7b00994581d30c2cd2680a79b0 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 10:30:04 +0000 Subject: [PATCH 11/18] feat: run eval lanes concurrently --- evals/run.ts | 99 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 79 insertions(+), 20 deletions(-) diff --git a/evals/run.ts b/evals/run.ts index 4084eac2..b77b5922 100644 --- a/evals/run.ts +++ b/evals/run.ts @@ -14,13 +14,14 @@ import { } from './lib/matrix.js'; import { generateJsonReport, generateMarkdownReport } from './lib/reporting.js'; import { RunMetadataSchema } from './lib/schemas.js'; -import type { - EvalCase, - EvalLane, - EvalResult, - ProviderRuntimeInfo, - RunMetadata, - SkillCondition, +import { + buildWorkItemKey, + type EvalCase, + type EvalLane, + type EvalResult, + type ProviderRuntimeInfo, + type RunMetadata, + type SkillCondition, } from './lib/types.js'; import { getAllPromptCases, runPromptLane } from './prompt/runner.js'; import { createProvider, SUPPORTED_PROVIDER_IDS } from './providers/base.js'; @@ -919,6 +920,18 @@ async function runLane( } } +function recordLaneFailure( + lane: EvalLane, + message: string, + laneErrors: LaneErrorSummary[], + metadata: RunMetadata, + options: ResolvedCliOptions, +): void { + laneErrors.push({ lane, message }); + appendMetadataNote(metadata.notes, `lane ${lane} failed: ${message}`); + logVerbose(options, `Lane ${lane} failed: ${message}`); +} + export async function runEvalCli( argumentsList: readonly string[], ): Promise { @@ -951,23 +964,69 @@ export async function runEvalCli( const results: EvalResult[] = []; const laneErrors: LaneErrorSummary[] = []; - for (const lane of options.activeLanes) { - logVerbose(options, `Running ${lane} lane`); - try { - const laneResults = await runLane(lane, provider, metadata, options); - results.push(...laneResults); - logVerbose( - options, - `Completed ${lane} lane with ${String(laneResults.length)} result(s)`, + if (options.concurrency > 1) { + const lanePromises = options.activeLanes.map(async (lane) => { + try { + logVerbose(options, `Running ${lane} lane`); + const laneResults = await runLane(lane, provider, metadata, options); + logVerbose( + options, + `Completed ${lane} lane with ${String(laneResults.length)} result(s)`, + ); + return { lane, results: laneResults } as const; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { lane, error: message } as const; + } + }); + const settled = await Promise.allSettled(lanePromises); + for (const [index, outcome] of settled.entries()) { + const lane = options.activeLanes[index]; + invariant( + lane !== undefined, + `Missing active lane for settled outcome at index ${String(index)}`, ); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - laneErrors.push({ lane, message }); - appendMetadataNote(metadata.notes, `lane ${lane} failed: ${message}`); - logVerbose(options, `Lane ${lane} failed: ${message}`); + if (outcome.status === 'rejected') { + const message = + outcome.reason instanceof Error + ? outcome.reason.message + : String(outcome.reason); + recordLaneFailure(lane, message, laneErrors, metadata, options); + continue; + } + if ('error' in outcome.value) { + recordLaneFailure( + outcome.value.lane, + outcome.value.error, + laneErrors, + metadata, + options, + ); + continue; + } + results.push(...outcome.value.results); + } + } else { + for (const lane of options.activeLanes) { + logVerbose(options, `Running ${lane} lane`); + try { + const laneResults = await runLane(lane, provider, metadata, options); + results.push(...laneResults); + logVerbose( + options, + `Completed ${lane} lane with ${String(laneResults.length)} result(s)`, + ); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + recordLaneFailure(lane, message, laneErrors, metadata, options); + } } } + results.sort((left, right) => + buildWorkItemKey(left).localeCompare(buildWorkItemKey(right)), + ); + const comparisonMetrics = options.activeConditions.length > 1 ? computeComparisonMetrics(results) From 017476092e1dd63a3f472b6eaf333162d4666bcc Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 14:41:49 +0000 Subject: [PATCH 12/18] feat: add eval trials flag --- evals/dogfood/runner.ts | 36 ++++++-- evals/execution/runner.ts | 60 ++++++++++--- evals/run.ts | 32 ++++++- test/integration/evals-trials.test.ts | 115 ++++++++++++++++++++++++ test/unit/evals/dogfoodRunner.test.ts | 15 ++++ test/unit/evals/executionRunner.test.ts | 16 ++++ 6 files changed, 254 insertions(+), 20 deletions(-) create mode 100644 test/integration/evals-trials.test.ts diff --git a/evals/dogfood/runner.ts b/evals/dogfood/runner.ts index e1bbb004..1840dab6 100644 --- a/evals/dogfood/runner.ts +++ b/evals/dogfood/runner.ts @@ -66,6 +66,7 @@ const NO_CAPABILITIES: ProviderRuntimeInfo['capabilities'] = { supportsToolCalls: false, supportsTranscriptCapture: false, }; +const DEFAULT_TOTAL_TRIALS = 1; type DogfoodWorkItem = EvalWorkItemIdentity & ScheduledWorkItem & { @@ -501,11 +502,21 @@ function validateCaseFilter( return [...caseFilter]; } +function resolveTotalTrials(totalTrials: number | undefined): number { + const resolvedTotalTrials = totalTrials ?? DEFAULT_TOTAL_TRIALS; + invariant( + Number.isInteger(resolvedTotalTrials) && resolvedTotalTrials > 0, + `Dogfood totalTrials must be a positive integer, got: ${String(totalTrials)}`, + ); + return resolvedTotalTrials; +} + function buildRequestedPaths( metadata: RunMetadata, providerId: string, evalCase: DogfoodEvalCase, condition: SkillCondition, + trial: number, ): { outputDir: string; homeDir: string; requestedBundlePath: string } { const outputDir = resolve( tmpdir(), @@ -514,6 +525,7 @@ function buildRequestedPaths( providerId, evalCase.id, condition, + `trial-${String(trial)}`, ); const homeDir = join(outputDir, 'agent-tty-home'); const requestedBundlePath = join(outputDir, evalCase.bundlePath); @@ -563,12 +575,13 @@ export function getAllDogfoodCases(): DogfoodEvalCase[] { function buildDogfoodWorkItem( evalCase: DogfoodEvalCase, condition: SkillCondition, + trial: number, ): DogfoodWorkItem { const identity: EvalWorkItemIdentity = { lane: 'dogfood', caseId: evalCase.id, condition, - trial: 1, + trial, }; return { @@ -616,7 +629,7 @@ function buildFailedDogfoodEvalResult(params: { category: params.workItem.evalCase.category, condition: params.workItem.condition, expectedSkill: params.workItem.evalCase.expectedSkill, - trial: 1, + trial: params.workItem.trial, ok: false, score: { total: 0, @@ -643,8 +656,10 @@ function buildFailedDogfoodEvalResult(params: { export function enumerateDogfoodWorkItems(options?: { conditions?: SkillCondition[]; caseFilter?: string[]; + totalTrials?: number; }): DogfoodWorkItem[] { const selectedConditions = validateConditionList(options?.conditions); + const totalTrials = resolveTotalTrials(options?.totalTrials); const selectedCaseIds = validateCaseFilter(options?.caseFilter); const allCases = getAllDogfoodCases(); const cases = @@ -662,7 +677,10 @@ export function enumerateDogfoodWorkItems(options?: { ); for (const condition of caseConditions) { - items.push(buildDogfoodWorkItem(evalCase, condition)); + for (let trialIndex = 0; trialIndex < totalTrials; trialIndex += 1) { + const trial = trialIndex + 1; + items.push(buildDogfoodWorkItem(evalCase, condition, trial)); + } } } @@ -688,7 +706,10 @@ export async function executeDogfoodWorkItem( workItem.evalCase.conditions.includes(workItem.condition), 'Dogfood work item condition must be supported by the eval case', ); - invariant(workItem.trial === 1, 'Dogfood work item trial must be 1'); + invariant( + Number.isInteger(workItem.trial) && workItem.trial > 0, + 'Dogfood work item trial must be a positive integer', + ); const startedAt = new Date().toISOString(); const { outputDir, homeDir, requestedBundlePath } = buildRequestedPaths( @@ -696,6 +717,7 @@ export async function executeDogfoodWorkItem( provider.id, workItem.evalCase, workItem.condition, + workItem.trial, ); try { @@ -902,7 +924,11 @@ export async function runDogfoodLane( concurrency?: number; }, ): Promise { - const items = enumerateDogfoodWorkItems(options); + const totalTrials = resolveTotalTrials(metadata.totalTrials); + const items = enumerateDogfoodWorkItems({ + ...options, + totalTrials, + }); let detectedRuntime: ProviderRuntimeInfo; try { diff --git a/evals/execution/runner.ts b/evals/execution/runner.ts index 8beea67c..187539a0 100644 --- a/evals/execution/runner.ts +++ b/evals/execution/runner.ts @@ -69,7 +69,7 @@ const CASE_CATEGORY_EXPECTATIONS = { recovery: 1, } as const; const RUNNER_OUTPUT_PREFIX = 'agent-tty-execution-eval-'; -const DEFAULT_TRIAL = 1; +const DEFAULT_TOTAL_TRIALS = 1; type ExecutionWorkItem = EvalWorkItemIdentity & ScheduledWorkItem & { @@ -681,15 +681,25 @@ function normalizeRequestedConditions( return new Set(conditions); } +function resolveTotalTrials(totalTrials: number | undefined): number { + const resolvedTotalTrials = totalTrials ?? DEFAULT_TOTAL_TRIALS; + invariant( + Number.isInteger(resolvedTotalTrials) && resolvedTotalTrials > 0, + `Execution totalTrials must be a positive integer, got: ${String(totalTrials)}`, + ); + return resolvedTotalTrials; +} + function buildExecutionWorkItem( evalCase: ExecutionEvalCase, condition: SkillCondition, + trial: number, ): ExecutionWorkItem { const identity: EvalWorkItemIdentity = { lane: 'execution', caseId: evalCase.id, condition, - trial: DEFAULT_TRIAL, + trial, }; return { @@ -767,6 +777,7 @@ async function createEvalRequest( metadata: RunMetadata, evalCase: ExecutionEvalCase, condition: SkillCondition, + trial: number, homeDir: string, outputDir: string, runtime: ProviderRuntimeInfo | undefined, @@ -778,7 +789,7 @@ async function createEvalRequest( runId: metadata.runId, providerId: provider.id, condition, - trial: DEFAULT_TRIAL, + trial, cwd: resolve(metadata.repoRoot), homeDir, outputDir, @@ -799,6 +810,7 @@ function buildResult( provider: EvalProvider, evalCase: ExecutionEvalCase, condition: SkillCondition, + trial: number, runtime: ProviderRuntimeInfo | undefined, normalizedOutput: NormalizedProviderOutput, providerOk: boolean, @@ -850,7 +862,7 @@ function buildResult( category: evalCase.category, condition, expectedSkill: evalCase.expectedSkill, - trial: DEFAULT_TRIAL, + trial, ok: scored.ok, score: scored.breakdown, workflowChecks, @@ -887,6 +899,7 @@ async function runSingleExecutionCase( metadata: RunMetadata, evalCase: ExecutionEvalCase, condition: SkillCondition, + trial: number, runtime: ProviderRuntimeInfo | undefined, ): Promise { const homeDir = await createIsolatedEvalHome(); @@ -898,6 +911,7 @@ async function runSingleExecutionCase( metadata, evalCase, condition, + trial, homeDir, outputDir, runtime, @@ -920,6 +934,7 @@ async function runSingleExecutionCase( provider, evalCase, condition, + trial, runtime, normalizedOutput, false, @@ -991,6 +1006,7 @@ async function runSingleExecutionCase( provider, evalCase, condition, + trial, providerResult.runtime, providerResult.normalized, providerResult.ok, @@ -1031,6 +1047,7 @@ async function runSingleExecutionCase( provider, evalCase, condition, + trial, runtime, normalizedOutput, false, @@ -1079,8 +1096,10 @@ export function getAllExecutionCases(): ExecutionEvalCase[] { export function enumerateExecutionWorkItems(options?: { conditions?: SkillCondition[]; caseFilter?: string[]; + totalTrials?: number; }): ExecutionWorkItem[] { const requestedConditions = normalizeRequestedConditions(options?.conditions); + const totalTrials = resolveTotalTrials(options?.totalTrials); const requestedCaseIds = options?.caseFilter === undefined || options.caseFilter.length === 0 ? undefined @@ -1104,13 +1123,21 @@ export function enumerateExecutionWorkItems(options?: { return []; } - return evalCase.conditions - .filter( - (condition) => - requestedConditions === undefined || - requestedConditions.has(condition), - ) - .map((condition) => buildExecutionWorkItem(evalCase, condition)); + return evalCase.conditions.flatMap((condition) => { + if ( + requestedConditions !== undefined && + !requestedConditions.has(condition) + ) { + return []; + } + + const conditionItems: ExecutionWorkItem[] = []; + for (let trialIndex = 0; trialIndex < totalTrials; trialIndex += 1) { + const trial = trialIndex + 1; + conditionItems.push(buildExecutionWorkItem(evalCase, condition, trial)); + } + return conditionItems; + }); }); assertUniqueWorkItems(items); @@ -1123,11 +1150,16 @@ export async function executeExecutionWorkItem( workItem: ExecutionWorkItem, runtime: ProviderRuntimeInfo | undefined, ): Promise { + invariant( + Number.isInteger(workItem.trial) && workItem.trial > 0, + 'Execution work item trial must be a positive integer', + ); return runSingleExecutionCase( provider, metadata, workItem.evalCase, workItem.condition, + workItem.trial, runtime, ); } @@ -1145,7 +1177,11 @@ export async function runExecutionLane( concurrency?: number; } = {}, ): Promise { - const items = enumerateExecutionWorkItems(options); + const totalTrials = resolveTotalTrials(metadata.totalTrials); + const items = enumerateExecutionWorkItems({ + ...options, + totalTrials, + }); const runtime = await detectRuntime(provider); const settlements = await runScheduled( items, diff --git a/evals/run.ts b/evals/run.ts index b77b5922..3a67e58b 100644 --- a/evals/run.ts +++ b/evals/run.ts @@ -56,6 +56,7 @@ const HELP_TEXT = [ ' --verbose Print verbose progress logs to stderr', ' --dry-run List cases that would run without invoking providers', ' --concurrency Maximum work items to run concurrently per lane. Default: 1', + ' --trials Number of independent trials per case/condition. Default: 1', ' --help Show this help text', '', 'Examples:', @@ -78,6 +79,7 @@ interface CliOptions { caseIds: string[]; outputDir?: string; concurrency?: string; + trials?: string; json: boolean; verbose: boolean; dryRun: boolean; @@ -93,6 +95,7 @@ interface ResolvedCliOptions { caseIds: string[]; outputBaseDir: string; concurrency: number; + totalTrials: number; json: boolean; verbose: boolean; dryRun: boolean; @@ -259,6 +262,13 @@ function parseCliArgs(argumentsList: readonly string[]): CliOptions { index = parsed.nextIndex; continue; } + if (argument === '--trials' || argument.startsWith('--trials=')) { + const parsed = parseOptionValue(argument, '--trials', argumentsList, index); + invariant(options.trials === undefined, '--trials may only be set once'); + options.trials = parsed.value; + index = parsed.nextIndex; + continue; + } if (argument === '--provider' || argument.startsWith('--provider=')) { const parsed = parseOptionValue( argument, @@ -438,6 +448,18 @@ function resolveConcurrency(value: string | undefined): number { return parsed; } +function resolveTotalTrials(value: string | undefined): number { + if (value === undefined) { + return DEFAULT_TOTAL_TRIALS; + } + const parsed = Number(value); + invariant( + Number.isInteger(parsed) && parsed > 0, + `--trials must be a positive integer, got: ${value}`, + ); + return parsed; +} + function resolveOutputBaseDir( repoRoot: string, outputDir: string | undefined, @@ -474,6 +496,7 @@ function buildCaseSelections( lanes: readonly EvalLane[], conditions: readonly SkillCondition[], caseIds: readonly string[], + totalTrials: number, ): { activeConditions: SkillCondition[]; activeLanes: EvalLane[]; @@ -546,8 +569,8 @@ function buildCaseSelections( const activeConditions = SKILL_CONDITIONS.filter((condition) => matrixEntries.some((entry) => entry.condition === condition), ); - const totalInvocations = matrixEntries.reduce((count, entry) => { - return count + (entry.lane === 'prompt' ? DEFAULT_TOTAL_TRIALS : 1); + const totalInvocations = matrixEntries.reduce((count) => { + return count + totalTrials; }, 0); return { @@ -573,11 +596,13 @@ function buildResolvedOptions( const caseIds = resolveRequestedCaseIds(options.caseIds); const outputBaseDir = resolveOutputBaseDir(repoRoot, options.outputDir); const concurrency = resolveConcurrency(options.concurrency); + const totalTrials = resolveTotalTrials(options.trials); const selection = buildCaseSelections( providerId, requestedLanes, requestedConditions, caseIds, + totalTrials, ); return { @@ -589,6 +614,7 @@ function buildResolvedOptions( caseIds, outputBaseDir, concurrency, + totalTrials, json: options.json, verbose: options.verbose, dryRun: options.dryRun, @@ -774,7 +800,7 @@ function buildRunMetadata( models: selectedModelId === undefined ? [] : [selectedModelId], lanes: options.activeLanes, conditions: options.activeConditions, - totalTrials: DEFAULT_TOTAL_TRIALS, + totalTrials: options.totalTrials, notes: metadataNotes, }); const metadata: RunMetadata = parsedMetadata; diff --git a/test/integration/evals-trials.test.ts b/test/integration/evals-trials.test.ts new file mode 100644 index 00000000..c8f93449 --- /dev/null +++ b/test/integration/evals-trials.test.ts @@ -0,0 +1,115 @@ +import { spawnSync } from 'node:child_process'; +import process from 'node:process'; + +import { describe, expect, it } from 'vitest'; + +interface EvalDryRunSummary { + ok: boolean; + providerId: string; + lanes: string[]; + conditions: string[]; + totalInvocations: number; + dryRun: boolean; +} + +function runEvalCli(argumentsList: readonly string[]) { + const result = spawnSync( + process.execPath, + ['--import', 'tsx', './evals/run.ts', ...argumentsList], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + expect(result.error).toBeUndefined(); + expect(result.status).not.toBeNull(); + return result; +} + +describe('eval CLI trials flag', () => { + it('shows --trials in the help output', () => { + const result = runEvalCli(['--help']); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('--trials '); + }); + + it('multiplies prompt dry-run invocations by the requested trials', () => { + const result = runEvalCli([ + '--provider', + 'stub', + '--lane', + 'prompt', + '--condition', + 'none', + '--trials', + '3', + '--dry-run', + '--json', + ]); + + expect(result.status).toBe(0); + const summary = JSON.parse(result.stdout) as EvalDryRunSummary; + expect(summary).toMatchObject({ + providerId: 'stub', + lanes: ['prompt'], + conditions: ['none'], + totalInvocations: 72, + dryRun: true, + ok: true, + }); + }); + + it('multiplies execution dry-run invocations by the requested trials', () => { + const result = runEvalCli([ + '--provider', + 'stub', + '--lane', + 'execution', + '--condition', + 'none', + '--trials', + '2', + '--dry-run', + '--json', + ]); + + expect(result.status).toBe(0); + const summary = JSON.parse(result.stdout) as EvalDryRunSummary; + expect(summary).toMatchObject({ + providerId: 'stub', + lanes: ['execution'], + conditions: ['none'], + totalInvocations: 20, + dryRun: true, + ok: true, + }); + }); + + it('multiplies dogfood dry-run invocations by the requested trials', () => { + const result = runEvalCli([ + '--provider', + 'stub', + '--lane', + 'dogfood', + '--condition', + 'none', + '--trials', + '2', + '--dry-run', + '--json', + ]); + + expect(result.status).toBe(0); + const summary = JSON.parse(result.stdout) as EvalDryRunSummary; + expect(summary).toMatchObject({ + providerId: 'stub', + lanes: ['dogfood'], + conditions: ['none'], + totalInvocations: 12, + dryRun: true, + ok: true, + }); + }); +}); diff --git a/test/unit/evals/dogfoodRunner.test.ts b/test/unit/evals/dogfoodRunner.test.ts index 441baef5..c9536a05 100644 --- a/test/unit/evals/dogfoodRunner.test.ts +++ b/test/unit/evals/dogfoodRunner.test.ts @@ -48,4 +48,19 @@ describe('enumerateDogfoodWorkItems', () => { key: 'dogfood:exploratory-qa:none:1', }); }); + + it('enumerates multiple dogfood trials per case and condition', () => { + const items = enumerateDogfoodWorkItems({ + caseFilter: [DOGFOOD_CASE_ID], + conditions: ['none'], + totalTrials: 2, + }); + + expect(items).toHaveLength(2); + expect(items.map((item) => item.trial)).toEqual([1, 2]); + expect(items.map((item) => item.key)).toEqual([ + 'dogfood:exploratory-qa:none:1', + 'dogfood:exploratory-qa:none:2', + ]); + }); }); diff --git a/test/unit/evals/executionRunner.test.ts b/test/unit/evals/executionRunner.test.ts index b6ea70f5..25a53c63 100644 --- a/test/unit/evals/executionRunner.test.ts +++ b/test/unit/evals/executionRunner.test.ts @@ -50,4 +50,20 @@ describe('enumerateExecutionWorkItems', () => { key: 'execution:hello-prompt:none:1', }); }); + + it('enumerates multiple execution trials per case and condition', () => { + const items = enumerateExecutionWorkItems({ + caseFilter: [EXECUTION_CASE_ID], + conditions: ['none'], + totalTrials: 3, + }); + + expect(items).toHaveLength(3); + expect(items.map((item) => item.trial)).toEqual([1, 2, 3]); + expect(items.map((item) => item.key)).toEqual([ + 'execution:hello-prompt:none:1', + 'execution:hello-prompt:none:2', + 'execution:hello-prompt:none:3', + ]); + }); }); From 612cb4461bcd8843013d161126e709bf5c0a91bb Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 15:03:48 +0000 Subject: [PATCH 13/18] feat: add shared eval statistics helpers --- evals/lib/matrix.ts | 15 +- evals/lib/statistics.ts | 493 +++++++++++++++++++++++++++++ test/unit/evals/statistics.test.ts | 199 ++++++++++++ 3 files changed, 693 insertions(+), 14 deletions(-) create mode 100644 evals/lib/statistics.ts create mode 100644 test/unit/evals/statistics.test.ts diff --git a/evals/lib/matrix.ts b/evals/lib/matrix.ts index 39a47cb4..3668fe90 100644 --- a/evals/lib/matrix.ts +++ b/evals/lib/matrix.ts @@ -1,5 +1,6 @@ import { invariant } from '../../src/util/assert.js'; import { ComparisonMetricsSchema, MatrixEntrySchema } from './schemas.js'; +import { computeMean } from './statistics.js'; import type { ComparisonMetrics, EvalCase, @@ -119,20 +120,6 @@ function resolveConditionList(evalCase: EvalCase): readonly SkillCondition[] { return evalCase.conditions; } -function computeMean(values: readonly number[]): number { - if (values.length === 0) { - return 0; - } - - let total = 0; - for (const value of values) { - assertFiniteNumber(value, 'Mean input value'); - total += value; - } - - return total / values.length; -} - function collectScoreTotals( results: readonly EvalResult[] | undefined, ): number[] { diff --git a/evals/lib/statistics.ts b/evals/lib/statistics.ts new file mode 100644 index 00000000..1648a205 --- /dev/null +++ b/evals/lib/statistics.ts @@ -0,0 +1,493 @@ +import { invariant } from '../../src/util/assert.js'; + +const DEFAULT_CONFIDENCE = 0.95; +const DEFAULT_BOOTSTRAP_ITERATIONS = 10_000; +const DEFAULT_BOOTSTRAP_SEED = 42; +const DEFAULT_ALPHA = 1 - DEFAULT_CONFIDENCE; +const T_CRITICAL_95 = [ + Number.NaN, + 12.706204736432095, + 4.302652729696142, + 3.182446305284264, + 2.7764451051977987, + 2.5705818366147395, + 2.446911846863915, + 2.3646242510102993, + 2.306004135204166, + 2.2621571628540993, + 2.2281388519649385, + 2.200985160082949, + 2.178812829663418, + 2.160368656461013, + 2.1447866879169273, + 2.131449545559323, + 2.1199052992210112, + 2.109815577833181, + 2.10092204024096, + 2.093024054408263, + 2.0859634472658364, + 2.079613844727662, + 2.0738730679040147, + 2.068657610419041, + 2.0638985616280205, + 2.059538552753294, + 2.055529438642871, + 2.0518305164802833, + 2.048407141795244, + 2.045229642132703, +] as const; + +type Interval = { + lower: number; + upper: number; +}; + +function assertFiniteNumber( + value: unknown, + label: string, +): asserts value is number { + invariant( + typeof value === 'number' && Number.isFinite(value), + `${label} must be a finite number`, + ); +} + +function assertConfidence(confidence: number): void { + invariant( + Number.isFinite(confidence) && confidence > 0 && confidence < 1, + 'confidence must be between 0 and 1', + ); +} + +function assertPairedLengths( + baseline: readonly number[], + candidate: readonly number[], +): void { + invariant( + baseline.length === candidate.length, + 'baseline and candidate must have equal length', + ); +} + +function collectPairedDifferences( + baseline: readonly number[], + candidate: readonly number[], +): number[] { + assertPairedLengths(baseline, candidate); + + const differences = new Array(baseline.length); + for (let index = 0; index < baseline.length; index += 1) { + const baselineValue = baseline[index]; + const candidateValue = candidate[index]; + assertFiniteNumber( + baselineValue, + `baseline value at index ${String(index)}`, + ); + assertFiniteNumber( + candidateValue, + `candidate value at index ${String(index)}`, + ); + differences[index] = candidateValue - baselineValue; + } + + return differences; +} + +function buildPointInterval(value: number): Interval { + return { + lower: value, + upper: value, + }; +} + +function confidenceIntervalExcludesZero(ci: Interval): boolean { + return ci.lower > 0 || ci.upper < 0; +} + +function inverseStandardNormal(probability: number): number { + invariant( + probability > 0 && probability < 1, + 'normal quantile probability must be between 0 and 1', + ); + + const a1 = -39.69683028665376; + const a2 = 220.9460984245205; + const a3 = -275.9285104469687; + const a4 = 138.357751867269; + const a5 = -30.66479806614716; + const a6 = 2.506628277459239; + const b1 = -54.47609879822406; + const b2 = 161.5858368580409; + const b3 = -155.6989798598866; + const b4 = 66.80131188771972; + const b5 = -13.28068155288572; + const c1 = -0.007784894002430293; + const c2 = -0.3223964580411365; + const c3 = -2.400758277161838; + const c4 = -2.549732539343734; + const c5 = 4.374664141464968; + const c6 = 2.938163982698783; + const d1 = 0.007784695709041462; + const d2 = 0.3224671290700398; + const d3 = 2.445134137142996; + const d4 = 3.754408661907416; + const lowerRegionBoundary = 0.02425; + const upperRegionBoundary = 1 - lowerRegionBoundary; + + if (probability < lowerRegionBoundary) { + const q = Math.sqrt(-2 * Math.log(probability)); + return ( + (((((c1 * q + c2) * q + c3) * q + c4) * q + c5) * q + c6) / + ((((d1 * q + d2) * q + d3) * q + d4) * q + 1) + ); + } + + if (probability <= upperRegionBoundary) { + const q = probability - 0.5; + const r = q * q; + return ( + (((((a1 * r + a2) * r + a3) * r + a4) * r + a5) * r + a6) * q / + (((((b1 * r + b2) * r + b3) * r + b4) * r + b5) * r + 1) + ); + } + + const q = Math.sqrt(-2 * Math.log(1 - probability)); + return ( + -(((((c1 * q + c2) * q + c3) * q + c4) * q + c5) * q + c6) / + ((((d1 * q + d2) * q + d3) * q + d4) * q + 1) + ); +} + +function tCriticalValue(df: number, alpha: number): number { + invariant(Number.isInteger(df) && df >= 1, 'df must be a positive integer'); + invariant( + Number.isFinite(alpha) && alpha > 0 && alpha < 1, + 'alpha must be between 0 and 1', + ); + + if (Math.abs(alpha - DEFAULT_ALPHA) < 1e-9 && df < T_CRITICAL_95.length) { + const critical = T_CRITICAL_95[df]; + invariant( + critical !== undefined && Number.isFinite(critical), + `Missing t critical value for df ${String(df)}`, + ); + return critical; + } + + const z = zCriticalValue(alpha); + const z2 = z * z; + const z3 = z2 * z; + const z5 = z3 * z2; + const z7 = z5 * z2; + const df2 = df * df; + const df3 = df2 * df; + + return ( + z + + (z3 + z) / (4 * df) + + (5 * z5 + 16 * z3 + 3 * z) / (96 * df2) + + (3 * z7 + 19 * z5 + 17 * z3 - 15 * z) / (384 * df3) + ); +} + +function zCriticalValue(alpha: number): number { + invariant( + Number.isFinite(alpha) && alpha > 0 && alpha < 1, + 'alpha must be between 0 and 1', + ); + return inverseStandardNormal(1 - alpha / 2); +} + +function percentile(sortedValues: readonly number[], probability: number): number { + invariant(sortedValues.length > 0, 'percentile input must not be empty'); + invariant( + probability >= 0 && probability <= 1, + 'percentile probability must be between 0 and 1', + ); + + const lastIndex = sortedValues.length - 1; + const rawIndex = probability * lastIndex; + const lowerIndex = Math.floor(rawIndex); + const upperIndex = Math.ceil(rawIndex); + const lowerValue = sortedValues[lowerIndex]; + const upperValue = sortedValues[upperIndex]; + invariant(lowerValue !== undefined, 'percentile lower value must exist'); + invariant(upperValue !== undefined, 'percentile upper value must exist'); + + if (lowerIndex === upperIndex) { + return lowerValue; + } + + const weight = rawIndex - lowerIndex; + return lowerValue + (upperValue - lowerValue) * weight; +} + +export function computeMean(values: readonly number[]): number { + if (values.length === 0) { + return 0; + } + + let total = 0; + for (let index = 0; index < values.length; index += 1) { + const value = values[index]; + assertFiniteNumber(value, `mean value at index ${String(index)}`); + total += value; + } + + return total / values.length; +} + +export function computeStdDev( + values: readonly number[], + mean?: number, +): number { + if (values.length === 0) { + return 0; + } + + let total = 0; + for (let index = 0; index < values.length; index += 1) { + const value = values[index]; + assertFiniteNumber( + value, + `standard deviation value at index ${String(index)}`, + ); + total += value; + } + + if (mean !== undefined) { + assertFiniteNumber(mean, 'standard deviation mean'); + } + + if (values.length < 2) { + return 0; + } + + const resolvedMean = mean ?? total / values.length; + let squaredDeviationTotal = 0; + for (let index = 0; index < values.length; index += 1) { + const value = values[index]; + invariant( + value !== undefined, + `Missing standard deviation value at ${String(index)}`, + ); + const delta = value - resolvedMean; + squaredDeviationTotal += delta * delta; + } + + return Math.sqrt(squaredDeviationTotal / (values.length - 1)); +} + +export function computeConfidenceInterval( + values: readonly number[], + confidence = DEFAULT_CONFIDENCE, +): { mean: number; ci: Interval; n: number } { + assertConfidence(confidence); + + const mean = computeMean(values); + const n = values.length; + if (n < 2) { + return { + mean, + ci: buildPointInterval(mean), + n, + }; + } + + const stdDev = computeStdDev(values, mean); + if (stdDev === 0) { + return { + mean, + ci: buildPointInterval(mean), + n, + }; + } + + const alpha = 1 - confidence; + const standardError = stdDev / Math.sqrt(n); + const critical = n < 30 ? tCriticalValue(n - 1, alpha) : zCriticalValue(alpha); + const margin = critical * standardError; + + return { + mean, + ci: { + lower: mean - margin, + upper: mean + margin, + }, + n, + }; +} + +export function computePassRate( + results: readonly { ok: boolean }[], +): { rate: number; ci: Interval; n: number; passed: number } { + const n = results.length; + if (n === 0) { + return { + rate: 0, + ci: buildPointInterval(0), + n, + passed: 0, + }; + } + + let passed = 0; + for (let index = 0; index < results.length; index += 1) { + const result = results[index] as unknown; + invariant( + typeof result === 'object' && + result !== null && + 'ok' in result && + typeof result.ok === 'boolean', + `pass-rate result at index ${String(index)} must include a boolean ok flag`, + ); + if (result.ok) { + passed += 1; + } + } + + const rate = passed / n; + const z = zCriticalValue(DEFAULT_ALPHA); + const zSquared = z * z; + const denominator = 1 + zSquared / n; + const center = (rate + zSquared / (2 * n)) / denominator; + const margin = + (z * Math.sqrt((rate * (1 - rate) + zSquared / (4 * n)) / n)) / + denominator; + + return { + rate, + ci: { + lower: Math.max(0, center - margin), + upper: Math.min(1, center + margin), + }, + n, + passed, + }; +} + +export function computePairedDelta( + baseline: readonly number[], + candidate: readonly number[], +): { mean: number; ci: Interval; n: number; significant: boolean } { + const differences = collectPairedDifferences(baseline, candidate); + const summary = computeConfidenceInterval(differences); + + return { + ...summary, + significant: confidenceIntervalExcludesZero(summary.ci), + }; +} + +export function computeWinRate( + baseline: readonly number[], + candidate: readonly number[], +): { + wins: number; + losses: number; + ties: number; + n: number; + winRate: number; +} { + const differences = collectPairedDifferences(baseline, candidate); + let wins = 0; + let losses = 0; + let ties = 0; + + for (const difference of differences) { + if (difference > 0) { + wins += 1; + continue; + } + + if (difference < 0) { + losses += 1; + continue; + } + + ties += 1; + } + + return { + wins, + losses, + ties, + n: differences.length, + winRate: differences.length === 0 ? 0 : wins / differences.length, + }; +} + +export function bootstrapPairedCI( + baseline: readonly number[], + candidate: readonly number[], + options?: { + confidence?: number; + iterations?: number; + seed?: number; + }, +): { mean: number; ci: Interval; n: number; significant: boolean } { + const confidence = options?.confidence ?? DEFAULT_CONFIDENCE; + const iterations = options?.iterations ?? DEFAULT_BOOTSTRAP_ITERATIONS; + const seed = options?.seed ?? DEFAULT_BOOTSTRAP_SEED; + assertConfidence(confidence); + invariant( + Number.isInteger(iterations) && iterations >= 100, + 'iterations must be an integer greater than or equal to 100', + ); + + const differences = collectPairedDifferences(baseline, candidate); + const mean = computeMean(differences); + const n = differences.length; + const pointCi = buildPointInterval(mean); + if (n < 2) { + return { + mean, + ci: pointCi, + n, + significant: confidenceIntervalExcludesZero(pointCi), + }; + } + + const random = mulberry32(seed); + const bootstrapMeans = new Array(iterations); + for (let iteration = 0; iteration < iterations; iteration += 1) { + let sampledTotal = 0; + for (let index = 0; index < n; index += 1) { + const sampledIndex = Math.floor(random() * n); + const difference = differences[sampledIndex]; + invariant( + difference !== undefined, + `bootstrap difference at index ${String(sampledIndex)} must exist`, + ); + sampledTotal += difference; + } + bootstrapMeans[iteration] = sampledTotal / n; + } + + bootstrapMeans.sort((left, right) => left - right); + const alpha = 1 - confidence; + const ci = { + lower: percentile(bootstrapMeans, alpha / 2), + upper: percentile(bootstrapMeans, 1 - alpha / 2), + }; + + return { + mean, + ci, + n, + significant: confidenceIntervalExcludesZero(ci), + }; +} + +function mulberry32(seed: number): () => number { + assertFiniteNumber(seed, 'bootstrap seed'); + + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let value = state; + value = Math.imul(value ^ (value >>> 15), value | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return ((value ^ (value >>> 14)) >>> 0) / 4294967296; + }; +} diff --git a/test/unit/evals/statistics.test.ts b/test/unit/evals/statistics.test.ts new file mode 100644 index 00000000..93984b90 --- /dev/null +++ b/test/unit/evals/statistics.test.ts @@ -0,0 +1,199 @@ +import { describe, expect, it } from 'vitest'; + +import { + bootstrapPairedCI, + computeConfidenceInterval, + computeMean, + computePairedDelta, + computePassRate, + computeStdDev, + computeWinRate, +} from '../../../evals/lib/statistics.js'; + +function createPassResults(values: readonly boolean[]): { ok: boolean }[] { + return values.map((ok) => ({ ok })); +} + +describe('computeMean', () => { + it('returns 0 for empty input', () => { + expect(computeMean([])).toBe(0); + }); + + it('computes the arithmetic mean for finite values', () => { + expect(computeMean([1, 2, 3, 4])).toBe(2.5); + }); + + it('throws for non-finite input values', () => { + expect(() => computeMean([1, Number.NaN])).toThrow('finite number'); + }); +}); + +describe('computeStdDev', () => { + it('returns 0 for empty, singleton, and repeated values', () => { + expect(computeStdDev([])).toBe(0); + expect(computeStdDev([7])).toBe(0); + expect(computeStdDev([5, 5, 5])).toBe(0); + }); + + it('computes the sample standard deviation and accepts an explicit mean', () => { + expect(computeStdDev([1, 2, 3, 4])).toBeCloseTo(1.2909944487); + expect(computeStdDev([1, 2, 3, 4], 2.5)).toBeCloseTo(1.2909944487); + }); + + it('throws for non-finite input values', () => { + expect(() => computeStdDev([1, Number.POSITIVE_INFINITY])).toThrow( + 'finite number', + ); + }); +}); + +describe('computeConfidenceInterval', () => { + it('uses t critical values for small samples', () => { + const summary = computeConfidenceInterval([1, 2, 3, 4, 5]); + + expect(summary.mean).toBe(3); + expect(summary.n).toBe(5); + expect(summary.ci.lower).toBeCloseTo(1.0367568385); + expect(summary.ci.upper).toBeCloseTo(4.9632431615); + }); + + it('uses z critical values for samples with at least 30 observations', () => { + const summary = computeConfidenceInterval( + Array.from({ length: 30 }, (_, index) => index + 1), + ); + + expect(summary.mean).toBe(15.5); + expect(summary.n).toBe(30); + expect(summary.ci.lower).toBeCloseTo(12.3497986356); + expect(summary.ci.upper).toBeCloseTo(18.6502013644); + }); + + it('returns a point interval when fewer than two observations exist', () => { + expect(computeConfidenceInterval([])).toEqual({ + mean: 0, + ci: { lower: 0, upper: 0 }, + n: 0, + }); + expect(computeConfidenceInterval([8])).toEqual({ + mean: 8, + ci: { lower: 8, upper: 8 }, + n: 1, + }); + }); + + it('throws for invalid confidence values', () => { + expect(() => computeConfidenceInterval([1, 2], 1)).toThrow( + 'confidence must be between 0 and 1', + ); + }); +}); + +describe('computePassRate', () => { + it('returns stable defaults for empty input', () => { + expect(computePassRate([])).toEqual({ + rate: 0, + ci: { lower: 0, upper: 0 }, + n: 0, + passed: 0, + }); + }); + + it('computes the Wilson score interval at 95% confidence', () => { + const summary = computePassRate(createPassResults([true, true, false, true])); + + expect(summary.rate).toBe(0.75); + expect(summary.passed).toBe(3); + expect(summary.n).toBe(4); + expect(summary.ci.lower).toBeCloseTo(0.3006418423); + expect(summary.ci.upper).toBeCloseTo(0.9544127392); + }); + + it('throws when a result does not expose a boolean ok flag', () => { + expect(() => + computePassRate( + [{ ok: true }, { ok: 1 }] as unknown as readonly { ok: boolean }[], + ), + ).toThrow('boolean ok flag'); + }); +}); + +describe('computePairedDelta', () => { + it('marks a positive paired delta as significant when the CI excludes zero', () => { + expect(computePairedDelta([1, 2, 3, 4], [2, 3, 4, 5])).toEqual({ + mean: 1, + ci: { lower: 1, upper: 1 }, + n: 4, + significant: true, + }); + }); + + it('marks a paired delta as not significant when the CI crosses zero', () => { + const summary = computePairedDelta([1, 2, 3, 4], [2, 1, 4, 3]); + + expect(summary.mean).toBe(0); + expect(summary.significant).toBe(false); + expect(summary.ci.lower).toBeLessThan(0); + expect(summary.ci.upper).toBeGreaterThan(0); + }); + + it('throws for paired-length mismatches', () => { + expect(() => computePairedDelta([1], [1, 2])).toThrow('equal length'); + }); +}); + +describe('computeWinRate', () => { + it('counts wins, losses, and ties across paired scores', () => { + expect(computeWinRate([1, 2, 3, 4], [2, 1, 3, 5])).toEqual({ + wins: 2, + losses: 1, + ties: 1, + n: 4, + winRate: 0.5, + }); + }); +}); + +describe('bootstrapPairedCI', () => { + it('is deterministic for the same seed', () => { + const first = bootstrapPairedCI([1, 2, 3, 4], [1, 4, 2, 6], { + iterations: 1000, + seed: 7, + }); + const second = bootstrapPairedCI([1, 2, 3, 4], [1, 4, 2, 6], { + iterations: 1000, + seed: 7, + }); + + expect(first).toEqual(second); + expect(first).toEqual({ + mean: 0.75, + ci: { + lower: -0.5062499999999943, + upper: 2, + }, + n: 4, + significant: false, + }); + }); + + it('reports significance when the bootstrap CI excludes zero', () => { + const summary = bootstrapPairedCI([1, 2, 3, 4], [2, 3, 5, 6], { + iterations: 1000, + seed: 42, + }); + + expect(summary.mean).toBe(1.5); + expect(summary.ci).toEqual({ lower: 1, upper: 2 }); + expect(summary.significant).toBe(true); + }); + + it('throws for invalid confidence, invalid iterations, and length mismatches', () => { + expect(() => + bootstrapPairedCI([1, 2], [1, 2], { confidence: 0 }), + ).toThrow('confidence must be between 0 and 1'); + expect(() => + bootstrapPairedCI([1, 2], [1, 2], { iterations: 99 }), + ).toThrow('iterations must be an integer greater than or equal to 100'); + expect(() => bootstrapPairedCI([1], [1, 2])).toThrow('equal length'); + }); +}); From 59c92a3ad6ece7be4f5a80dcdf1977982e3e594e Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 15:12:10 +0000 Subject: [PATCH 14/18] feat: add eval trial aggregation reporting --- evals/lib/reporting.ts | 112 ++++++++++++ evals/lib/schemas.ts | 116 ++++++++++++ evals/lib/types.ts | 73 +++++++- test/unit/evals/reporting.test.ts | 290 ++++++++++++++++++++++++++++++ 4 files changed, 590 insertions(+), 1 deletion(-) create mode 100644 test/unit/evals/reporting.test.ts diff --git a/evals/lib/reporting.ts b/evals/lib/reporting.ts index 5c110248..ad5e4836 100644 --- a/evals/lib/reporting.ts +++ b/evals/lib/reporting.ts @@ -1,5 +1,11 @@ import { invariant } from '../../src/util/assert.js'; import { AggregateMetricsSchema, JsonReportSchema } from './schemas.js'; +import { + computeConfidenceInterval, + computeMean, + computePassRate, + computeStdDev, +} from './statistics.js'; import { computeAggregateMetrics } from './scoring.js'; import type { AggregateMetrics, @@ -12,6 +18,7 @@ import type { ProviderComparisonReport, RunMetadata, SkillCondition, + TrialAggregation, } from './types.js'; const LANE_ORDER: readonly EvalLane[] = ['prompt', 'execution', 'dogfood']; @@ -80,6 +87,8 @@ export function generateJsonReport( metadata, normalizedComparisons, ); + const aggregated = + metadata.totalTrials > 1 ? buildTrialAggregations(sortedResults) : undefined; const resultRefs = sortedResults.map((result) => result.caseId); const coreReport = { @@ -88,6 +97,7 @@ export function generateJsonReport( comparisons: normalizedComparisons, results: sortedResults, ...(providerComparison === undefined ? {} : { providerComparison }), + ...(aggregated === undefined ? {} : { aggregated }), } satisfies JsonReport; JsonReportSchema.parse(coreReport); @@ -300,6 +310,53 @@ export function generateMarkdownReport( ); } + if (report.aggregated !== undefined) { + sections.push('', '## Trial Aggregation', ''); + sections.push( + buildMarkdownTable( + [ + 'Lane', + 'Case', + 'Condition', + 'Trials', + 'Pass Rate', + 'Pass Rate CI', + 'Mean Score', + 'Std Dev', + 'Score CI', + 'Min', + 'Max', + ], + [ + 'left', + 'left', + 'left', + 'right', + 'right', + 'right', + 'right', + 'right', + 'right', + 'right', + 'right', + ], + report.aggregated.map((aggregation) => [ + `\`${aggregation.lane}\``, + `\`${sanitizeInline(aggregation.caseId)}\``, + `\`${aggregation.condition}\``, + String(aggregation.trials), + formatPercent(aggregation.passRate), + formatConfidenceInterval(aggregation.passRateCI, formatPercent), + formatScore(aggregation.meanScore), + formatScore(aggregation.stdDev), + formatConfidenceInterval(aggregation.scoreCI, formatScore), + formatScore(aggregation.minScore), + formatScore(aggregation.maxScore), + ]), + ), + ); + } + sections.push('', '## Anti-pattern summary', ''); if (antiPatternRows.length === 0) { sections.push('- None.'); @@ -522,6 +579,54 @@ function readAggregateStat( return typeof value === 'number' && Number.isFinite(value) ? value : fallback; } +function buildTrialAggregations( + results: readonly EvalResult[], +): TrialAggregation[] { + const groups = new Map(); + + for (const result of results) { + const key = `${result.lane}:${result.caseId}:${result.condition}`; + const group = groups.get(key); + if (group === undefined) { + groups.set(key, [result]); + continue; + } + + group.push(result); + } + + return [...groups.values()] + .map((group) => { + const first = group[0]; + invariant(first !== undefined, 'Trial aggregation group must not be empty'); + const normalizedScores = group.map((result) => normalizeScore(result)); + const passRateSummary = computePassRate(group); + const meanScore = computeMean(normalizedScores); + const stdDev = computeStdDev(normalizedScores, meanScore); + const scoreSummary = computeConfidenceInterval(normalizedScores); + + return { + lane: first.lane, + caseId: first.caseId, + condition: first.condition, + trials: group.length, + passRate: passRateSummary.rate, + passRateCI: passRateSummary.ci, + meanScore, + stdDev, + scoreCI: scoreSummary.ci, + minScore: Math.min(...normalizedScores), + maxScore: Math.max(...normalizedScores), + }; + }) + .sort( + (left, right) => + compareLane(left.lane, right.lane) || + compareStrings(left.caseId, right.caseId) || + compareCondition(left.condition, right.condition), + ); +} + function buildLaneSummaries( results: EvalResult[], metadataLanes: readonly EvalLane[], @@ -939,6 +1044,13 @@ function formatScore(value: number): string { return value.toFixed(3); } +function formatConfidenceInterval( + interval: { lower: number; upper: number }, + formatter: (value: number) => string, +): string { + return `[${formatter(interval.lower)}, ${formatter(interval.upper)}]`; +} + function formatComparisonValue(value: number | undefined): string { if (value === undefined) { return '—'; diff --git a/evals/lib/schemas.ts b/evals/lib/schemas.ts index 3c7a51a8..21b3fa25 100644 --- a/evals/lib/schemas.ts +++ b/evals/lib/schemas.ts @@ -6,8 +6,11 @@ import type { BundleValidationProfile } from '../../src/tools/validate-bundle.js import type { AggregateMetrics, AntiPatternFinding, + BaselineComparison, + BaselineOverall, BundleCompletenessScore, ComparisonMetrics, + ConfidenceInterval, DogfoodEvalCase, EvalCase, EvalCliOptions, @@ -19,6 +22,7 @@ import type { MatrixEntry, NormalizedProviderOutput, PatternMatchResult, + PerCaseComparison, PromptCaseScore, PromptEvalCase, ProviderAgentRequest, @@ -30,6 +34,7 @@ import type { ProviderPromptResult, ProviderRuntimeInfo, ReportCompletenessScore, + TrialAggregation, } from './types.js'; const NonEmptyStringSchema = z.string().min(1); @@ -681,6 +686,86 @@ export const ProviderComparisonReportSchema = z }) .strict(); +export const ConfidenceIntervalSchema = z + .object({ + lower: FiniteNumberSchema, + upper: FiniteNumberSchema, + }) + .strict(); + +export const TrialAggregationSchema = z + .object({ + lane: EvalLaneSchema, + caseId: NonEmptyStringSchema, + condition: SkillConditionSchema, + trials: PositiveIntSchema, + passRate: UnitIntervalSchema, + passRateCI: ConfidenceIntervalSchema, + meanScore: FiniteNumberSchema, + stdDev: NonNegativeNumberSchema, + scoreCI: ConfidenceIntervalSchema, + minScore: FiniteNumberSchema, + maxScore: FiniteNumberSchema, + }) + .strict(); + +export const PerCaseComparisonSchema = z + .object({ + caseId: NonEmptyStringSchema, + condition: SkillConditionSchema, + baselinePassRate: UnitIntervalSchema, + candidatePassRate: UnitIntervalSchema, + baselineMeanScore: FiniteNumberSchema, + candidateMeanScore: FiniteNumberSchema, + scoreDelta: z + .object({ + mean: FiniteNumberSchema, + ci: ConfidenceIntervalSchema, + significant: z.boolean(), + }) + .strict(), + passRateDelta: z + .object({ + mean: FiniteNumberSchema, + ci: ConfidenceIntervalSchema, + significant: z.boolean(), + }) + .strict(), + winRate: z + .object({ + wins: NonNegativeIntSchema, + losses: NonNegativeIntSchema, + ties: NonNegativeIntSchema, + n: NonNegativeIntSchema, + winRate: UnitIntervalSchema, + }) + .strict(), + verdict: z.enum(['improved', 'regressed', 'inconclusive']), + }) + .strict(); + +export const BaselineOverallSchema = z + .object({ + baselineMeanScore: FiniteNumberSchema, + candidateMeanScore: FiniteNumberSchema, + baselinePassRate: UnitIntervalSchema, + candidatePassRate: UnitIntervalSchema, + totalWins: NonNegativeIntSchema, + totalLosses: NonNegativeIntSchema, + totalTies: NonNegativeIntSchema, + verdict: z.enum(['improved', 'regressed', 'inconclusive']), + }) + .strict(); + +export const BaselineComparisonSchema = z + .object({ + baselineRunId: NonEmptyStringSchema, + baselineCreatedAt: IsoTimestampSchema, + overall: BaselineOverallSchema, + perCase: z.array(PerCaseComparisonSchema), + }) + .strict(); + export const JsonReportSchema = z .object({ metadata: RunMetadataSchema, @@ -688,6 +773,8 @@ export const JsonReportSchema = z comparisons: z.array(ComparisonMetricsSchema), results: z.array(EvalResultSchema), providerComparison: ProviderComparisonReportSchema.optional(), + aggregated: z.array(TrialAggregationSchema).optional(), + baselineComparison: BaselineComparisonSchema.optional(), }) .strict(); @@ -879,6 +966,15 @@ export type AggregateMetricsSchemaType = z.infer; export type ProviderComparisonReportSchemaType = z.infer< typeof ProviderComparisonReportSchema >; +export type ConfidenceIntervalSchemaType = z.infer< + typeof ConfidenceIntervalSchema +>; +export type TrialAggregationSchemaType = z.infer; +export type PerCaseComparisonSchemaType = z.infer; +export type BaselineOverallSchemaType = z.infer; +export type BaselineComparisonSchemaType = z.infer< + typeof BaselineComparisonSchema +>; export type JsonReportSchemaType = z.infer; export type EvalCliOptionsSchemaType = z.infer; export type EvalCliResultSchemaType = z.infer; @@ -960,6 +1056,26 @@ export type _AggregateMetricsSchemaParity = AssertExact< AggregateMetrics, AggregateMetricsSchemaType >; +export type _ConfidenceIntervalSchemaParity = AssertExact< + ConfidenceInterval, + ConfidenceIntervalSchemaType +>; +export type _TrialAggregationSchemaParity = AssertExact< + TrialAggregation, + TrialAggregationSchemaType +>; +export type _PerCaseComparisonSchemaParity = AssertExact< + PerCaseComparison, + PerCaseComparisonSchemaType +>; +export type _BaselineOverallSchemaParity = AssertExact< + BaselineOverall, + BaselineOverallSchemaType +>; +export type _BaselineComparisonSchemaParity = AssertExact< + BaselineComparison, + BaselineComparisonSchemaType +>; export type _JsonReportSchemaParity = AssertExact< JsonReport, JsonReportSchemaType diff --git a/evals/lib/types.ts b/evals/lib/types.ts index 74681843..05b9b5ab 100644 --- a/evals/lib/types.ts +++ b/evals/lib/types.ts @@ -34,7 +34,7 @@ export function buildWorkItemKey(identity: EvalWorkItemIdentity): string { 'identity.trial must be a positive integer', ); - return `${identity.lane}:${identity.caseId}:${identity.condition}:${identity.trial}`; + return `${identity.lane}:${identity.caseId}:${identity.condition}:${String(identity.trial)}`; } /** Assert that a list of work item identities contains no duplicates. */ @@ -403,6 +403,77 @@ export interface JsonReport { comparisons: ComparisonMetrics[]; results: EvalResult[]; providerComparison?: ProviderComparisonReport; + aggregated?: TrialAggregation[]; + baselineComparison?: BaselineComparison; +} + +/** Confidence interval bounds. */ +export interface ConfidenceInterval { + lower: number; + upper: number; +} + +/** Trial-aggregated statistics for one case/condition group. */ +export interface TrialAggregation { + lane: EvalLane; + caseId: string; + condition: SkillCondition; + trials: number; + passRate: number; + passRateCI: ConfidenceInterval; + meanScore: number; + stdDev: number; + scoreCI: ConfidenceInterval; + minScore: number; + maxScore: number; +} + +/** Per-case comparison between baseline and candidate runs. */ +export interface PerCaseComparison { + caseId: string; + condition: SkillCondition; + baselinePassRate: number; + candidatePassRate: number; + baselineMeanScore: number; + candidateMeanScore: number; + scoreDelta: { + mean: number; + ci: ConfidenceInterval; + significant: boolean; + }; + passRateDelta: { + mean: number; + ci: ConfidenceInterval; + significant: boolean; + }; + winRate: { + wins: number; + losses: number; + ties: number; + n: number; + winRate: number; + }; + verdict: 'improved' | 'regressed' | 'inconclusive'; +} + +/** Overall comparison summary across all matched cases. */ +export interface BaselineOverall { + baselineMeanScore: number; + candidateMeanScore: number; + baselinePassRate: number; + candidatePassRate: number; + totalWins: number; + totalLosses: number; + totalTies: number; + verdict: 'improved' | 'regressed' | 'inconclusive'; +} + +/** Top-level baseline comparison attached to a report. */ +export interface BaselineComparison { + baselineRunId: string; + baselineCreatedAt: string; + overall: BaselineOverall; + perCase: PerCaseComparison[]; } /** Cross-provider comparison view for an eval run. */ diff --git a/test/unit/evals/reporting.test.ts b/test/unit/evals/reporting.test.ts new file mode 100644 index 00000000..66f537cb --- /dev/null +++ b/test/unit/evals/reporting.test.ts @@ -0,0 +1,290 @@ +import { describe, expect, it } from 'vitest'; + +import { + generateJsonReport, + generateMarkdownReport, +} from '../../../evals/lib/reporting.js'; +import { + computeConfidenceInterval, + computeMean, + computePassRate, + computeStdDev, +} from '../../../evals/lib/statistics.js'; +import type { EvalResult, RunMetadata } from '../../../evals/lib/types.js'; + +function createRunMetadata(overrides: Partial = {}): RunMetadata { + return { + runId: 'report-run', + createdAt: '2026-01-01T00:00:00.000Z', + repoRoot: '/tmp/report-repo', + providers: + overrides.providers === undefined ? ['stub'] : [...overrides.providers], + models: overrides.models === undefined ? [] : [...overrides.models], + lanes: + overrides.lanes === undefined + ? ['prompt', 'execution'] + : [...overrides.lanes], + conditions: + overrides.conditions === undefined + ? ['none', 'self-load'] + : [...overrides.conditions], + totalTrials: overrides.totalTrials ?? 3, + notes: overrides.notes === undefined ? [] : [...overrides.notes], + ...(overrides.runId === undefined ? {} : { runId: overrides.runId }), + ...(overrides.createdAt === undefined + ? {} + : { createdAt: overrides.createdAt }), + ...(overrides.repoRoot === undefined + ? {} + : { repoRoot: overrides.repoRoot }), + }; +} + +function createEvalResult(overrides: Partial = {}): EvalResult { + return { + runId: 'report-run', + providerId: 'stub', + lane: 'prompt', + caseId: 'case-1', + category: 'trigger', + condition: 'none', + expectedSkill: 'agent-tty', + trial: 1, + ok: true, + score: { total: 10, maxPossible: 10, items: [] }, + workflowChecks: [], + antiPatternFindings: [], + normalizedOutput: { + finalText: '', + messages: [], + referencedSkills: [], + toolCalls: [], + }, + startedAt: '2026-01-01T00:00:00.000Z', + completedAt: '2026-01-01T00:00:01.000Z', + durationMs: 1000, + ...overrides, + }; +} + +function createMultiTrialResults(): EvalResult[] { + return [ + createEvalResult({ + lane: 'prompt', + caseId: 'case-1', + condition: 'none', + trial: 1, + ok: true, + score: { total: 10, maxPossible: 10, items: [] }, + }), + createEvalResult({ + lane: 'prompt', + caseId: 'case-1', + condition: 'none', + trial: 2, + ok: false, + score: { total: 5, maxPossible: 10, items: [] }, + }), + createEvalResult({ + lane: 'prompt', + caseId: 'case-1', + condition: 'none', + trial: 3, + ok: false, + score: { total: 0, maxPossible: 10, items: [] }, + }), + createEvalResult({ + lane: 'execution', + caseId: 'case-2', + category: 'session', + condition: 'self-load', + trial: 1, + ok: false, + score: { total: 2, maxPossible: 10, items: [] }, + }), + createEvalResult({ + lane: 'execution', + caseId: 'case-2', + category: 'session', + condition: 'self-load', + trial: 2, + ok: true, + score: { total: 4, maxPossible: 10, items: [] }, + }), + createEvalResult({ + lane: 'execution', + caseId: 'case-2', + category: 'session', + condition: 'self-load', + trial: 3, + ok: true, + score: { total: 6, maxPossible: 10, items: [] }, + }), + ]; +} + +function createSingleTrialResults(): EvalResult[] { + return [ + createEvalResult({ + lane: 'prompt', + caseId: 'case-1', + condition: 'none', + trial: 1, + ok: true, + score: { total: 10, maxPossible: 10, items: [] }, + }), + createEvalResult({ + lane: 'execution', + caseId: 'case-2', + category: 'session', + condition: 'self-load', + trial: 1, + ok: false, + score: { total: 2, maxPossible: 10, items: [] }, + }), + ]; +} + +function formatPercent(value: number): string { + return `${(value * 100).toFixed(1)}%`; +} + +function formatScore(value: number): string { + return value.toFixed(3); +} + +function formatConfidenceInterval( + interval: { lower: number; upper: number }, + formatter: (value: number) => string, +): string { + return `[${formatter(interval.lower)}, ${formatter(interval.upper)}]`; +} + +describe('generateJsonReport trial aggregation', () => { + it('includes aggregated data only when totalTrials is greater than one', () => { + const report = generateJsonReport( + createMultiTrialResults(), + createRunMetadata({ totalTrials: 3 }), + ); + + expect(report.aggregated).toBeDefined(); + expect(report.aggregated).toHaveLength(2); + }); + + it('omits aggregated data when totalTrials equals one', () => { + const report = generateJsonReport( + createSingleTrialResults(), + createRunMetadata({ totalTrials: 1 }), + ); + + expect(report.aggregated).toBeUndefined(); + }); + + it('computes aggregated pass rate, score, and confidence intervals', () => { + const results = createMultiTrialResults(); + const report = generateJsonReport(results, createRunMetadata({ totalTrials: 3 })); + + expect(report.aggregated).toBeDefined(); + const aggregated = report.aggregated ?? []; + + expect(aggregated).toHaveLength(2); + expect(aggregated[0]).toMatchObject({ + lane: 'prompt', + caseId: 'case-1', + condition: 'none', + trials: 3, + }); + expect(aggregated[1]).toMatchObject({ + lane: 'execution', + caseId: 'case-2', + condition: 'self-load', + trials: 3, + }); + + const promptScores = [1, 0.5, 0]; + const promptPassRate = computePassRate([{ ok: true }, { ok: false }, { ok: false }]); + const promptScoreCI = computeConfidenceInterval(promptScores); + const promptMean = computeMean(promptScores); + const promptStdDev = computeStdDev(promptScores, promptMean); + const promptAggregation = aggregated[0]; + + expect(promptAggregation).toBeDefined(); + expect(promptAggregation?.passRate).toBeCloseTo(promptPassRate.rate); + expect(promptAggregation?.passRateCI.lower).toBeCloseTo(promptPassRate.ci.lower); + expect(promptAggregation?.passRateCI.upper).toBeCloseTo(promptPassRate.ci.upper); + expect(promptAggregation?.meanScore).toBeCloseTo(promptMean); + expect(promptAggregation?.stdDev).toBeCloseTo(promptStdDev); + expect(promptAggregation?.scoreCI.lower).toBeCloseTo(promptScoreCI.ci.lower); + expect(promptAggregation?.scoreCI.upper).toBeCloseTo(promptScoreCI.ci.upper); + expect(promptAggregation?.minScore).toBe(0); + expect(promptAggregation?.maxScore).toBe(1); + + const executionScores = [0.2, 0.4, 0.6]; + const executionPassRate = computePassRate([ + { ok: false }, + { ok: true }, + { ok: true }, + ]); + const executionScoreCI = computeConfidenceInterval(executionScores); + const executionMean = computeMean(executionScores); + const executionStdDev = computeStdDev(executionScores, executionMean); + const executionAggregation = aggregated[1]; + + expect(executionAggregation).toBeDefined(); + expect(executionAggregation?.passRate).toBeCloseTo(executionPassRate.rate); + expect(executionAggregation?.passRateCI.lower).toBeCloseTo( + executionPassRate.ci.lower, + ); + expect(executionAggregation?.passRateCI.upper).toBeCloseTo( + executionPassRate.ci.upper, + ); + expect(executionAggregation?.meanScore).toBeCloseTo(executionMean); + expect(executionAggregation?.stdDev).toBeCloseTo(executionStdDev); + expect(executionAggregation?.scoreCI.lower).toBeCloseTo( + executionScoreCI.ci.lower, + ); + expect(executionAggregation?.scoreCI.upper).toBeCloseTo( + executionScoreCI.ci.upper, + ); + expect(executionAggregation?.minScore).toBeCloseTo(0.2); + expect(executionAggregation?.maxScore).toBeCloseTo(0.6); + }); +}); + +describe('generateMarkdownReport trial aggregation', () => { + it('includes a Trial Aggregation table for multi-trial runs', () => { + const markdown = generateMarkdownReport( + createMultiTrialResults(), + createRunMetadata({ totalTrials: 3 }), + ); + + const promptPassRate = computePassRate([{ ok: true }, { ok: false }, { ok: false }]); + const promptScoreCI = computeConfidenceInterval([1, 0.5, 0]); + const executionPassRate = computePassRate([ + { ok: false }, + { ok: true }, + { ok: true }, + ]); + const executionScoreCI = computeConfidenceInterval([0.2, 0.4, 0.6]); + + expect(markdown).toContain('## Trial Aggregation'); + expect(markdown).toContain( + '| Lane | Case | Condition | Trials | Pass Rate | Pass Rate CI | Mean Score | Std Dev | Score CI | Min | Max |', + ); + expect(markdown).toContain( + `| \`prompt\` | \`case-1\` | \`none\` | 3 | 33.3% | ${formatConfidenceInterval(promptPassRate.ci, formatPercent)} | 0.500 | 0.500 | ${formatConfidenceInterval(promptScoreCI.ci, formatScore)} | 0.000 | 1.000 |`, + ); + expect(markdown).toContain( + `| \`execution\` | \`case-2\` | \`self-load\` | 3 | 66.7% | ${formatConfidenceInterval(executionPassRate.ci, formatPercent)} | 0.400 | 0.200 | ${formatConfidenceInterval(executionScoreCI.ci, formatScore)} | 0.200 | 0.600 |`, + ); + }); + + it('omits the Trial Aggregation section for single-trial runs', () => { + const markdown = generateMarkdownReport( + createSingleTrialResults(), + createRunMetadata({ totalTrials: 1 }), + ); + + expect(markdown).not.toContain('## Trial Aggregation'); + }); +}); From 88bd2d48e7b15ad424c11f78b557739a4a05969e Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 15:31:25 +0000 Subject: [PATCH 15/18] feat: add eval baseline comparison reporting --- evals/lib/reporting.ts | 336 +++++++++++++++++- evals/run.ts | 89 ++++- .../evals-compare-baseline.test.ts | 195 ++++++++++ 3 files changed, 613 insertions(+), 7 deletions(-) create mode 100644 test/integration/evals-compare-baseline.test.ts diff --git a/evals/lib/reporting.ts b/evals/lib/reporting.ts index ad5e4836..c504bb71 100644 --- a/evals/lib/reporting.ts +++ b/evals/lib/reporting.ts @@ -1,20 +1,24 @@ import { invariant } from '../../src/util/assert.js'; import { AggregateMetricsSchema, JsonReportSchema } from './schemas.js'; import { + bootstrapPairedCI, computeConfidenceInterval, computeMean, computePassRate, computeStdDev, + computeWinRate, } from './statistics.js'; import { computeAggregateMetrics } from './scoring.js'; import type { AggregateMetrics, AntiPatternSeverity, + BaselineComparison, ComparisonMetrics, EvalLane, EvalResult, JsonReport, MatrixEntry, + PerCaseComparison, ProviderComparisonReport, RunMetadata, SkillCondition, @@ -69,6 +73,7 @@ export function generateJsonReport( results: EvalResult[], metadata: RunMetadata, comparisonMetrics?: ComparisonMetrics | ComparisonMetrics[], + baselineComparison?: BaselineComparison, ): JsonReport { assertResults(results); assertMetadata(metadata); @@ -88,7 +93,9 @@ export function generateJsonReport( normalizedComparisons, ); const aggregated = - metadata.totalTrials > 1 ? buildTrialAggregations(sortedResults) : undefined; + metadata.totalTrials > 1 + ? buildTrialAggregations(sortedResults) + : undefined; const resultRefs = sortedResults.map((result) => result.caseId); const coreReport = { @@ -98,6 +105,7 @@ export function generateJsonReport( results: sortedResults, ...(providerComparison === undefined ? {} : { providerComparison }), ...(aggregated === undefined ? {} : { aggregated }), + ...(baselineComparison === undefined ? {} : { baselineComparison }), } satisfies JsonReport; JsonReportSchema.parse(coreReport); @@ -121,6 +129,7 @@ export function generateMarkdownReport( results: EvalResult[], metadata: RunMetadata, comparisonMetrics?: ComparisonMetrics | ComparisonMetrics[], + baselineComparison?: BaselineComparison, ): string { assertResults(results); assertMetadata(metadata); @@ -129,6 +138,7 @@ export function generateMarkdownReport( results, metadata, comparisonMetrics, + baselineComparison, ) as JsonReport & RichJsonReport; const providers = collectProviders(report.results, report.metadata.providers); const lanes = collectLanes(report.results, report.metadata.lanes); @@ -357,6 +367,13 @@ export function generateMarkdownReport( ); } + if (report.baselineComparison !== undefined) { + sections.push( + '', + ...buildBaselineComparisonMarkdownSection(report.baselineComparison), + ); + } + sections.push('', '## Anti-pattern summary', ''); if (antiPatternRows.length === 0) { sections.push('- None.'); @@ -389,6 +406,318 @@ export function generateMarkdownReport( return `${sections.join('\n').trimEnd()}\n`; } +interface BaselineComparisonGroup { + caseId: string; + condition: SkillCondition; + results: EvalResult[]; +} + +export function buildBaselineComparison( + baseline: JsonReport, + candidate: JsonReport, +): BaselineComparison { + assertResults(baseline.results); + assertMetadata(baseline.metadata); + assertResults(candidate.results); + assertMetadata(candidate.metadata); + + const baselineGroups = groupResultsForBaselineComparison(baseline.results); + const candidateGroups = groupResultsForBaselineComparison(candidate.results); + const matchedGroups = [...baselineGroups.values()] + .filter((group) => + candidateGroups.has( + buildBaselineComparisonGroupKey(group.caseId, group.condition), + ), + ) + .sort( + (left, right) => + compareStrings(left.caseId, right.caseId) || + compareCondition(left.condition, right.condition), + ); + + invariant( + matchedGroups.length > 0, + 'Baseline and candidate reports must share at least one case/condition group', + ); + + const perCase: PerCaseComparison[] = []; + const allBaselineScores: number[] = []; + const allCandidateScores: number[] = []; + const allBaselinePassBits: number[] = []; + const allCandidatePassBits: number[] = []; + let totalWins = 0; + let totalLosses = 0; + let totalTies = 0; + + for (const baselineGroup of matchedGroups) { + const groupLabel = buildBaselineComparisonGroupKey( + baselineGroup.caseId, + baselineGroup.condition, + ); + const candidateGroup = candidateGroups.get(groupLabel); + invariant( + candidateGroup !== undefined, + `Missing candidate group for ${groupLabel}`, + ); + + const baselineTrials = sortBaselineComparisonTrials( + baselineGroup.results, + `baseline ${groupLabel}`, + ); + const candidateTrials = sortBaselineComparisonTrials( + candidateGroup.results, + `candidate ${groupLabel}`, + ); + invariant( + baselineTrials.length === candidateTrials.length, + `Baseline and candidate must have equal trial counts for ${groupLabel}`, + ); + + const baselineScores: number[] = []; + const candidateScores: number[] = []; + const baselinePassBits: number[] = []; + const candidatePassBits: number[] = []; + + for (let index = 0; index < baselineTrials.length; index += 1) { + const baselineResult = baselineTrials[index]; + const candidateResult = candidateTrials[index]; + invariant( + baselineResult !== undefined, + `Missing baseline trial ${String(index)} for ${groupLabel}`, + ); + invariant( + candidateResult !== undefined, + `Missing candidate trial ${String(index)} for ${groupLabel}`, + ); + invariant( + baselineResult.trial === candidateResult.trial, + `Baseline and candidate trials must align for ${groupLabel}`, + ); + + baselineScores.push(normalizeScore(baselineResult)); + candidateScores.push(normalizeScore(candidateResult)); + baselinePassBits.push(toPassBit(baselineResult)); + candidatePassBits.push(toPassBit(candidateResult)); + } + + const scoreDeltaSummary = bootstrapPairedCI( + baselineScores, + candidateScores, + ); + const passRateDeltaSummary = bootstrapPairedCI( + baselinePassBits, + candidatePassBits, + ); + const scoreDelta = { + mean: scoreDeltaSummary.mean, + ci: scoreDeltaSummary.ci, + significant: scoreDeltaSummary.significant, + }; + const passRateDelta = { + mean: passRateDeltaSummary.mean, + ci: passRateDeltaSummary.ci, + significant: passRateDeltaSummary.significant, + }; + const winRate = computeWinRate(baselineScores, candidateScores); + + perCase.push({ + caseId: baselineGroup.caseId, + condition: baselineGroup.condition, + baselinePassRate: computePassRate(baselineTrials).rate, + candidatePassRate: computePassRate(candidateTrials).rate, + baselineMeanScore: computeMean(baselineScores), + candidateMeanScore: computeMean(candidateScores), + scoreDelta, + passRateDelta, + winRate, + verdict: buildPerCaseComparisonVerdict(scoreDelta), + }); + + allBaselineScores.push(...baselineScores); + allCandidateScores.push(...candidateScores); + allBaselinePassBits.push(...baselinePassBits); + allCandidatePassBits.push(...candidatePassBits); + totalWins += winRate.wins; + totalLosses += winRate.losses; + totalTies += winRate.ties; + } + + const overallScoreDelta = bootstrapPairedCI( + allBaselineScores, + allCandidateScores, + ); + const overallPassRateDelta = bootstrapPairedCI( + allBaselinePassBits, + allCandidatePassBits, + ); + + return { + baselineRunId: baseline.metadata.runId, + baselineCreatedAt: baseline.metadata.createdAt, + overall: { + baselineMeanScore: computeMean(allBaselineScores), + candidateMeanScore: computeMean(allCandidateScores), + baselinePassRate: computePassRateFromBits(allBaselinePassBits), + candidatePassRate: computePassRateFromBits(allCandidatePassBits), + totalWins, + totalLosses, + totalTies, + verdict: buildOverallComparisonVerdict( + overallScoreDelta, + overallPassRateDelta, + ), + }, + perCase, + }; +} + +function buildBaselineComparisonMarkdownSection( + comparison: BaselineComparison, +): string[] { + const scoreDelta = + comparison.overall.candidateMeanScore - + comparison.overall.baselineMeanScore; + const passRateDelta = + comparison.overall.candidatePassRate - comparison.overall.baselinePassRate; + + return [ + '## Baseline comparison', + '', + `- Baseline run ID: \`${sanitizeInline(comparison.baselineRunId)}\``, + `- Baseline created: \`${sanitizeInline(comparison.baselineCreatedAt)}\``, + `- Overall verdict: \`${comparison.overall.verdict}\``, + `- Mean score: ${formatScore(comparison.overall.baselineMeanScore)} → ${formatScore(comparison.overall.candidateMeanScore)} (Δ ${formatScore(scoreDelta)})`, + `- Pass rate: ${formatPercent(comparison.overall.baselinePassRate)} → ${formatPercent(comparison.overall.candidatePassRate)} (Δ ${formatPercent(passRateDelta)})`, + `- Wins / Losses / Ties: ${String(comparison.overall.totalWins)} / ${String(comparison.overall.totalLosses)} / ${String(comparison.overall.totalTies)}`, + '', + buildMarkdownTable( + [ + 'Case', + 'Condition', + 'Verdict', + 'Score Δ', + 'Score CI', + 'Pass Rate Δ', + 'Wins/Losses/Ties', + ], + ['left', 'left', 'left', 'right', 'right', 'right', 'right'], + comparison.perCase.map((perCase) => [ + `\`${sanitizeInline(perCase.caseId)}\``, + `\`${perCase.condition}\``, + `\`${perCase.verdict}\``, + formatScore(perCase.scoreDelta.mean), + formatConfidenceInterval(perCase.scoreDelta.ci, formatScore), + formatPercent(perCase.passRateDelta.mean), + `${String(perCase.winRate.wins)}/${String(perCase.winRate.losses)}/${String(perCase.winRate.ties)}`, + ]), + ), + ]; +} + +function groupResultsForBaselineComparison( + results: readonly EvalResult[], +): Map { + const groups = new Map(); + + for (const result of results) { + const key = buildBaselineComparisonGroupKey( + result.caseId, + result.condition, + ); + const existing = groups.get(key); + if (existing === undefined) { + groups.set(key, { + caseId: result.caseId, + condition: result.condition, + results: [result], + }); + continue; + } + + existing.results.push(result); + } + + return groups; +} + +function buildBaselineComparisonGroupKey( + caseId: string, + condition: SkillCondition, +): string { + return `${caseId}:${condition}`; +} + +function sortBaselineComparisonTrials( + results: readonly EvalResult[], + groupLabel: string, +): EvalResult[] { + invariant(results.length > 0, `${groupLabel} results must not be empty`); + const sorted = [...results].sort( + (left, right) => + left.trial - right.trial || + compareLane(left.lane, right.lane) || + compareStrings(left.providerId, right.providerId) || + compareStrings(left.runId, right.runId), + ); + + const seenTrials = new Set(); + for (const result of sorted) { + invariant( + !seenTrials.has(result.trial), + `Baseline comparison requires unique trial numbers for ${groupLabel}`, + ); + seenTrials.add(result.trial); + } + + return sorted; +} + +function toPassBit(result: EvalResult): 0 | 1 { + return result.ok ? 1 : 0; +} + +function computePassRateFromBits(passBits: readonly number[]): number { + return computePassRate( + passBits.map((value) => { + invariant(value === 0 || value === 1, 'pass bit must be either 0 or 1'); + return { ok: value === 1 }; + }), + ).rate; +} + +function buildPerCaseComparisonVerdict(comparison: { + mean: number; + ci: { lower: number; upper: number }; +}): 'improved' | 'regressed' | 'inconclusive' { + if (comparison.ci.lower > 0 && comparison.mean >= 0.05) { + return 'improved'; + } + if (comparison.ci.upper < 0) { + return 'regressed'; + } + return 'inconclusive'; +} + +function buildOverallComparisonVerdict( + scoreDelta: { mean: number; ci: { lower: number; upper: number } }, + passRateDelta: { mean: number; ci: { lower: number; upper: number } }, +): 'improved' | 'regressed' | 'inconclusive' { + const scoreRegressed = scoreDelta.ci.upper < 0; + const passRateRegressed = passRateDelta.ci.upper < 0; + if (scoreRegressed || passRateRegressed) { + return 'regressed'; + } + + const scoreImproved = scoreDelta.ci.lower > 0 && scoreDelta.mean >= 0.05; + const passRateImproved = + passRateDelta.ci.lower > 0 && passRateDelta.mean >= 0.05; + if (scoreImproved || passRateImproved) { + return 'improved'; + } + + return 'inconclusive'; +} + /** * Build deterministic pairwise provider comparisons with per-condition * aggregate breakdowns. @@ -598,7 +927,10 @@ function buildTrialAggregations( return [...groups.values()] .map((group) => { const first = group[0]; - invariant(first !== undefined, 'Trial aggregation group must not be empty'); + invariant( + first !== undefined, + 'Trial aggregation group must not be empty', + ); const normalizedScores = group.map((result) => normalizeScore(result)); const passRateSummary = computePassRate(group); const meanScore = computeMean(normalizedScores); diff --git a/evals/run.ts b/evals/run.ts index 3a67e58b..476c22f6 100644 --- a/evals/run.ts +++ b/evals/run.ts @@ -1,4 +1,4 @@ -import { mkdir, writeFile } from 'node:fs/promises'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import process from 'node:process'; import { fileURLToPath, pathToFileURL } from 'node:url'; @@ -12,13 +12,18 @@ import { computeComparisonMetrics, SKILL_CONDITIONS, } from './lib/matrix.js'; -import { generateJsonReport, generateMarkdownReport } from './lib/reporting.js'; -import { RunMetadataSchema } from './lib/schemas.js'; +import { + buildBaselineComparison, + generateJsonReport, + generateMarkdownReport, +} from './lib/reporting.js'; +import { JsonReportSchema, RunMetadataSchema } from './lib/schemas.js'; import { buildWorkItemKey, type EvalCase, type EvalLane, type EvalResult, + type JsonReport, type ProviderRuntimeInfo, type RunMetadata, type SkillCondition, @@ -57,6 +62,7 @@ const HELP_TEXT = [ ' --dry-run List cases that would run without invoking providers', ' --concurrency Maximum work items to run concurrently per lane. Default: 1', ' --trials Number of independent trials per case/condition. Default: 1', + ' --compare-baseline Path to a prior report.json for paired baseline comparison', ' --help Show this help text', '', 'Examples:', @@ -80,6 +86,7 @@ interface CliOptions { outputDir?: string; concurrency?: string; trials?: string; + compareBaseline?: string; json: boolean; verbose: boolean; dryRun: boolean; @@ -94,6 +101,7 @@ interface ResolvedCliOptions { requestedCondition: string; caseIds: string[]; outputBaseDir: string; + compareBaselinePath?: string; concurrency: number; totalTrials: number; json: boolean; @@ -263,12 +271,35 @@ function parseCliArgs(argumentsList: readonly string[]): CliOptions { continue; } if (argument === '--trials' || argument.startsWith('--trials=')) { - const parsed = parseOptionValue(argument, '--trials', argumentsList, index); + const parsed = parseOptionValue( + argument, + '--trials', + argumentsList, + index, + ); invariant(options.trials === undefined, '--trials may only be set once'); options.trials = parsed.value; index = parsed.nextIndex; continue; } + if ( + argument === '--compare-baseline' || + argument.startsWith('--compare-baseline=') + ) { + const parsed = parseOptionValue( + argument, + '--compare-baseline', + argumentsList, + index, + ); + invariant( + options.compareBaseline === undefined, + '--compare-baseline may only be set once', + ); + options.compareBaseline = parsed.value; + index = parsed.nextIndex; + continue; + } if (argument === '--provider' || argument.startsWith('--provider=')) { const parsed = parseOptionValue( argument, @@ -595,6 +626,10 @@ function buildResolvedOptions( const requestedConditions = resolveRequestedConditions(options.condition); const caseIds = resolveRequestedCaseIds(options.caseIds); const outputBaseDir = resolveOutputBaseDir(repoRoot, options.outputDir); + const compareBaselinePath = resolveOptionalStringOption( + options.compareBaseline, + '--compare-baseline', + ); const concurrency = resolveConcurrency(options.concurrency); const totalTrials = resolveTotalTrials(options.trials); const selection = buildCaseSelections( @@ -613,6 +648,9 @@ function buildResolvedOptions( requestedCondition: options.condition ?? 'all', caseIds, outputBaseDir, + ...(compareBaselinePath === undefined + ? {} + : { compareBaselinePath: resolve(repoRoot, compareBaselinePath) }), concurrency, totalTrials, json: options.json, @@ -807,6 +845,26 @@ function buildRunMetadata( return metadata; } +async function loadBaselineReport(path: string): Promise { + const text = await readFile(path, 'utf8'); + const parsed = JSON.parse(text) as JsonReport & Record; + return JsonReportSchema.parse({ + metadata: parsed.metadata, + aggregate: parsed.aggregate, + comparisons: parsed.comparisons, + results: parsed.results, + ...(parsed.providerComparison === undefined + ? {} + : { providerComparison: parsed.providerComparison }), + ...(parsed.aggregated === undefined + ? {} + : { aggregated: parsed.aggregated }), + ...(parsed.baselineComparison === undefined + ? {} + : { baselineComparison: parsed.baselineComparison }), + }) as JsonReport; +} + function buildSummary( options: ResolvedCliOptions, results: EvalResult[], @@ -1057,11 +1115,32 @@ export async function runEvalCli( options.activeConditions.length > 1 ? computeComparisonMetrics(results) : []; - const jsonReport = generateJsonReport(results, metadata, comparisonMetrics); + const candidateCoreReport = generateJsonReport( + results, + metadata, + comparisonMetrics, + ); + const baselineComparison = + options.compareBaselinePath === undefined + ? undefined + : buildBaselineComparison( + await loadBaselineReport(options.compareBaselinePath), + candidateCoreReport, + ); + const jsonReport = + baselineComparison === undefined + ? candidateCoreReport + : generateJsonReport( + results, + metadata, + comparisonMetrics, + baselineComparison, + ); const markdownReport = generateMarkdownReport( results, metadata, comparisonMetrics, + baselineComparison, ); const artifactStore = new EvalArtifactStore(options.outputBaseDir); diff --git a/test/integration/evals-compare-baseline.test.ts b/test/integration/evals-compare-baseline.test.ts new file mode 100644 index 00000000..b0536580 --- /dev/null +++ b/test/integration/evals-compare-baseline.test.ts @@ -0,0 +1,195 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, realpathSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import process from 'node:process'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import type { JsonReport, PerCaseComparison } from '../../evals/lib/types.js'; + +const DEFAULT_EVAL_TIMEOUT_MS = 30_000; + +interface EvalRunSummary { + ok: boolean; + providerId: string; + lanes: string[]; + conditions: string[]; + totalResults: number; + jsonReportPath: string; + markdownReportPath: string; +} + +function runEvalCli(argumentsList: readonly string[]) { + const result = spawnSync( + process.execPath, + ['--import', 'tsx', './evals/run.ts', ...argumentsList], + { + cwd: process.cwd(), + encoding: 'utf8', + timeout: DEFAULT_EVAL_TIMEOUT_MS, + }, + ); + + expect(result.error).toBeUndefined(); + expect(result.status).not.toBeNull(); + return result; +} + +function readJsonFile(path: string): T { + return JSON.parse(readFileSync(path, 'utf8')) as T; +} + +function expectPerCaseComparisonShape(comparison: PerCaseComparison): void { + expect(comparison).toEqual( + expect.objectContaining({ + caseId: expect.any(String), + condition: expect.any(String), + baselinePassRate: expect.any(Number), + candidatePassRate: expect.any(Number), + baselineMeanScore: expect.any(Number), + candidateMeanScore: expect.any(Number), + scoreDelta: expect.objectContaining({ + mean: expect.any(Number), + ci: expect.objectContaining({ + lower: expect.any(Number), + upper: expect.any(Number), + }), + significant: expect.any(Boolean), + }), + passRateDelta: expect.objectContaining({ + mean: expect.any(Number), + ci: expect.objectContaining({ + lower: expect.any(Number), + upper: expect.any(Number), + }), + significant: expect.any(Boolean), + }), + winRate: expect.objectContaining({ + wins: expect.any(Number), + losses: expect.any(Number), + ties: expect.any(Number), + n: expect.any(Number), + winRate: expect.any(Number), + }), + verdict: expect.any(String), + }), + ); +} + +let testRoot = ''; + +describe( + 'eval CLI compare-baseline reporting', + { + timeout: DEFAULT_EVAL_TIMEOUT_MS, + }, + () => { + beforeEach(() => { + // prettier-ignore + testRoot = realpathSync(mkdtempSync(join(tmpdir(), 'agent-tty-evals-compare-baseline-'))); + }); + + afterEach(() => { + rmSync(testRoot, { recursive: true, force: true }); + testRoot = ''; + }); + + it('shows --compare-baseline in the help output', () => { + const result = runEvalCli(['--help']); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('--compare-baseline '); + }); + + it('adds paired baseline comparison data to JSON and Markdown reports', () => { + const baselineOutputDir = join(testRoot, 'baseline'); + const candidateOutputDir = join(testRoot, 'candidate'); + const sharedArguments = [ + '--provider', + 'stub', + '--lane', + 'prompt', + '--condition', + 'none', + '--trials', + '3', + '--json', + ] as const; + + const baselineResult = runEvalCli([ + ...sharedArguments, + '--output', + baselineOutputDir, + ]); + const baselineSummary = JSON.parse( + baselineResult.stdout, + ) as EvalRunSummary; + expect(baselineResult.status).toBe(baselineSummary.ok ? 0 : 1); + expect(baselineSummary).toMatchObject({ + providerId: 'stub', + lanes: ['prompt'], + conditions: ['none'], + }); + + const baselineReport = readJsonFile( + baselineSummary.jsonReportPath, + ); + + const candidateResult = runEvalCli([ + ...sharedArguments, + '--output', + candidateOutputDir, + '--compare-baseline', + baselineSummary.jsonReportPath, + ]); + const candidateSummary = JSON.parse( + candidateResult.stdout, + ) as EvalRunSummary; + expect(candidateResult.status).toBe(candidateSummary.ok ? 0 : 1); + expect(candidateSummary).toMatchObject({ + providerId: 'stub', + lanes: ['prompt'], + conditions: ['none'], + }); + + const candidateReport = readJsonFile( + candidateSummary.jsonReportPath, + ); + const candidateMarkdown = readFileSync( + candidateSummary.markdownReportPath, + 'utf8', + ); + const baselineComparison = candidateReport.baselineComparison; + + expect(baselineComparison).toBeDefined(); + if (baselineComparison === undefined) { + throw new Error( + 'Expected baseline comparison data in candidate report', + ); + } + + expect(baselineComparison.baselineRunId).toBe( + baselineReport.metadata.runId, + ); + expect(baselineComparison.baselineCreatedAt).toBe( + baselineReport.metadata.createdAt, + ); + expect(baselineComparison.perCase.length).toBeGreaterThan(0); + for (const perCase of baselineComparison.perCase) { + expectPerCaseComparisonShape(perCase); + expect(perCase.verdict).toBe('inconclusive'); + } + + expect(baselineComparison.overall.verdict).toBe('inconclusive'); + expect(candidateMarkdown).toContain('## Trial Aggregation'); + expect(candidateMarkdown).toContain('## Baseline comparison'); + expect(candidateMarkdown.indexOf('## Trial Aggregation')).toBeLessThan( + candidateMarkdown.indexOf('## Baseline comparison'), + ); + expect(candidateMarkdown.indexOf('## Baseline comparison')).toBeLessThan( + candidateMarkdown.indexOf('## Anti-pattern summary'), + ); + }); + }, +); From cfc1a445b70871856672ccf3e44e29bc108c15bd Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 15:34:14 +0000 Subject: [PATCH 16/18] fix: resolve eval compare-baseline test lint errors --- test/integration/evals-compare-baseline.test.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/integration/evals-compare-baseline.test.ts b/test/integration/evals-compare-baseline.test.ts index b0536580..170e5ee3 100644 --- a/test/integration/evals-compare-baseline.test.ts +++ b/test/integration/evals-compare-baseline.test.ts @@ -36,11 +36,12 @@ function runEvalCli(argumentsList: readonly string[]) { return result; } -function readJsonFile(path: string): T { - return JSON.parse(readFileSync(path, 'utf8')) as T; +function readJsonFile(filePath: string): unknown { + return JSON.parse(readFileSync(filePath, 'utf8')) as unknown; } function expectPerCaseComparisonShape(comparison: PerCaseComparison): void { + /* eslint-disable @typescript-eslint/no-unsafe-assignment -- expect.any() and expect.objectContaining() return any */ expect(comparison).toEqual( expect.objectContaining({ caseId: expect.any(String), @@ -75,6 +76,7 @@ function expectPerCaseComparisonShape(comparison: PerCaseComparison): void { verdict: expect.any(String), }), ); + /* eslint-enable @typescript-eslint/no-unsafe-assignment */ } let testRoot = ''; @@ -132,9 +134,9 @@ describe( conditions: ['none'], }); - const baselineReport = readJsonFile( + const baselineReport = readJsonFile( baselineSummary.jsonReportPath, - ); + ) as JsonReport; const candidateResult = runEvalCli([ ...sharedArguments, @@ -153,9 +155,9 @@ describe( conditions: ['none'], }); - const candidateReport = readJsonFile( + const candidateReport = readJsonFile( candidateSummary.jsonReportPath, - ); + ) as JsonReport; const candidateMarkdown = readFileSync( candidateSummary.markdownReportPath, 'utf8', From d77f50a755e326e6e149ed8893d55860737a3c62 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 15:36:43 +0000 Subject: [PATCH 17/18] style: fix formatting in statistics, schemas, and tests --- evals/lib/schemas.ts | 4 +++- evals/lib/statistics.ts | 22 ++++++++++++-------- test/unit/evals/reporting.test.ts | 33 +++++++++++++++++++++++------- test/unit/evals/statistics.test.ts | 22 +++++++++++--------- 4 files changed, 55 insertions(+), 26 deletions(-) diff --git a/evals/lib/schemas.ts b/evals/lib/schemas.ts index 21b3fa25..c01b871e 100644 --- a/evals/lib/schemas.ts +++ b/evals/lib/schemas.ts @@ -970,7 +970,9 @@ export type ConfidenceIntervalSchemaType = z.infer< typeof ConfidenceIntervalSchema >; export type TrialAggregationSchemaType = z.infer; -export type PerCaseComparisonSchemaType = z.infer; +export type PerCaseComparisonSchemaType = z.infer< + typeof PerCaseComparisonSchema +>; export type BaselineOverallSchemaType = z.infer; export type BaselineComparisonSchemaType = z.infer< typeof BaselineComparisonSchema diff --git a/evals/lib/statistics.ts b/evals/lib/statistics.ts index 1648a205..97932368 100644 --- a/evals/lib/statistics.ts +++ b/evals/lib/statistics.ts @@ -146,7 +146,7 @@ function inverseStandardNormal(probability: number): number { const q = probability - 0.5; const r = q * q; return ( - (((((a1 * r + a2) * r + a3) * r + a4) * r + a5) * r + a6) * q / + ((((((a1 * r + a2) * r + a3) * r + a4) * r + a5) * r + a6) * q) / (((((b1 * r + b2) * r + b3) * r + b4) * r + b5) * r + 1) ); } @@ -198,7 +198,10 @@ function zCriticalValue(alpha: number): number { return inverseStandardNormal(1 - alpha / 2); } -function percentile(sortedValues: readonly number[], probability: number): number { +function percentile( + sortedValues: readonly number[], + probability: number, +): number { invariant(sortedValues.length > 0, 'percentile input must not be empty'); invariant( probability >= 0 && probability <= 1, @@ -305,7 +308,8 @@ export function computeConfidenceInterval( const alpha = 1 - confidence; const standardError = stdDev / Math.sqrt(n); - const critical = n < 30 ? tCriticalValue(n - 1, alpha) : zCriticalValue(alpha); + const critical = + n < 30 ? tCriticalValue(n - 1, alpha) : zCriticalValue(alpha); const margin = critical * standardError; return { @@ -318,9 +322,12 @@ export function computeConfidenceInterval( }; } -export function computePassRate( - results: readonly { ok: boolean }[], -): { rate: number; ci: Interval; n: number; passed: number } { +export function computePassRate(results: readonly { ok: boolean }[]): { + rate: number; + ci: Interval; + n: number; + passed: number; +} { const n = results.length; if (n === 0) { return { @@ -352,8 +359,7 @@ export function computePassRate( const denominator = 1 + zSquared / n; const center = (rate + zSquared / (2 * n)) / denominator; const margin = - (z * Math.sqrt((rate * (1 - rate) + zSquared / (4 * n)) / n)) / - denominator; + (z * Math.sqrt((rate * (1 - rate) + zSquared / (4 * n)) / n)) / denominator; return { rate, diff --git a/test/unit/evals/reporting.test.ts b/test/unit/evals/reporting.test.ts index 66f537cb..976703d5 100644 --- a/test/unit/evals/reporting.test.ts +++ b/test/unit/evals/reporting.test.ts @@ -182,7 +182,10 @@ describe('generateJsonReport trial aggregation', () => { it('computes aggregated pass rate, score, and confidence intervals', () => { const results = createMultiTrialResults(); - const report = generateJsonReport(results, createRunMetadata({ totalTrials: 3 })); + const report = generateJsonReport( + results, + createRunMetadata({ totalTrials: 3 }), + ); expect(report.aggregated).toBeDefined(); const aggregated = report.aggregated ?? []; @@ -202,7 +205,11 @@ describe('generateJsonReport trial aggregation', () => { }); const promptScores = [1, 0.5, 0]; - const promptPassRate = computePassRate([{ ok: true }, { ok: false }, { ok: false }]); + const promptPassRate = computePassRate([ + { ok: true }, + { ok: false }, + { ok: false }, + ]); const promptScoreCI = computeConfidenceInterval(promptScores); const promptMean = computeMean(promptScores); const promptStdDev = computeStdDev(promptScores, promptMean); @@ -210,12 +217,20 @@ describe('generateJsonReport trial aggregation', () => { expect(promptAggregation).toBeDefined(); expect(promptAggregation?.passRate).toBeCloseTo(promptPassRate.rate); - expect(promptAggregation?.passRateCI.lower).toBeCloseTo(promptPassRate.ci.lower); - expect(promptAggregation?.passRateCI.upper).toBeCloseTo(promptPassRate.ci.upper); + expect(promptAggregation?.passRateCI.lower).toBeCloseTo( + promptPassRate.ci.lower, + ); + expect(promptAggregation?.passRateCI.upper).toBeCloseTo( + promptPassRate.ci.upper, + ); expect(promptAggregation?.meanScore).toBeCloseTo(promptMean); expect(promptAggregation?.stdDev).toBeCloseTo(promptStdDev); - expect(promptAggregation?.scoreCI.lower).toBeCloseTo(promptScoreCI.ci.lower); - expect(promptAggregation?.scoreCI.upper).toBeCloseTo(promptScoreCI.ci.upper); + expect(promptAggregation?.scoreCI.lower).toBeCloseTo( + promptScoreCI.ci.lower, + ); + expect(promptAggregation?.scoreCI.upper).toBeCloseTo( + promptScoreCI.ci.upper, + ); expect(promptAggregation?.minScore).toBe(0); expect(promptAggregation?.maxScore).toBe(1); @@ -258,7 +273,11 @@ describe('generateMarkdownReport trial aggregation', () => { createRunMetadata({ totalTrials: 3 }), ); - const promptPassRate = computePassRate([{ ok: true }, { ok: false }, { ok: false }]); + const promptPassRate = computePassRate([ + { ok: true }, + { ok: false }, + { ok: false }, + ]); const promptScoreCI = computeConfidenceInterval([1, 0.5, 0]); const executionPassRate = computePassRate([ { ok: false }, diff --git a/test/unit/evals/statistics.test.ts b/test/unit/evals/statistics.test.ts index 93984b90..1b4ff822 100644 --- a/test/unit/evals/statistics.test.ts +++ b/test/unit/evals/statistics.test.ts @@ -99,7 +99,9 @@ describe('computePassRate', () => { }); it('computes the Wilson score interval at 95% confidence', () => { - const summary = computePassRate(createPassResults([true, true, false, true])); + const summary = computePassRate( + createPassResults([true, true, false, true]), + ); expect(summary.rate).toBe(0.75); expect(summary.passed).toBe(3); @@ -110,9 +112,9 @@ describe('computePassRate', () => { it('throws when a result does not expose a boolean ok flag', () => { expect(() => - computePassRate( - [{ ok: true }, { ok: 1 }] as unknown as readonly { ok: boolean }[], - ), + computePassRate([{ ok: true }, { ok: 1 }] as unknown as readonly { + ok: boolean; + }[]), ).toThrow('boolean ok flag'); }); }); @@ -188,12 +190,12 @@ describe('bootstrapPairedCI', () => { }); it('throws for invalid confidence, invalid iterations, and length mismatches', () => { - expect(() => - bootstrapPairedCI([1, 2], [1, 2], { confidence: 0 }), - ).toThrow('confidence must be between 0 and 1'); - expect(() => - bootstrapPairedCI([1, 2], [1, 2], { iterations: 99 }), - ).toThrow('iterations must be an integer greater than or equal to 100'); + expect(() => bootstrapPairedCI([1, 2], [1, 2], { confidence: 0 })).toThrow( + 'confidence must be between 0 and 1', + ); + expect(() => bootstrapPairedCI([1, 2], [1, 2], { iterations: 99 })).toThrow( + 'iterations must be an integer greater than or equal to 100', + ); expect(() => bootstrapPairedCI([1], [1, 2])).toThrow('equal length'); }); }); From 314b4d5cbd33c475042d9a934bfd07b4272c93bc Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 16 Apr 2026 17:56:53 +0000 Subject: [PATCH 18/18] docs: add eval guide skill --- .mux/skills/eval-guide/SKILL.md | 181 ++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 .mux/skills/eval-guide/SKILL.md diff --git a/.mux/skills/eval-guide/SKILL.md b/.mux/skills/eval-guide/SKILL.md new file mode 100644 index 00000000..563558dd --- /dev/null +++ b/.mux/skills/eval-guide/SKILL.md @@ -0,0 +1,181 @@ +--- +name: eval-guide +description: Guide for running statistically meaningful agent-tty evals with trials, parallelism, and A/B comparison. Covers non-determinism baseline, recommended sample sizes, and result interpretation. +--- + +# Eval Guide + +Use this guide when you are trying to answer **"did this skill or prompt change actually help?"** for `agent-tty` evals. + +The short version: **do not trust a single run**. This eval stack now supports multi-trial sampling, parallel execution, trial aggregation, and paired baseline comparison because the underlying model behavior is noisy enough that one pass/fail result is not decision-grade. + +## 1. What we learned about eval non-determinism + +- Identical serial reruns showed a **~15-17% pass/fail flip rate** in practice, across both Codex and Claude runs. +- Scores moved even more often than hard pass/fail: **~30-39% of identical reruns changed score**. +- The movement was directionally balanced, which is the important point: this looked like **noise**, not a systematic drift up or down. +- Cross-provider checks reinforced that conclusion: in the parallel safety analysis, **Codex and Claude shared zero common regressions**, which is strong evidence that parallelism itself was not introducing consistent failures. +- Treat these findings as the baseline noise floor. If your "improvement" is smaller than that noise, it is not persuasive. + +## 2. Run evals with enough statistical power + +Always set `--trials` for real prompt or skill experiments. + +Recommended trial counts: + +- **Prompt lane:** `--trials 5` to `--trials 10` +- **Execution lane:** `--trials 3` +- **Dogfood lane:** `--trials 2` to `--trials 3` + +Use concurrency to keep those sample sizes affordable: + +- Start with **`--concurrency 4`** for real-provider runs. +- Recent measurements showed about a **3.4x wall-clock speedup** at that setting. +- Leave `--concurrency 1` only when you explicitly want fully serial behavior. + +When `--trials` is greater than `1`, reports automatically include **Trial Aggregation** in `report.md` and `report.json`, including per-case: + +- pass rate, +- pass-rate confidence interval, +- mean score, +- score confidence interval, +- standard deviation, +- and min/max score. + +## 3. Compare before vs. after a skill change + +Use a paired baseline comparison whenever you want to know whether a change helped. The comparison report uses **paired bootstrap confidence intervals** and paired win/loss/tie counts, so it is much more reliable than eyeballing two single runs. + +1. Run a **baseline** on the exact lane, cases, provider, model, and trial count you care about. +2. Save the baseline `report.json` path. +3. Make the skill or prompt change. +4. Run the **candidate** with `--compare-baseline `. +5. Read the comparison verdicts: `improved`, `regressed`, or `inconclusive`. + +Practical reading rules: + +- Do **not** call something better just because the mean moved in the right direction. +- Treat a win as meaningful only when the paired CI excludes `0` **and** the effect is practically large enough to matter. +- In the current implementation, the practical cutoffs are **`0.05` score delta** and, for overall pass rate, **`0.05` absolute pass-rate delta**. +- Tiny but statistically significant deltas are still not worth celebrating. + +A reliable prompt-lane A/B loop looks like this: + +```bash +BASELINE_JSON=$(npx tsx evals/run.ts \ + --provider codex \ + --model gpt-5.4 \ + --lane prompt \ + --condition self-load \ + --trials 5 \ + --concurrency 4 \ + --output evals/reports/prompt-baseline \ + --json | jq -r '.jsonReportPath') + +# edit the skill or prompt + +npx tsx evals/run.ts \ + --provider codex \ + --model gpt-5.4 \ + --lane prompt \ + --condition self-load \ + --trials 5 \ + --concurrency 4 \ + --output evals/reports/prompt-candidate \ + --compare-baseline "$BASELINE_JSON" \ + --json +``` + +## 4. Interpret results correctly + +- **`inconclusive` is the default, healthy outcome when nothing meaningful changed.** In our same-skill sanity check, **23 of 24 cases were inconclusive** and the paired win/loss/tie total was **14W / 15L / 43T**. +- A few false positives are normal. At **95% confidence**, a **~1 in 24** spurious finding in a 24-case sweep is not surprising even when nothing changed. +- Read **wins / losses / ties** alongside the CI. They give an intuitive sense of whether the candidate is consistently helping or just bouncing around. +- If a comparison is mostly ties with a wide CI, the result is noise-dominated; add more trials before making a claim. +- If you only have two standalone reports and no paired baseline comparison, diff them only after stripping timing- and path-specific fields. That can catch large shifts, but it is much less trustworthy than `--trials` plus `--compare-baseline`. + +## 5. Concurrency is safe and should be used + +- `--concurrency 1` remains the default and preserves serial behavior. +- **Parallelism did not introduce regressions** in the safety checks; observed parallel flip rates were at or below the normal serial noise floor. +- `--concurrency 4-20` is a reasonable operating range when provider limits and budget allow. +- The current runner can execute **all three lanes concurrently** when concurrency is above `1`. +- Execution and dogfood work items run in **isolated temp homes/output directories** and clean up in `finally` blocks, so parallel runs do not share session state. + +Use serial mode only when debugging; use parallel mode when sampling. + +## 6. Quick reference commands + +### Prompt-lane experiment + +```bash +npx tsx evals/run.ts \ + --provider claude \ + --model claude-opus-4-6 \ + --lane prompt \ + --condition self-load \ + --trials 5 \ + --concurrency 4 \ + --output evals/reports/prompt-self-load +``` + +### Execution-lane check after workflow changes + +```bash +npx tsx evals/run.ts \ + --provider codex \ + --model gpt-5.4 \ + --lane execution \ + --case hello-prompt \ + --case resize-demo \ + --trials 3 \ + --concurrency 4 \ + --output evals/reports/execution-smoke +``` + +### Dogfood proof-bundle spot check + +```bash +npx tsx evals/run.ts \ + --provider claude \ + --model claude-opus-4-6 \ + --lane dogfood \ + --case exploratory-qa \ + --case evidence-completeness \ + --trials 2 \ + --concurrency 4 \ + --output evals/reports/dogfood-sample +``` + +### Full before/after comparison + +```bash +BASELINE_JSON=$(npx tsx evals/run.ts \ + --provider codex \ + --model gpt-5.4 \ + --lane all \ + --condition all \ + --trials 3 \ + --concurrency 4 \ + --output evals/reports/baseline \ + --json | jq -r '.jsonReportPath') + +npx tsx evals/run.ts \ + --provider codex \ + --model gpt-5.4 \ + --lane all \ + --condition all \ + --trials 3 \ + --concurrency 4 \ + --output evals/reports/candidate \ + --compare-baseline "$BASELINE_JSON" \ + --json +``` + +### Plumbing smoke test + +```bash +npx tsx evals/run.ts --provider stub --lane prompt --trials 3 --concurrency 4 +``` + +Use `stub` to validate wiring, not to judge whether a real-provider prompt or skill change helped.