diff --git a/.gitignore b/.gitignore index 37302e4b..45aae921 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ pids/ tmp/ .tmp/ .cache/ + +# Eval reports (generated per-run under evals/reports/{timestamp}/) +evals/reports/ diff --git a/evals/README.md b/evals/README.md index 121c7d03..7acaa4b0 100644 --- a/evals/README.md +++ b/evals/README.md @@ -28,11 +28,14 @@ npx tsx evals/run.ts --provider codex --lane all # Specific cases npx tsx evals/run.ts --provider stub --lane execution --case hello-prompt --case resize-demo + +# Compare conditions in one dry run +npx tsx evals/run.ts --provider stub --lane execution --condition none --condition preloaded --dry-run ``` Useful flags: -- `--condition ` — `none`, `self-load`, `preloaded`, `stale`, or `all` (default: `all`) +- `--condition ` — `none`, `self-load`, `preloaded`, `stale`, or `all` (default: `all`). May be repeated. - `--output ` — output base directory (default: `evals/reports/{timestamp}`) - `--dry-run` — list the case/condition matrix without invoking a provider - `--json` — emit only a JSON summary to stdout @@ -47,6 +50,9 @@ npx tsx evals/run.ts --provider stub --lane all --dry-run # Restrict to one condition npx tsx evals/run.ts --provider stub --lane prompt --condition self-load +# Compare two conditions in one run +npx tsx evals/run.ts --provider stub --lane prompt --condition none --condition preloaded + # Write results under a custom directory npx tsx evals/run.ts --provider stub --lane execution --output evals/reports/local-smoke ``` @@ -199,6 +205,45 @@ Prompt cases currently cover positive routing, negative routing, and explicit an | `run-command` | `session` | `hello-prompt` | all four | Use `agent-tty run` instead of simulated typing, then snapshot the result. | | `doctor-gated` | `artifact` | `hello-prompt` | all four | Run `doctor --json` before taking a renderer-dependent screenshot. | +#### Lane B readiness and renderer requirements + +Lane B coverage is intentionally uneven today. Use the **readiness tier** to decide where to add smoke coverage next, and use the **renderer requirement** column to decide whether a failure is likely a skill regression or an environment problem. + +- **`battle-tested`** — safest execution smoke cases today. +- **`non-renderer unproven`** — next expansion batch before spending more time on renderer-heavy cases. `export-proof` stays in this rollout bucket even though its required `.webm` export is renderer-backed. +- **`renderer-optional`** — the case can still pass without Playwright/Chromium, but renderer-backed artifacts are useful when available. +- **`renderer-required`** — failing Playwright/Chromium/ghostty-web checks should be treated as an `environment-blocked` result, not as a skill regression. + +| Case ID | Readiness tier | Renderer requirement | Notes | +| ----------------- | ----------------------- | -------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `hello-prompt` | `battle-tested` | `none` | Core create → input → wait → snapshot → cleanup loop. | +| `crash-recovery` | `battle-tested` | `none` | Core crash inspection and cleanup flow. | +| `run-command` | `battle-tested` | `none` | Core programmatic input flow via `agent-tty run`. | +| `resize-demo` | `non-renderer unproven` | `none` | Snapshot-only resize verifier; high-value next smoke target. | +| `alt-screen-demo` | `non-renderer unproven` | `none` | Event-log plus snapshot proof still needs broader smoke coverage. | +| `scrollback-demo` | `non-renderer unproven` | `none` | Scrollback snapshot proof still needs broader smoke coverage. | +| `unicode-grid` | `non-renderer unproven` | `none` | Semantic snapshot verifier still needs broader smoke coverage. | +| `export-proof` | `non-renderer unproven` | `required` | Still part of the next smoke batch, but the required `.webm` export is renderer-backed. | +| `color-grid` | `renderer-optional` | `optional` | Screenshot evidence is preferred when renderer support exists, but non-renderer verification can still satisfy the case. | +| `doctor-gated` | `renderer-required` | `required` | Must run `doctor --json` first; missing renderer support blocks the required screenshot. | + +**Expansion order:** keep prioritizing `resize-demo`, `alt-screen-demo`, `scrollback-demo`, `unicode-grid`, and `export-proof` before spending more time on `color-grid` or `doctor-gated`. Within that batch, `export-proof` is the one case that still needs renderer-backed WebM export, so preflight renderer availability first. + +To avoid mistaking a missing browser/runtime dependency for a skill regression, pair execution-lane dry runs with a renderer preflight: + +```sh +# Preview the case/condition matrix +npx tsx evals/run.ts --provider stub --lane execution --dry-run + +# Check renderer prerequisites before renderer-backed cases +npx tsx src/cli/main.ts doctor --json + +# If doctor reports a missing Playwright browser cache +npx playwright install chromium +``` + +`--dry-run` currently tells you which execution cases and conditions will run, but it does not yet annotate renderer needs inline. Use the table above to decide whether `color-grid`, `export-proof`, or `doctor-gated` should be treated as renderer-backed for your environment. + ### Dogfood lane | Case ID | Category | Fixture | Conditions | Description | diff --git a/evals/execution/cases/shared.ts b/evals/execution/cases/shared.ts index 57b4a272..6c6e5838 100644 --- a/evals/execution/cases/shared.ts +++ b/evals/execution/cases/shared.ts @@ -21,6 +21,93 @@ const DEFAULT_MAX_WALL_CLOCK_MS = 90_000; export const ALL_EXECUTION_CONDITIONS: SkillCondition[] = [...SKILL_CONDITIONS]; +export type ExecutionRendererRequirement = 'none' | 'optional' | 'required'; + +export type ExecutionReadinessTier = + | 'battle-tested' + | 'non-renderer-unproven' + | 'renderer-optional' + | 'renderer-required'; + +export interface ExecutionCaseCoverageMetadata { + readinessTier: ExecutionReadinessTier; + rendererRequirement: ExecutionRendererRequirement; + summary: string; +} + +const EXECUTION_CASE_COVERAGE_BY_ID = { + 'hello-prompt': { + readinessTier: 'battle-tested', + rendererRequirement: 'none', + summary: 'Battle-tested create → input → wait → snapshot → cleanup flow.', + }, + 'crash-recovery': { + readinessTier: 'battle-tested', + rendererRequirement: 'none', + summary: 'Battle-tested crash inspection and cleanup flow.', + }, + 'run-command': { + readinessTier: 'battle-tested', + rendererRequirement: 'none', + summary: 'Battle-tested programmatic input flow via agent-tty run.', + }, + 'resize-demo': { + readinessTier: 'non-renderer-unproven', + rendererRequirement: 'none', + summary: + 'Snapshot-only resize verification still needs broader smoke coverage.', + }, + 'alt-screen-demo': { + readinessTier: 'non-renderer-unproven', + rendererRequirement: 'none', + summary: + 'Alt-screen event-log and snapshot proof still needs broader smoke coverage.', + }, + 'scrollback-demo': { + readinessTier: 'non-renderer-unproven', + rendererRequirement: 'none', + summary: 'Scrollback snapshot proof still needs broader smoke coverage.', + }, + 'unicode-grid': { + readinessTier: 'non-renderer-unproven', + rendererRequirement: 'none', + summary: + 'Unicode snapshot verification still needs broader smoke coverage.', + }, + 'export-proof': { + readinessTier: 'non-renderer-unproven', + rendererRequirement: 'required', + summary: + 'Still in the next smoke batch, but the required WebM export is renderer-backed.', + }, + 'color-grid': { + readinessTier: 'renderer-optional', + rendererRequirement: 'optional', + summary: + 'Can pass with non-renderer verification, but renderer-backed screenshots remain useful reviewer evidence.', + }, + 'doctor-gated': { + readinessTier: 'renderer-required', + rendererRequirement: 'required', + summary: + 'Must pass doctor --json renderer checks before the required screenshot capture.', + }, +} as const satisfies Record; + +export const EXECUTION_CASE_COVERAGE = EXECUTION_CASE_COVERAGE_BY_ID; + +export function getExecutionCaseCoverage( + caseId: string, +): ExecutionCaseCoverageMetadata { + invariant( + Object.prototype.hasOwnProperty.call(EXECUTION_CASE_COVERAGE_BY_ID, caseId), + `Missing execution case coverage metadata for ${caseId}`, + ); + return EXECUTION_CASE_COVERAGE_BY_ID[ + caseId as keyof typeof EXECUTION_CASE_COVERAGE_BY_ID + ]; +} + export function anyOf(...patterns: string[]): string { invariant(patterns.length > 0, 'anyOf() requires at least one pattern'); return `(?:${patterns.join('|')})`; diff --git a/evals/execution/runner.ts b/evals/execution/runner.ts index 187539a0..d61546e6 100644 --- a/evals/execution/runner.ts +++ b/evals/execution/runner.ts @@ -46,6 +46,10 @@ import { resizeDemoCase } from './cases/resize-demo.js'; import { runCommandCase } from './cases/run-command.js'; import { scrollbackDemoCase } from './cases/scrollback-demo.js'; import { unicodeGridCase } from './cases/unicode-grid.js'; +import { + EXECUTION_CASE_COVERAGE, + getExecutionCaseCoverage, +} from './cases/shared.js'; import type { VerifierResult } from './verifiers/index.js'; import { verify, verifyArtifactExists } from './verifiers/index.js'; @@ -106,6 +110,31 @@ const STALE_EXECUTION_SKILL_TEXT = [ 'Some of the guidance above is wrong for this repository snapshot. Verify commands against the current CLI behavior while completing the task.', ].join('\n'); +const ENVIRONMENT_BLOCKED_ERROR_CLASS = 'environment-blocked'; +const RENDERER_ENVIRONMENT_HINT = + 'Run `npx tsx src/cli/main.ts doctor --json` before retrying. If the renderer checks report a missing browser cache, run `npx playwright install chromium`.'; +const RENDERER_ENVIRONMENT_ERROR_PATTERNS = [ + /Playwright browser cache not found/iu, + /Run 'npx playwright install chromium' to install\./iu, + /\bplaywright unavailable\b/iu, + /\bplaywright not installed\b/iu, + /\bplaywright import failed\b/iu, + /\bghostty-web unavailable\b/iu, + /\bbrowser launch failed\b/iu, + /\bscreenshot smoke test failed\b/iu, + /Failed to boot or replay renderer/iu, + /Executable doesn't exist/iu, +] as const; +const RENDERER_DOCTOR_FAILURE_PATTERNS = [ + /"name"\s*:\s*"playwright_available"[\s\S]*?"status"\s*:\s*"fail"/iu, + /"name"\s*:\s*"browser_cache_accessible"[\s\S]*?"status"\s*:\s*"(?:fail|skip)"/iu, + /"name"\s*:\s*"browser_launch"[\s\S]*?"status"\s*:\s*"fail"/iu, + /"name"\s*:\s*"ghostty_web_available"[\s\S]*?"status"\s*:\s*"fail"/iu, + /"name"\s*:\s*"screenshot_viable"[\s\S]*?"status"\s*:\s*"fail"/iu, + /"name"\s*:\s*"screenshot"[\s\S]*?"status"\s*:\s*"(?:unavailable|degraded)"/iu, + /"name"\s*:\s*"record-export-webm"[\s\S]*?"status"\s*:\s*"(?:unavailable|degraded)"/iu, +] as const; + async function readSkillFile(relativePath: string): Promise { const content = await readFile( new URL(relativePath, import.meta.url), @@ -174,6 +203,64 @@ function assertExecutionCaseInventory( } } +assertExecutionCaseCoverageMetadata(EXECUTION_CASES); + +function assertExecutionCaseCoverageMetadata( + cases: readonly ExecutionEvalCase[], +): void { + const caseIds = new Set(cases.map((evalCase) => evalCase.id)); + const coverageIds = Object.keys(EXECUTION_CASE_COVERAGE); + + for (const caseId of caseIds) { + invariant( + coverageIds.includes(caseId), + `Missing execution coverage metadata for case ${caseId}`, + ); + } + + for (const coverageId of coverageIds) { + invariant( + caseIds.has(coverageId), + `Execution coverage metadata references unknown case ${coverageId}`, + ); + } +} + +function matchesAnyPattern(text: string, patterns: readonly RegExp[]): boolean { + return patterns.some((pattern) => pattern.test(text)); +} + +function detectRendererEnvironmentBlock( + evalCase: ExecutionEvalCase, + transcript: string, + providerErrorMessage: string | undefined, +): string | undefined { + const coverage = getExecutionCaseCoverage(evalCase.id); + if (coverage.rendererRequirement !== 'required') { + return undefined; + } + + const combinedText = [transcript, providerErrorMessage] + .filter((value): value is string => value !== undefined && value.length > 0) + .join('\n'); + if (combinedText.length === 0) { + return undefined; + } + + if ( + !matchesAnyPattern(combinedText, RENDERER_ENVIRONMENT_ERROR_PATTERNS) && + !matchesAnyPattern(combinedText, RENDERER_DOCTOR_FAILURE_PATTERNS) + ) { + return undefined; + } + + return [ + `Renderer-backed case ${evalCase.id} was blocked by missing or unhealthy Playwright/Chromium/ghostty-web dependencies.`, + `Case note: ${coverage.summary}`, + RENDERER_ENVIRONMENT_HINT, + ].join(' '); +} + function safeRatio(numerator: number, denominator: number): number { if (denominator === 0) { return 1; @@ -812,6 +899,7 @@ function buildResult( condition: SkillCondition, trial: number, runtime: ProviderRuntimeInfo | undefined, + transcript: string, normalizedOutput: NormalizedProviderOutput, providerOk: boolean, startedAt: string, @@ -839,16 +927,30 @@ function buildResult( normalizedOutput, ); const modelId = resolveModelId(metadata, runtime); + const summarizedFailure = summarizeFailureReasons( + providerOk, + verifierResults, + workflowChecks, + antiPatternFindings, + artifactResults, + providerErrorMessage, + ); + const environmentBlockedMessage = detectRendererEnvironmentBlock( + evalCase, + transcript, + providerErrorMessage, + ); + const effectiveErrorClass = + environmentBlockedMessage === undefined + ? errorClass + : ENVIRONMENT_BLOCKED_ERROR_CLASS; const failureMessage = scored.ok ? undefined - : summarizeFailureReasons( - providerOk, - verifierResults, - workflowChecks, - antiPatternFindings, - artifactResults, - providerErrorMessage, - ); + : environmentBlockedMessage === undefined + ? summarizedFailure + : [environmentBlockedMessage, summarizedFailure] + .filter((value) => value.length > 0) + .join(' '); const result: EvalResult = { runId: metadata.runId, @@ -873,7 +975,9 @@ function buildResult( ...(bundlePath === undefined ? {} : { bundlePath }), ...(eventLogPath === undefined ? {} : { eventLogPath }), normalizedOutput, - ...(scored.ok || errorClass === undefined ? {} : { errorClass }), + ...(scored.ok || effectiveErrorClass === undefined + ? {} + : { errorClass: effectiveErrorClass }), ...(failureMessage === undefined ? {} : { errorMessage: failureMessage }), startedAt, completedAt, @@ -936,6 +1040,7 @@ async function runSingleExecutionCase( condition, trial, runtime, + transcript, normalizedOutput, false, invocationStartedAt, @@ -1008,6 +1113,7 @@ async function runSingleExecutionCase( condition, trial, providerResult.runtime, + transcript, providerResult.normalized, providerResult.ok, providerResult.startedAt, @@ -1049,6 +1155,7 @@ async function runSingleExecutionCase( condition, trial, runtime, + transcript, normalizedOutput, false, invocationStartedAt, @@ -1086,7 +1193,6 @@ async function runSingleExecutionCase( } } } - /** Return all registered execution-lane cases in deterministic order. */ export function getAllExecutionCases(): ExecutionEvalCase[] { assertExecutionCaseInventory(EXECUTION_CASES); diff --git a/evals/lib/antiPatterns.ts b/evals/lib/antiPatterns.ts index 3b4bd835..1ffc2ef0 100644 --- a/evals/lib/antiPatterns.ts +++ b/evals/lib/antiPatterns.ts @@ -24,19 +24,25 @@ const SHELL_TOOL_NAME_PATTERN = const SHELL_TOOL_FALLBACK_TYPE_PATTERN = /^(?:command_execution|function_call|tool_use)$/iu; const SHELL_TOOL_HINT_KEYS = ['script', 'command', 'cmd'] as const; -const SCANNABLE_TOOL_TEXT_KEYS = [ +const SCANNABLE_TOOL_INPUT_TEXT_KEYS = [ 'script', 'command', 'cmd', 'arguments', 'text', 'content', +] as const; +const SCANNABLE_TOOL_OUTPUT_TEXT_KEYS = [ 'output', 'stdout', 'stderr', 'result', 'aggregated_output', ] as const; +const SCANNABLE_TOOL_TEXT_KEYS = [ + ...SCANNABLE_TOOL_INPUT_TEXT_KEYS, + ...SCANNABLE_TOOL_OUTPUT_TEXT_KEYS, +] as const; const AGENT_TTY_TOOL_INPUT_PATTERN = new RegExp( AGENT_TTY_COMMAND_TEXT_PATTERN, 'iu', @@ -198,7 +204,7 @@ export function compileAntiPatternRegex(pattern: string): RegExp { } /** - * Build a scannable transcript from shell-style tool-call inputs and outputs. + * Build a scannable transcript from shell-style tool-call inputs. */ export function buildScannableTranscript( normalized: NormalizedProviderOutput, @@ -216,9 +222,9 @@ export function countAgentTtyCalls( normalized: NormalizedProviderOutput, ): number { return collectShellToolCalls(normalized).reduce((count, toolCall) => { - return extractScannableToolCallTexts(toolCall).some((fragment) => - AGENT_TTY_TOOL_INPUT_PATTERN.test(fragment), - ) + return extractScannableToolCallTexts(toolCall, { + includeOutputs: true, + }).some((fragment) => AGENT_TTY_TOOL_INPUT_PATTERN.test(fragment)) ? count + 1 : count; }, 0); @@ -393,7 +399,7 @@ function isShellToolCall(toolCall: Record): boolean { return ( typeof toolType === 'string' && SHELL_TOOL_FALLBACK_TYPE_PATTERN.test(toolType) && - extractScannableToolCallTexts(toolCall).length > 0 + extractScannableToolCallTexts(toolCall, { includeOutputs: true }).length > 0 ); } @@ -409,12 +415,13 @@ function hasShellToolHints(value: unknown): boolean { function extractScannableToolCallTexts( toolCall: Record, + options: { includeOutputs?: boolean } = {}, ): string[] { + const includeOutputs = options.includeOutputs ?? false; const fragments: string[] = []; for (const value of [ toolCall.input, - toolCall.output, { script: toolCall.script, command: toolCall.command, @@ -423,24 +430,44 @@ function extractScannableToolCallTexts( text: toolCall.text, content: toolCall.content, }, - { - output: toolCall.output, - stdout: toolCall.stdout, - stderr: toolCall.stderr, - result: toolCall.result, - aggregated_output: toolCall.aggregated_output, - }, ]) { - const fragment = extractScannableToolText(value); + const fragment = extractScannableToolText( + value, + SCANNABLE_TOOL_INPUT_TEXT_KEYS, + ); if (fragment.length > 0 && !fragments.includes(fragment)) { fragments.push(fragment); } } + if (includeOutputs) { + for (const value of [ + toolCall.output, + { + output: toolCall.output, + stdout: toolCall.stdout, + stderr: toolCall.stderr, + result: toolCall.result, + aggregated_output: toolCall.aggregated_output, + }, + ]) { + const fragment = extractScannableToolText( + value, + SCANNABLE_TOOL_TEXT_KEYS, + ); + if (fragment.length > 0 && !fragments.includes(fragment)) { + fragments.push(fragment); + } + } + } + return fragments; } -function extractScannableToolText(value: unknown): string { +function extractScannableToolText( + value: unknown, + scannableKeys: readonly string[] = SCANNABLE_TOOL_TEXT_KEYS, +): string { if (typeof value === 'string') { return value.trim(); } @@ -461,7 +488,7 @@ function extractScannableToolText(value: unknown): string { const fragments: string[] = []; for (const entry of value) { - const fragment = extractScannableToolText(entry); + const fragment = extractScannableToolText(entry, scannableKeys); if (fragment.length > 0 && !fragments.includes(fragment)) { fragments.push(fragment); } @@ -476,8 +503,8 @@ function extractScannableToolText(value: unknown): string { const fragments: string[] = []; - for (const key of SCANNABLE_TOOL_TEXT_KEYS) { - const fragment = extractScannableToolText(value[key]); + for (const key of scannableKeys) { + const fragment = extractScannableToolText(value[key], scannableKeys); if (fragment.length > 0 && !fragments.includes(fragment)) { fragments.push(fragment); } diff --git a/evals/lib/reporting.ts b/evals/lib/reporting.ts index c504bb71..d45ed889 100644 --- a/evals/lib/reporting.ts +++ b/evals/lib/reporting.ts @@ -14,6 +14,7 @@ import type { AntiPatternSeverity, BaselineComparison, ComparisonMetrics, + ConditionComparisonSummary, EvalLane, EvalResult, JsonReport, @@ -82,6 +83,11 @@ export function generateJsonReport( const normalizedComparisons = normalizeComparisonMetrics(comparisonMetrics); const aggregateMetrics = buildAggregateMetrics(sortedResults); const aggregate = toAggregateMetricsCore(aggregateMetrics); + const conditionComparisonSummary = buildConditionComparisonSummary( + sortedResults, + metadata.conditions, + normalizedComparisons, + ); const laneSummaries = buildLaneSummaries(sortedResults, metadata.lanes); const providerComparisons = buildPairwiseProviderComparisons( sortedResults, @@ -101,6 +107,9 @@ export function generateJsonReport( const coreReport = { metadata, aggregate, + ...(conditionComparisonSummary === undefined + ? {} + : { conditionComparisonSummary }), comparisons: normalizedComparisons, results: sortedResults, ...(providerComparison === undefined ? {} : { providerComparison }), @@ -162,6 +171,11 @@ export function generateMarkdownReport( `- Run ID: \`${sanitizeInline(report.metadata.runId)}\``, `- Created: \`${sanitizeInline(report.metadata.createdAt)}\``, `- Repo root: \`${sanitizeInline(report.metadata.repoRoot)}\``, + ...(report.metadata.outputBaseDir === undefined + ? [] + : [ + `- Output directory: \`${sanitizeInline(report.metadata.outputBaseDir)}\``, + ]), `- Providers: ${formatList(providers.map((providerId) => `\`${providerId}\``))}`, `- Models: ${formatList( [...report.metadata.models] @@ -187,6 +201,13 @@ export function generateMarkdownReport( sections.push('', formatSummaryTable(report.aggregateMetrics)); + if (report.conditionComparisonSummary !== undefined) { + sections.push( + '', + ...buildConditionComparisonMarkdown(report.conditionComparisonSummary), + ); + } + if (lanes.length > 0) { sections.push('', '## Lane breakdown', ''); sections.push( @@ -221,85 +242,6 @@ export function generateMarkdownReport( ); } - if (report.comparisons.length > 0) { - sections.push('', '## Condition comparison', ''); - sections.push( - buildMarkdownTable( - ['Metric', 'Value'], - ['left', 'right'], - [ - ['Compared groups', String(report.comparisons.length)], - [ - 'Compared cases', - String( - report.comparisons.reduce( - (count, metric) => count + metric.totalCompared, - 0, - ), - ), - ], - [ - 'Realized skill lift', - formatComparisonValue( - averageDefined( - report.comparisons, - (metric) => metric.realizedSkillLift, - ), - ), - ], - [ - 'Oracle skill lift', - formatComparisonValue( - averageDefined( - report.comparisons, - (metric) => metric.oracleSkillLift, - ), - ), - ], - [ - 'Routing gap', - formatComparisonValue( - averageDefined(report.comparisons, (metric) => metric.routingGap), - ), - ], - [ - 'Stale-skill harm', - formatComparisonValue( - averageDefined( - report.comparisons, - (metric) => metric.staleSkillHarm, - ), - ), - ], - [ - 'Regression rate', - formatComparisonValue( - averageDefined( - report.comparisons, - (metric) => metric.regressionRate, - ), - ), - ], - [ - 'Unlock rate', - formatComparisonValue( - averageDefined(report.comparisons, (metric) => metric.unlockRate), - ), - ], - [ - 'Routing efficiency', - formatComparisonValue( - averageDefined( - report.comparisons, - (metric) => metric.routingEfficiency, - ), - ), - ], - ], - ), - ); - } - sections.push('', '## Failed cases', ''); if (failedResults.length === 0) { sections.push('- None.'); @@ -718,6 +660,130 @@ function buildOverallComparisonVerdict( return 'inconclusive'; } +function buildConditionComparisonMarkdown( + summary: ConditionComparisonSummary, +): string[] { + invariant( + summary.comparedConditions.length > 1, + 'Condition comparison markdown requires at least two conditions', + ); + + return [ + '## Condition comparison', + '', + `- Compared conditions: ${formatList( + summary.comparedConditions.map((condition) => `\`${condition}\``), + )}`, + `- Compared groups / cases: ${String(summary.comparedGroups)} / ${String(summary.comparedCases)}`, + '', + buildMarkdownTable( + ['Condition', 'Total', 'Passed', 'Failed', 'Pass Rate', 'Mean'], + ['left', 'right', 'right', 'right', 'right', 'right'], + summary.conditionBreakdown.map((row) => [ + `\`${row.condition}\``, + String(row.totalCases), + String(row.passed), + String(row.failed), + formatPercent(row.passRate), + formatScore(row.averageScore), + ]), + ), + '', + buildMarkdownTable( + ['Delta', 'Value'], + ['left', 'right'], + [ + [ + 'Realized skill lift', + formatComparisonValue(summary.keyDeltas.realizedSkillLift), + ], + [ + 'Oracle skill lift', + formatComparisonValue(summary.keyDeltas.oracleSkillLift), + ], + ['Routing gap', formatComparisonValue(summary.keyDeltas.routingGap)], + [ + 'Stale-skill harm', + formatComparisonValue(summary.keyDeltas.staleSkillHarm), + ], + [ + 'Regression rate', + formatComparisonValue(summary.keyDeltas.regressionRate), + ], + ['Unlock rate', formatComparisonValue(summary.keyDeltas.unlockRate)], + [ + 'Routing efficiency', + formatComparisonValue(summary.keyDeltas.routingEfficiency), + ], + ], + ), + ]; +} + +function buildConditionComparisonSummary( + results: EvalResult[], + metadataConditions: readonly SkillCondition[], + comparisons: ComparisonMetrics[], +): ConditionComparisonSummary | undefined { + if (comparisons.length === 0) { + return undefined; + } + + const comparedConditions = collectConditions(results, metadataConditions); + invariant( + comparedConditions.length > 1, + 'Condition comparison summary requires at least two conditions', + ); + + const conditionBreakdown = comparedConditions.map((condition) => { + const metrics = buildAggregateMetrics( + results.filter((result) => result.condition === condition), + ); + return { + condition, + totalCases: metrics.totalCases, + passed: metrics.passed, + failed: metrics.failed, + passRate: metrics.passRate, + averageScore: metrics.averageScore, + }; + }); + + return { + comparedConditions, + comparedGroups: comparisons.length, + comparedCases: comparisons.reduce( + (count, metric) => count + metric.totalCompared, + 0, + ), + conditionBreakdown, + keyDeltas: { + realizedSkillLift: averageDefined( + comparisons, + (metric) => metric.realizedSkillLift, + ), + oracleSkillLift: averageDefined( + comparisons, + (metric) => metric.oracleSkillLift, + ), + routingGap: averageDefined(comparisons, (metric) => metric.routingGap), + staleSkillHarm: averageDefined( + comparisons, + (metric) => metric.staleSkillHarm, + ), + regressionRate: averageDefined( + comparisons, + (metric) => metric.regressionRate, + ), + unlockRate: averageDefined(comparisons, (metric) => metric.unlockRate), + routingEfficiency: averageDefined( + comparisons, + (metric) => metric.routingEfficiency, + ), + }, + }; +} + /** * Build deterministic pairwise provider comparisons with per-condition * aggregate breakdowns. @@ -787,6 +853,12 @@ function assertMetadata(metadata: RunMetadata): void { typeof metadata.runId === 'string' && metadata.runId.length > 0, 'Run metadata runId must be a non-empty string', ); + invariant( + metadata.outputBaseDir === undefined || + (typeof metadata.outputBaseDir === 'string' && + metadata.outputBaseDir.length > 0), + 'Run metadata outputBaseDir must be a non-empty string when provided', + ); invariant( Array.isArray(metadata.providers), 'Run metadata providers must be an array', diff --git a/evals/lib/schemas.ts b/evals/lib/schemas.ts index c01b871e..ea2c231d 100644 --- a/evals/lib/schemas.ts +++ b/evals/lib/schemas.ts @@ -388,6 +388,7 @@ export const RunMetadataSchema = z runId: NonEmptyStringSchema, createdAt: IsoTimestampSchema, repoRoot: PathStringSchema, + outputBaseDir: PathStringSchema.optional(), providers: z.array(NonEmptyStringSchema), models: z.array(NonEmptyStringSchema), lanes: z.array(EvalLaneSchema), @@ -667,6 +668,56 @@ export const AggregateMetricsSchema = z } }); +export const ConditionAggregateSummarySchema = z + .object({ + condition: SkillConditionSchema, + totalCases: NonNegativeIntSchema, + passed: NonNegativeIntSchema, + failed: NonNegativeIntSchema, + passRate: UnitIntervalSchema, + averageScore: NonNegativeNumberSchema, + }) + .strict() + .superRefine((obj, ctx) => { + if (obj.passed + obj.failed !== obj.totalCases) { + ctx.addIssue({ + code: 'custom', + message: 'passed + failed must equal totalCases', + path: ['totalCases'], + }); + } + }); + +export const ConditionComparisonSummarySchema = z + .object({ + comparedConditions: z.array(SkillConditionSchema).min(2), + comparedGroups: NonNegativeIntSchema, + comparedCases: NonNegativeIntSchema, + conditionBreakdown: z.array(ConditionAggregateSummarySchema).min(2), + keyDeltas: z + .object({ + realizedSkillLift: FiniteNumberSchema.optional(), + oracleSkillLift: FiniteNumberSchema.optional(), + routingGap: FiniteNumberSchema.optional(), + staleSkillHarm: FiniteNumberSchema.optional(), + regressionRate: FiniteNumberSchema.optional(), + unlockRate: FiniteNumberSchema.optional(), + routingEfficiency: FiniteNumberSchema.optional(), + }) + .strict(), + }) + .strict() + .superRefine((obj, ctx) => { + if (obj.conditionBreakdown.length !== obj.comparedConditions.length) { + ctx.addIssue({ + code: 'custom', + message: + 'conditionBreakdown length must match comparedConditions length', + path: ['conditionBreakdown'], + }); + } + }); + const ProviderAggregateSchema = z .object({ providerId: NonEmptyStringSchema, @@ -770,6 +821,7 @@ export const JsonReportSchema = z .object({ metadata: RunMetadataSchema, aggregate: AggregateMetricsSchema, + conditionComparisonSummary: ConditionComparisonSummarySchema.optional(), comparisons: z.array(ComparisonMetricsSchema), results: z.array(EvalResultSchema), providerComparison: ProviderComparisonReportSchema.optional(), diff --git a/evals/lib/scoring.ts b/evals/lib/scoring.ts index e80da660..9355a602 100644 --- a/evals/lib/scoring.ts +++ b/evals/lib/scoring.ts @@ -31,6 +31,14 @@ const NEGATION_CONTEXT_PATTERNS = Object.freeze([ /\bunlike(?:\s+\S+){0,2}$/iu, /\bno need for(?:\s+\S+){0,4}$/iu, ]); +const SKILL_REJECTION_LEADING_PATTERNS = Object.freeze([ + /\brule\s+out(?:\s+\S+){0,4}$/iu, +]); +const SKILL_REJECTION_TRAILING_PATTERNS = Object.freeze([ + /^(?:would\s+be|is|seems|feels)\s+too\s+specialized\b/iu, + /^(?:is(?:n['’]t|\s+not)|would(?:n['’]t|\s+not)\s+be)\s+the\s+right\s+skill\b/iu, + /^(?:is|would\s+be)\s+the\s+wrong\s+skill\b/iu, +]); const PROMPT_SCORE_WEIGHTS = Object.freeze({ expectedPatterns: 0.4, skillSelection: 0.4, @@ -778,8 +786,10 @@ function detectAntiPatternFindings( return findings; } -function inferSelectedSkill(response: string): ExpectedSkill { - if (matchesAny(response, ['\\bdogfood-tui\\b'])) { +export function inferSelectedSkill(response: string): ExpectedSkill { + assertString(response, 'Prompt response must be a string'); + + if (matchesAnyOutsideSkillRejectionContext(response, ['\\bdogfood-tui\\b'])) { return 'dogfood-tui'; } @@ -796,6 +806,68 @@ function inferSelectedSkill(response: string): ExpectedSkill { return 'none'; } +function matchesAnyOutsideSkillRejectionContext( + text: string, + patterns: string[], +): boolean { + assertString(text, 'Text to scan must be a string'); + invariant(Array.isArray(patterns), 'Patterns must be an array'); + + const lineStarts = computeLineStarts(text); + return patterns.some((pattern) => { + assertString(pattern, 'Each pattern must be a string'); + invariant(pattern.length > 0, 'Patterns must not be empty'); + const occurrences = collectOccurrences( + text, + compilePattern(pattern), + lineStarts, + ); + + return occurrences.some( + (occurrence) => !isRejectedSkillMention(text, occurrence), + ); + }); +} + +function isRejectedSkillMention( + text: string, + occurrence: MatchOccurrence, +): boolean { + if (isInNegationContext(text, occurrence.offset)) { + return true; + } + + const leadingContext = text + .slice( + Math.max(0, occurrence.offset - NEGATION_CONTEXT_WINDOW), + occurrence.offset, + ) + .replace(/\s+/gu, ' ') + .trim() + .toLowerCase(); + if ( + SKILL_REJECTION_LEADING_PATTERNS.some((pattern) => + pattern.test(leadingContext), + ) + ) { + return true; + } + + const trailingContext = text + .slice( + occurrence.offset + occurrence.matchedText.length, + occurrence.offset + + occurrence.matchedText.length + + NEGATION_CONTEXT_WINDOW, + ) + .replace(/\s+/gu, ' ') + .trim() + .toLowerCase(); + return SKILL_REJECTION_TRAILING_PATTERNS.some((pattern) => + pattern.test(trailingContext), + ); +} + function matchesAny(text: string, patterns: string[]): boolean { return matchPatterns(text, patterns).some((result) => result.matched); } diff --git a/evals/lib/types.ts b/evals/lib/types.ts index 05b9b5ab..7c40b259 100644 --- a/evals/lib/types.ts +++ b/evals/lib/types.ts @@ -225,6 +225,7 @@ export interface RunMetadata { runId: string; createdAt: string; repoRoot: string; + outputBaseDir?: string | undefined; providers: string[]; models: string[]; lanes: EvalLane[]; @@ -396,10 +397,38 @@ export interface AggregateMetrics { evidenceQualityRate?: number; } +/** Per-condition pass/fail summary for a comparison view. */ +export interface ConditionAggregateSummary { + condition: SkillCondition; + totalCases: number; + passed: number; + failed: number; + passRate: number; + averageScore: number; +} + +/** Concise condition comparison summary promoted near the top of reports. */ +export interface ConditionComparisonSummary { + comparedConditions: SkillCondition[]; + comparedGroups: number; + comparedCases: number; + conditionBreakdown: ConditionAggregateSummary[]; + keyDeltas: { + realizedSkillLift?: number | undefined; + oracleSkillLift?: number | undefined; + routingGap?: number | undefined; + staleSkillHarm?: number | undefined; + regressionRate?: number | undefined; + unlockRate?: number | undefined; + routingEfficiency?: number | undefined; + }; +} + /** JSON-serializable top-level eval report. */ export interface JsonReport { metadata: RunMetadata; aggregate: AggregateMetrics; + conditionComparisonSummary?: ConditionComparisonSummary; comparisons: ComparisonMetrics[]; results: EvalResult[]; providerComparison?: ProviderComparisonReport; diff --git a/evals/run.ts b/evals/run.ts index 476c22f6..c55c69c5 100644 --- a/evals/run.ts +++ b/evals/run.ts @@ -54,7 +54,7 @@ const HELP_TEXT = [ ' --model Model to use (for example: opus, claude-opus-4-6, gpt-5.4, o4-mini)', ' --effort Claude Code effort/thinking level (low, medium, high, max)', ' --lane Lane to run (prompt, execution, dogfood, all)', - ' --condition Skill condition (none, self-load, preloaded, stale, all). Default: all', + ' --condition Skill condition (none, self-load, preloaded, stale, all). Default: all. May be repeated.', ' --case Run specific case(s) by ID. May be repeated.', ' --output Output directory. Default: evals/reports/{timestamp}', ' --json Print JSON summary only', @@ -70,6 +70,7 @@ const HELP_TEXT = [ ' npx tsx evals/run.ts --provider claude --model opus --effort high --lane prompt', ' npx tsx evals/run.ts --provider codex --model gpt-5.4 --lane all', ' npx tsx evals/run.ts --provider stub --lane execution --case hello-prompt --case resize-demo', + ' npx tsx evals/run.ts --provider stub --lane execution --condition none --condition preloaded --dry-run', '', 'Notes:', ' - Relative --output paths resolve from the repository root.', @@ -81,7 +82,7 @@ interface CliOptions { modelId?: string; effortLevel?: string; lane?: string; - condition?: string; + conditions: string[]; caseIds: string[]; outputDir?: string; concurrency?: string; @@ -98,7 +99,7 @@ interface ResolvedCliOptions { modelId?: string; effortLevel?: string; requestedLane: string; - requestedCondition: string; + requestedConditions: string[]; caseIds: string[]; outputBaseDir: string; compareBaselinePath?: string; @@ -223,8 +224,9 @@ function parseOptionValue( }; } -function parseCliArgs(argumentsList: readonly string[]): CliOptions { +export function parseCliArgs(argumentsList: readonly string[]): CliOptions { const options: CliOptions = { + conditions: [], caseIds: [], json: false, verbose: false, @@ -356,11 +358,7 @@ function parseCliArgs(argumentsList: readonly string[]): CliOptions { argumentsList, index, ); - invariant( - options.condition === undefined, - '--condition may only be set once', - ); - options.condition = parsed.value; + options.conditions.push(parsed.value); index = parsed.nextIndex; continue; } @@ -436,20 +434,45 @@ function resolveRequestedLanes(lane: string | undefined): EvalLane[] { return [lane]; } -function resolveRequestedConditions( - condition: string | undefined, +export function resolveRequestedConditions( + conditions: readonly string[], ): SkillCondition[] { - if (condition === undefined || condition === 'all') { + invariant(Array.isArray(conditions), '--condition values must be an array'); + if (conditions.length === 0) { return [...SKILL_CONDITIONS]; } - assertString(condition, '--condition must be a string'); - invariant(condition.length > 0, '--condition must not be empty'); - invariant( - isSkillCondition(condition), - `Unsupported condition: ${condition}. Expected one of ${[...SKILL_CONDITIONS, 'all'].join(', ')}`, + let requestedAll = false; + const requestedConditions = new Set(); + for (const condition of conditions) { + assertString(condition, '--condition values must be strings'); + invariant(condition.length > 0, '--condition values must not be empty'); + if (condition === 'all') { + requestedAll = true; + continue; + } + invariant( + isSkillCondition(condition), + `Unsupported condition: ${condition}. Expected one of ${[...SKILL_CONDITIONS, 'all'].join(', ')}`, + ); + invariant( + !requestedAll, + '--condition all may not be combined with specific values', + ); + requestedConditions.add(condition); + } + + if (requestedAll) { + invariant( + requestedConditions.size === 0, + '--condition all may not be combined with specific values', + ); + return [...SKILL_CONDITIONS]; + } + + return SKILL_CONDITIONS.filter((condition) => + requestedConditions.has(condition), ); - return [condition]; } function resolveRequestedCaseIds(caseIds: readonly string[]): string[] { @@ -623,7 +646,12 @@ function buildResolvedOptions( '--effort', ); const requestedLanes = resolveRequestedLanes(options.lane); - const requestedConditions = resolveRequestedConditions(options.condition); + const requestedConditions = resolveRequestedConditions(options.conditions); + const requestedConditionFilters = + options.conditions.length === 0 || + options.conditions.some((condition) => condition === 'all') + ? ['all'] + : [...requestedConditions]; const caseIds = resolveRequestedCaseIds(options.caseIds); const outputBaseDir = resolveOutputBaseDir(repoRoot, options.outputDir); const compareBaselinePath = resolveOptionalStringOption( @@ -645,7 +673,7 @@ function buildResolvedOptions( ...(modelId === undefined ? {} : { modelId }), ...(effortLevel === undefined ? {} : { effortLevel }), requestedLane: options.lane ?? 'all', - requestedCondition: options.condition ?? 'all', + requestedConditions: requestedConditionFilters, caseIds, outputBaseDir, ...(compareBaselinePath === undefined @@ -716,7 +744,7 @@ function writeDryRunSummary(options: ResolvedCliOptions): void { process.stdout, `Conditions: ${options.activeConditions.join(', ')}`, ); - writeLine(process.stdout, `Output base dir: ${options.outputBaseDir}`); + writeLine(process.stdout, `Output directory: ${options.outputBaseDir}`); writeLine( process.stdout, `Total eval invocations: ${String(options.totalInvocations)}`, @@ -775,9 +803,13 @@ function buildRunMetadata( ): RunMetadata { const metadataNotes: string[] = []; appendMetadataNote(metadataNotes, `requested lane: ${options.requestedLane}`); + invariant( + options.requestedConditions.length > 0, + 'resolved requestedConditions must contain at least one value', + ); appendMetadataNote( metadataNotes, - `requested condition: ${options.requestedCondition}`, + `requested conditions: ${options.requestedConditions.join(', ')}`, ); if (options.caseIds.length > 0) { appendMetadataNote( @@ -785,10 +817,6 @@ function buildRunMetadata( `case filter: ${options.caseIds.join(', ')}`, ); } - appendMetadataNote( - metadataNotes, - `output base dir: ${options.outputBaseDir}`, - ); appendMetadataNote( metadataNotes, options.modelId === undefined @@ -834,6 +862,7 @@ function buildRunMetadata( runId, createdAt: new Date().toISOString(), repoRoot, + outputBaseDir: options.outputBaseDir, providers: [options.providerId], models: selectedModelId === undefined ? [] : [selectedModelId], lanes: options.activeLanes, @@ -916,7 +945,7 @@ function writeHumanSummary(summary: RunSummary): void { } writeLine(process.stdout, `Lanes: ${summary.lanes.join(', ')}`); writeLine(process.stdout, `Conditions: ${summary.conditions.join(', ')}`); - writeLine(process.stdout, `Output base dir: ${summary.outputBaseDir}`); + writeLine(process.stdout, `Output directory: ${summary.outputBaseDir}`); if (summary.runDir !== undefined) { writeLine(process.stdout, `Run dir: ${summary.runDir}`); } diff --git a/test/unit/evals/antiPatterns.test.ts b/test/unit/evals/antiPatterns.test.ts index 7a2bfdd5..7215b4f7 100644 --- a/test/unit/evals/antiPatterns.test.ts +++ b/test/unit/evals/antiPatterns.test.ts @@ -167,6 +167,38 @@ describe('detectAntiPatterns', () => { expect(detectAntiPatterns(transcript)).toEqual([]); }); + + it('ignores anti-pattern keywords that appear only in shell tool output', () => { + const normalized = createNormalizedOutput({ + toolCalls: [ + { + name: 'bash', + input: { script: 'agent-tty skills get agent-tty --json' }, + output: 'scrot\ntmux\nxdotool\ngnome-screenshot', + }, + ], + }); + + expect(detectAntiPatterns(buildScannableTranscript(normalized))).toEqual( + [], + ); + }); + + it('detects tmux usage from shell tool-call inputs', () => { + const normalized = createNormalizedOutput({ + toolCalls: [{ name: 'bash', input: { script: 'tmux new-session' } }], + }); + + const findings = detectAntiPatterns(buildScannableTranscript(normalized)); + + expect(findings).toHaveLength(1); + expect(findings[0]).toMatchObject({ + ruleId: 'tmux-usage', + severity: 'error', + matchedText: 'tmux new-session', + lineNumber: 1, + }); + }); }); describe('missing-json-flag', () => { @@ -392,7 +424,7 @@ describe('summarizeFindings', () => { }); describe('buildScannableTranscript', () => { - it('includes only bash/shell tool call content', () => { + it('includes only bash/shell tool-call inputs', () => { const normalized = createNormalizedOutput({ finalText: 'Some planning text about tmux and sleep', messages: ['Use agent-tty instead of tmux'], @@ -420,13 +452,15 @@ describe('buildScannableTranscript', () => { expect(transcript).toContain('agent-tty create'); expect(transcript).toContain('agent-tty snapshot'); + expect(transcript).not.toContain('session created'); + expect(transcript).not.toContain('snapshot taken'); expect(transcript).not.toContain('tmux'); expect(transcript).not.toContain('sleep'); expect(transcript).not.toContain('read_file'); expect(transcript).not.toContain('file contents'); }); - it('includes unnamed Codex command execution records', () => { + it('includes unnamed Codex command execution inputs', () => { const normalized = createNormalizedOutput({ toolCalls: [ { @@ -442,7 +476,7 @@ describe('buildScannableTranscript', () => { expect(transcript).toContain( 'npx tsx src/cli/main.ts wait --json --session demo', ); - expect(transcript).toContain('wait completed'); + expect(transcript).not.toContain('wait completed'); }); it('returns empty string when toolCalls is empty', () => { diff --git a/test/unit/evals/codex.test.ts b/test/unit/evals/codex.test.ts index 1e2063b1..608bc0c0 100644 --- a/test/unit/evals/codex.test.ts +++ b/test/unit/evals/codex.test.ts @@ -53,7 +53,9 @@ describe('CodexProvider.parse', () => { expect(buildScannableTranscript(normalized)).toContain( 'npx tsx src/cli/main.ts snapshot --json --session demo', ); - expect(buildScannableTranscript(normalized)).toContain('snapshot ready'); + expect(buildScannableTranscript(normalized)).not.toContain( + 'snapshot ready', + ); expect(countAgentTtyCalls(normalized)).toBe(1); }); @@ -103,7 +105,9 @@ describe('CodexProvider.parse', () => { expect(buildScannableTranscript(normalized)).toContain( 'npx tsx src/cli/main.ts run --json --session demo', ); - expect(buildScannableTranscript(normalized)).toContain('command finished'); + expect(buildScannableTranscript(normalized)).not.toContain( + 'command finished', + ); expect(countAgentTtyCalls(normalized)).toBe(1); }); }); diff --git a/test/unit/evals/reporting.test.ts b/test/unit/evals/reporting.test.ts index 976703d5..eff13b17 100644 --- a/test/unit/evals/reporting.test.ts +++ b/test/unit/evals/reporting.test.ts @@ -1,9 +1,11 @@ import { describe, expect, it } from 'vitest'; +import { computeComparisonMetrics } from '../../../evals/lib/matrix.js'; import { generateJsonReport, generateMarkdownReport, } from '../../../evals/lib/reporting.js'; +import { JsonReportSchema } from '../../../evals/lib/schemas.js'; import { computeConfidenceInterval, computeMean, @@ -307,3 +309,136 @@ describe('generateMarkdownReport trial aggregation', () => { expect(markdown).not.toContain('## Trial Aggregation'); }); }); + +describe('eval reporting condition comparison summary', () => { + function createConditionComparisonMetadata(): RunMetadata { + return { + runId: 'run-1', + createdAt: '2025-01-01T00:00:00.000Z', + repoRoot: '/repo', + outputBaseDir: '/tmp/evals/out', + providers: ['stub'], + models: ['stub'], + lanes: ['execution'], + conditions: ['none', 'preloaded'], + totalTrials: 1, + notes: [], + }; + } + + function createConditionComparisonResults(): EvalResult[] { + return [ + createEvalResult({ + runId: 'run-1', + lane: 'execution', + category: 'session', + caseId: 'case-1', + condition: 'none', + ok: false, + score: { total: 0.2, maxPossible: 1, items: [] }, + startedAt: '2025-01-01T00:00:00.000Z', + completedAt: '2025-01-01T00:00:01.000Z', + }), + createEvalResult({ + runId: 'run-1', + lane: 'execution', + category: 'session', + caseId: 'case-1', + condition: 'preloaded', + trial: 2, + score: { total: 0.8, maxPossible: 1, items: [] }, + startedAt: '2025-01-01T00:00:00.000Z', + completedAt: '2025-01-01T00:00:01.000Z', + }), + createEvalResult({ + runId: 'run-1', + lane: 'execution', + category: 'session', + caseId: 'case-2', + condition: 'none', + trial: 3, + ok: false, + score: { total: 0.4, maxPossible: 1, items: [] }, + startedAt: '2025-01-01T00:00:00.000Z', + completedAt: '2025-01-01T00:00:01.000Z', + }), + createEvalResult({ + runId: 'run-1', + lane: 'execution', + category: 'session', + caseId: 'case-2', + condition: 'preloaded', + trial: 4, + score: { total: 1, maxPossible: 1, items: [] }, + startedAt: '2025-01-01T00:00:00.000Z', + completedAt: '2025-01-01T00:00:01.000Z', + }), + ]; + } + + it('adds a structured condition comparison summary to the JSON report', () => { + const results = createConditionComparisonResults(); + const metadata = createConditionComparisonMetadata(); + const comparisons = computeComparisonMetrics(results); + + const report = generateJsonReport(results, metadata, comparisons); + + expect(report.metadata.outputBaseDir).toBe('/tmp/evals/out'); + expect(report.conditionComparisonSummary).toBeDefined(); + const summary = report.conditionComparisonSummary; + if (summary === undefined) { + throw new Error('Expected conditionComparisonSummary to be defined'); + } + + expect(summary.comparedConditions).toEqual(['none', 'preloaded']); + expect(summary.comparedGroups).toBe(2); + expect(summary.comparedCases).toBe(2); + expect( + summary.conditionBreakdown.map((row) => [ + row.condition, + row.totalCases, + row.passed, + row.failed, + ]), + ).toEqual([ + ['none', 2, 0, 2], + ['preloaded', 2, 2, 0], + ]); + expect(summary.conditionBreakdown[0]?.averageScore).toBeCloseTo(0.3); + expect(summary.conditionBreakdown[1]?.averageScore).toBeCloseTo(0.9); + expect(summary.keyDeltas.realizedSkillLift).toBeCloseTo(0); + expect(summary.keyDeltas.oracleSkillLift).toBeCloseTo(0.6); + + expect(() => + JsonReportSchema.parse({ + metadata: report.metadata, + aggregate: report.aggregate, + conditionComparisonSummary: report.conditionComparisonSummary, + comparisons: report.comparisons, + results: report.results, + ...(report.providerComparison === undefined + ? {} + : { providerComparison: report.providerComparison }), + }), + ).not.toThrow(); + }); + + it('promotes the condition comparison summary near the top of the markdown report', () => { + const results = createConditionComparisonResults().slice(0, 2); + const metadata = createConditionComparisonMetadata(); + const comparisons = computeComparisonMetrics(results); + + const markdown = generateMarkdownReport(results, metadata, comparisons); + + expect(markdown).toContain('- Output directory: `/tmp/evals/out`'); + expect(markdown).toContain('## Condition comparison'); + expect(markdown).toContain('- Compared conditions: `none`, `preloaded`'); + expect(markdown).toContain('| `none` | 1 | 0 | 1 | 0.0% | 0.200 |'); + expect(markdown).toContain('| `preloaded` | 1 | 1 | 0 | 100.0% | 0.800 |'); + expect(markdown).toContain('| Realized skill lift | 0.0% |'); + expect(markdown).toContain('| Oracle skill lift | 60.0% |'); + expect(markdown.indexOf('## Condition comparison')).toBeLessThan( + markdown.indexOf('## Lane breakdown'), + ); + }); +}); diff --git a/test/unit/evals/run.test.ts b/test/unit/evals/run.test.ts new file mode 100644 index 00000000..5141efa3 --- /dev/null +++ b/test/unit/evals/run.test.ts @@ -0,0 +1,170 @@ +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { SKILL_CONDITIONS } from '../../../evals/lib/matrix.js'; +import { + parseCliArgs, + resolveRequestedConditions, + runEvalCli, +} from '../../../evals/run.js'; + +function getWrittenStdout(calls: readonly unknown[][]): string { + return calls + .map((call) => { + const [chunk] = call; + expect(typeof chunk).toBe('string'); + if (typeof chunk !== 'string') { + throw new Error('expected stdout to be written as a string'); + } + return chunk; + }) + .join(''); +} + +function resolveRepoPath(...segments: string[]): string { + return resolve( + fileURLToPath(new URL('../../..', import.meta.url)), + ...segments, + ); +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('parseCliArgs', () => { + it('collects a single --condition value', () => { + const options = parseCliArgs([ + '--provider', + 'stub', + '--lane', + 'prompt', + '--condition', + 'none', + ]); + + expect(options.conditions).toEqual(['none']); + }); + + it('collects repeated --condition values in CLI order', () => { + const options = parseCliArgs([ + '--provider', + 'stub', + '--lane', + 'prompt', + '--condition', + 'none', + '--condition', + 'preloaded', + ]); + + expect(options.conditions).toEqual(['none', 'preloaded']); + }); + + it('defaults to no explicit condition filters when --condition is omitted', () => { + const options = parseCliArgs(['--provider', 'stub', '--lane', 'prompt']); + + expect(options.conditions).toEqual([]); + }); +}); + +describe('resolveRequestedConditions', () => { + it('defaults to all conditions when no filters are provided', () => { + expect(resolveRequestedConditions([])).toEqual(SKILL_CONDITIONS); + }); + + it('resolves a single condition', () => { + expect(resolveRequestedConditions(['none'])).toEqual(['none']); + }); + + it('deduplicates repeated conditions and restores canonical ordering', () => { + expect( + resolveRequestedConditions(['preloaded', 'none', 'none', 'stale']), + ).toEqual(['none', 'preloaded', 'stale']); + }); + + it('expands all when requested by itself', () => { + expect(resolveRequestedConditions(['all'])).toEqual(SKILL_CONDITIONS); + expect(resolveRequestedConditions(['all', 'all'])).toEqual( + SKILL_CONDITIONS, + ); + }); + + it('rejects all when mixed with specific conditions', () => { + expect(() => resolveRequestedConditions(['all', 'none'])).toThrow( + '--condition all may not be combined with specific values', + ); + expect(() => resolveRequestedConditions(['none', 'all'])).toThrow( + '--condition all may not be combined with specific values', + ); + }); + + it('rejects invalid conditions', () => { + expect(() => resolveRequestedConditions(['invalid'])).toThrow( + 'Unsupported condition: invalid. Expected one of none, self-load, preloaded, stale, all', + ); + }); +}); + +describe('runEvalCli dry-run output', () => { + it('prints the resolved output directory in human-readable output', async () => { + const stdoutWriteSpy = vi + .spyOn(process.stdout, 'write') + .mockReturnValue(true); + const outputDir = resolveRepoPath('evals', 'reports', 'test-dry-run-human'); + + const exitCode = await runEvalCli([ + '--provider', + 'stub', + '--lane', + 'execution', + '--case', + 'hello-prompt', + '--condition', + 'none', + '--condition', + 'preloaded', + '--output', + 'evals/reports/test-dry-run-human', + '--dry-run', + ]); + + expect(exitCode).toBe(0); + expect( + getWrittenStdout(stdoutWriteSpy.mock.calls as unknown[][]), + ).toContain(`Output directory: ${outputDir}\n`); + }); + + it('emits the resolved output directory in the JSON summary', async () => { + const stdoutWriteSpy = vi + .spyOn(process.stdout, 'write') + .mockReturnValue(true); + const outputDir = resolveRepoPath('evals', 'reports', 'test-dry-run-json'); + + const exitCode = await runEvalCli([ + '--provider', + 'stub', + '--lane', + 'execution', + '--case', + 'hello-prompt', + '--condition', + 'none', + '--condition', + 'preloaded', + '--output', + 'evals/reports/test-dry-run-json', + '--dry-run', + '--json', + ]); + + expect(exitCode).toBe(0); + expect( + JSON.parse( + getWrittenStdout(stdoutWriteSpy.mock.calls as unknown[][]), + ) as { outputBaseDir: string }, + ).toMatchObject({ outputBaseDir: outputDir }); + }); +}); diff --git a/test/unit/evals/scoring.test.ts b/test/unit/evals/scoring.test.ts index 7cef011d..8b1f3437 100644 --- a/test/unit/evals/scoring.test.ts +++ b/test/unit/evals/scoring.test.ts @@ -6,6 +6,7 @@ import { checkWorkflow, compilePattern, computePrecisionRecall, + inferSelectedSkill, isInNegationContext, matchPatterns, scorePromptCase, @@ -296,6 +297,31 @@ describe('isInNegationContext', () => { }); }); +describe('inferSelectedSkill', () => { + it('falls back to agent-tty when dogfood-tui only appears in a rejection context', () => { + const response = [ + 'agent-tty', + '', + 'This is a task... dogfood-tui would be too specialized because the user is not asking for QA.', + ].join('\n'); + + expect(inferSelectedSkill(response)).toBe('agent-tty'); + }); + + it('returns dogfood-tui when it is positively selected', () => { + expect( + inferSelectedSkill('Use the dogfood-tui skill to explore the TUI.'), + ).toBe('dogfood-tui'); + }); + + it('ignores negated dogfood-tui mentions in favor of agent-tty', () => { + const response = + 'We should NOT use dogfood-tui here. I will use agent-tty.'; + + expect(inferSelectedSkill(response)).toBe('agent-tty'); + }); +}); + describe('PASS_THRESHOLD', () => { it('matches the documented pass threshold', () => { expect(PASS_THRESHOLD).toBe(0.6);