feat: improve eval setup for skill optimization + fix two scoring bugs

.gitignore

@@ -43,3 +43,6 @@ pids/
 tmp/
 .tmp/
 .cache/
+
+# Eval reports (generated per-run under evals/reports/{timestamp}/)
+evals/reports/

evals/README.md

@@ -28,11 +28,14 @@ npx tsx evals/run.ts --provider codex --lane all
 
 # Specific cases
 npx tsx evals/run.ts --provider stub --lane execution --case hello-prompt --case resize-demo
+
+# Compare conditions in one dry run
+npx tsx evals/run.ts --provider stub --lane execution --condition none --condition preloaded --dry-run
 ```
 
 Useful flags:
 
-- `--condition <cond>` — `none`, `self-load`, `preloaded`, `stale`, or `all` (default: `all`)
+- `--condition <cond>` — `none`, `self-load`, `preloaded`, `stale`, or `all` (default: `all`). May be repeated.
 - `--output <dir>` — output base directory (default: `evals/reports/{timestamp}`)
 - `--dry-run` — list the case/condition matrix without invoking a provider
 - `--json` — emit only a JSON summary to stdout
@@ -47,6 +50,9 @@ npx tsx evals/run.ts --provider stub --lane all --dry-run
 # Restrict to one condition
 npx tsx evals/run.ts --provider stub --lane prompt --condition self-load
 
+# Compare two conditions in one run
+npx tsx evals/run.ts --provider stub --lane prompt --condition none --condition preloaded
+
 # Write results under a custom directory
 npx tsx evals/run.ts --provider stub --lane execution --output evals/reports/local-smoke
 ```
@@ -199,6 +205,45 @@ Prompt cases currently cover positive routing, negative routing, and explicit an
 | `run-command`     | `session`  | `hello-prompt`    | all four   | Use `agent-tty run` instead of simulated typing, then snapshot the result.         |
 | `doctor-gated`    | `artifact` | `hello-prompt`    | all four   | Run `doctor --json` before taking a renderer-dependent screenshot.                 |
 
+#### Lane B readiness and renderer requirements
+
+Lane B coverage is intentionally uneven today. Use the **readiness tier** to decide where to add smoke coverage next, and use the **renderer requirement** column to decide whether a failure is likely a skill regression or an environment problem.
+
+- **`battle-tested`** — safest execution smoke cases today.
+- **`non-renderer unproven`** — next expansion batch before spending more time on renderer-heavy cases. `export-proof` stays in this rollout bucket even though its required `.webm` export is renderer-backed.
+- **`renderer-optional`** — the case can still pass without Playwright/Chromium, but renderer-backed artifacts are useful when available.
+- **`renderer-required`** — failing Playwright/Chromium/ghostty-web checks should be treated as an `environment-blocked` result, not as a skill regression.
+
+| Case ID           | Readiness tier          | Renderer requirement | Notes                                                                                                                    |
+| ----------------- | ----------------------- | -------------------- | ------------------------------------------------------------------------------------------------------------------------ |
+| `hello-prompt`    | `battle-tested`         | `none`               | Core create → input → wait → snapshot → cleanup loop.                                                                    |
+| `crash-recovery`  | `battle-tested`         | `none`               | Core crash inspection and cleanup flow.                                                                                  |
+| `run-command`     | `battle-tested`         | `none`               | Core programmatic input flow via `agent-tty run`.                                                                        |
+| `resize-demo`     | `non-renderer unproven` | `none`               | Snapshot-only resize verifier; high-value next smoke target.                                                             |
+| `alt-screen-demo` | `non-renderer unproven` | `none`               | Event-log plus snapshot proof still needs broader smoke coverage.                                                        |
+| `scrollback-demo` | `non-renderer unproven` | `none`               | Scrollback snapshot proof still needs broader smoke coverage.                                                            |
+| `unicode-grid`    | `non-renderer unproven` | `none`               | Semantic snapshot verifier still needs broader smoke coverage.                                                           |
+| `export-proof`    | `non-renderer unproven` | `required`           | Still part of the next smoke batch, but the required `.webm` export is renderer-backed.                                  |
+| `color-grid`      | `renderer-optional`     | `optional`           | Screenshot evidence is preferred when renderer support exists, but non-renderer verification can still satisfy the case. |
+| `doctor-gated`    | `renderer-required`     | `required`           | Must run `doctor --json` first; missing renderer support blocks the required screenshot.                                 |
+
+**Expansion order:** keep prioritizing `resize-demo`, `alt-screen-demo`, `scrollback-demo`, `unicode-grid`, and `export-proof` before spending more time on `color-grid` or `doctor-gated`. Within that batch, `export-proof` is the one case that still needs renderer-backed WebM export, so preflight renderer availability first.
+
+To avoid mistaking a missing browser/runtime dependency for a skill regression, pair execution-lane dry runs with a renderer preflight:
+
+```sh
+# Preview the case/condition matrix
+npx tsx evals/run.ts --provider stub --lane execution --dry-run
+
+# Check renderer prerequisites before renderer-backed cases
+npx tsx src/cli/main.ts doctor --json
+
+# If doctor reports a missing Playwright browser cache
+npx playwright install chromium
+```
+
+`--dry-run` currently tells you which execution cases and conditions will run, but it does not yet annotate renderer needs inline. Use the table above to decide whether `color-grid`, `export-proof`, or `doctor-gated` should be treated as renderer-backed for your environment.
+
 ### Dogfood lane
 
 | Case ID                  | Category            | Fixture           | Conditions | Description                                                                                          |

evals/execution/cases/shared.ts

@@ -21,6 +21,93 @@ const DEFAULT_MAX_WALL_CLOCK_MS = 90_000;
 
 export const ALL_EXECUTION_CONDITIONS: SkillCondition[] = [...SKILL_CONDITIONS];
 
+export type ExecutionRendererRequirement = 'none' | 'optional' | 'required';
+
+export type ExecutionReadinessTier =
+  | 'battle-tested'
+  | 'non-renderer-unproven'
+  | 'renderer-optional'
+  | 'renderer-required';
+
+export interface ExecutionCaseCoverageMetadata {
+  readinessTier: ExecutionReadinessTier;
+  rendererRequirement: ExecutionRendererRequirement;
+  summary: string;
+}
+
+const EXECUTION_CASE_COVERAGE_BY_ID = {
+  'hello-prompt': {
+    readinessTier: 'battle-tested',
+    rendererRequirement: 'none',
+    summary: 'Battle-tested create → input → wait → snapshot → cleanup flow.',
+  },
+  'crash-recovery': {
+    readinessTier: 'battle-tested',
+    rendererRequirement: 'none',
+    summary: 'Battle-tested crash inspection and cleanup flow.',
+  },
+  'run-command': {
+    readinessTier: 'battle-tested',
+    rendererRequirement: 'none',
+    summary: 'Battle-tested programmatic input flow via agent-tty run.',
+  },
+  'resize-demo': {
+    readinessTier: 'non-renderer-unproven',
+    rendererRequirement: 'none',
+    summary:
+      'Snapshot-only resize verification still needs broader smoke coverage.',
+  },
+  'alt-screen-demo': {
+    readinessTier: 'non-renderer-unproven',
+    rendererRequirement: 'none',
+    summary:
+      'Alt-screen event-log and snapshot proof still needs broader smoke coverage.',
+  },
+  'scrollback-demo': {
+    readinessTier: 'non-renderer-unproven',
+    rendererRequirement: 'none',
+    summary: 'Scrollback snapshot proof still needs broader smoke coverage.',
+  },
+  'unicode-grid': {
+    readinessTier: 'non-renderer-unproven',
+    rendererRequirement: 'none',
+    summary:
+      'Unicode snapshot verification still needs broader smoke coverage.',
+  },
+  'export-proof': {
+    readinessTier: 'non-renderer-unproven',
+    rendererRequirement: 'required',
+    summary:
+      'Still in the next smoke batch, but the required WebM export is renderer-backed.',
+  },
+  'color-grid': {
+    readinessTier: 'renderer-optional',
+    rendererRequirement: 'optional',
+    summary:
+      'Can pass with non-renderer verification, but renderer-backed screenshots remain useful reviewer evidence.',
+  },
+  'doctor-gated': {
+    readinessTier: 'renderer-required',
+    rendererRequirement: 'required',
+    summary:
+      'Must pass doctor --json renderer checks before the required screenshot capture.',
+  },
+} as const satisfies Record<string, ExecutionCaseCoverageMetadata>;
+
+export const EXECUTION_CASE_COVERAGE = EXECUTION_CASE_COVERAGE_BY_ID;
+
+export function getExecutionCaseCoverage(
+  caseId: string,
+): ExecutionCaseCoverageMetadata {
+  invariant(
+    Object.prototype.hasOwnProperty.call(EXECUTION_CASE_COVERAGE_BY_ID, caseId),
+    `Missing execution case coverage metadata for ${caseId}`,
+  );
+  return EXECUTION_CASE_COVERAGE_BY_ID[
+    caseId as keyof typeof EXECUTION_CASE_COVERAGE_BY_ID
+  ];
+}
+
 export function anyOf(...patterns: string[]): string {
   invariant(patterns.length > 0, 'anyOf() requires at least one pattern');
   return `(?:${patterns.join('|')})`;

evals/execution/runner.ts

@@ -46,6 +46,10 @@ import { resizeDemoCase } from './cases/resize-demo.js';
 import { runCommandCase } from './cases/run-command.js';
 import { scrollbackDemoCase } from './cases/scrollback-demo.js';
 import { unicodeGridCase } from './cases/unicode-grid.js';
+import {
+  EXECUTION_CASE_COVERAGE,
+  getExecutionCaseCoverage,
+} from './cases/shared.js';
 import type { VerifierResult } from './verifiers/index.js';
 import { verify, verifyArtifactExists } from './verifiers/index.js';
 
@@ -106,6 +110,31 @@ const STALE_EXECUTION_SKILL_TEXT = [
   'Some of the guidance above is wrong for this repository snapshot. Verify commands against the current CLI behavior while completing the task.',
 ].join('\n');
 
+const ENVIRONMENT_BLOCKED_ERROR_CLASS = 'environment-blocked';
+const RENDERER_ENVIRONMENT_HINT =
+  'Run `npx tsx src/cli/main.ts doctor --json` before retrying. If the renderer checks report a missing browser cache, run `npx playwright install chromium`.';
+const RENDERER_ENVIRONMENT_ERROR_PATTERNS = [
+  /Playwright browser cache not found/iu,
+  /Run 'npx playwright install chromium' to install\./iu,
+  /\bplaywright unavailable\b/iu,
+  /\bplaywright not installed\b/iu,
+  /\bplaywright import failed\b/iu,
+  /\bghostty-web unavailable\b/iu,
+  /\bbrowser launch failed\b/iu,
+  /\bscreenshot smoke test failed\b/iu,
+  /Failed to boot or replay renderer/iu,
+  /Executable doesn't exist/iu,
+] as const;
+const RENDERER_DOCTOR_FAILURE_PATTERNS = [
+  /"name"\s*:\s*"playwright_available"[\s\S]*?"status"\s*:\s*"fail"/iu,
+  /"name"\s*:\s*"browser_cache_accessible"[\s\S]*?"status"\s*:\s*"(?:fail|skip)"/iu,
+  /"name"\s*:\s*"browser_launch"[\s\S]*?"status"\s*:\s*"fail"/iu,
+  /"name"\s*:\s*"ghostty_web_available"[\s\S]*?"status"\s*:\s*"fail"/iu,
+  /"name"\s*:\s*"screenshot_viable"[\s\S]*?"status"\s*:\s*"fail"/iu,
+  /"name"\s*:\s*"screenshot"[\s\S]*?"status"\s*:\s*"(?:unavailable|degraded)"/iu,
+  /"name"\s*:\s*"record-export-webm"[\s\S]*?"status"\s*:\s*"(?:unavailable|degraded)"/iu,
+] as const;
+
 async function readSkillFile(relativePath: string): Promise<string> {
   const content = await readFile(
     new URL(relativePath, import.meta.url),
@@ -174,6 +203,64 @@ function assertExecutionCaseInventory(
   }
 }
 
+assertExecutionCaseCoverageMetadata(EXECUTION_CASES);
+
+function assertExecutionCaseCoverageMetadata(
+  cases: readonly ExecutionEvalCase[],
+): void {
+  const caseIds = new Set(cases.map((evalCase) => evalCase.id));
+  const coverageIds = Object.keys(EXECUTION_CASE_COVERAGE);
+
+  for (const caseId of caseIds) {
+    invariant(
+      coverageIds.includes(caseId),
+      `Missing execution coverage metadata for case ${caseId}`,
+    );
+  }
+
+  for (const coverageId of coverageIds) {
+    invariant(
+      caseIds.has(coverageId),
+      `Execution coverage metadata references unknown case ${coverageId}`,
+    );
+  }
+}
+
+function matchesAnyPattern(text: string, patterns: readonly RegExp[]): boolean {
+  return patterns.some((pattern) => pattern.test(text));
+}
+
+function detectRendererEnvironmentBlock(
+  evalCase: ExecutionEvalCase,
+  transcript: string,
+  providerErrorMessage: string | undefined,
+): string | undefined {
+  const coverage = getExecutionCaseCoverage(evalCase.id);
+  if (coverage.rendererRequirement !== 'required') {
+    return undefined;
+  }
+
+  const combinedText = [transcript, providerErrorMessage]
+    .filter((value): value is string => value !== undefined && value.length > 0)
+    .join('\n');
+  if (combinedText.length === 0) {
+    return undefined;
+  }
+
+  if (
+    !matchesAnyPattern(combinedText, RENDERER_ENVIRONMENT_ERROR_PATTERNS) &&
+    !matchesAnyPattern(combinedText, RENDERER_DOCTOR_FAILURE_PATTERNS)
+  ) {
+    return undefined;
+  }
+
+  return [
+    `Renderer-backed case ${evalCase.id} was blocked by missing or unhealthy Playwright/Chromium/ghostty-web dependencies.`,
+    `Case note: ${coverage.summary}`,
+    RENDERER_ENVIRONMENT_HINT,
+  ].join(' ');
+}
+
 function safeRatio(numerator: number, denominator: number): number {
   if (denominator === 0) {
     return 1;
@@ -812,6 +899,7 @@ function buildResult(
   condition: SkillCondition,
   trial: number,
   runtime: ProviderRuntimeInfo | undefined,
+  transcript: string,
   normalizedOutput: NormalizedProviderOutput,
   providerOk: boolean,
   startedAt: string,
@@ -839,16 +927,30 @@ function buildResult(
     normalizedOutput,
   );
   const modelId = resolveModelId(metadata, runtime);
+  const summarizedFailure = summarizeFailureReasons(
+    providerOk,
+    verifierResults,
+    workflowChecks,
+    antiPatternFindings,
+    artifactResults,
+    providerErrorMessage,
+  );
+  const environmentBlockedMessage = detectRendererEnvironmentBlock(
+    evalCase,
+    transcript,
+    providerErrorMessage,
+  );
+  const effectiveErrorClass =
+    environmentBlockedMessage === undefined
+      ? errorClass
+      : ENVIRONMENT_BLOCKED_ERROR_CLASS;
   const failureMessage = scored.ok
     ? undefined
-    : summarizeFailureReasons(
-        providerOk,
-        verifierResults,
-        workflowChecks,
-        antiPatternFindings,
-        artifactResults,
-        providerErrorMessage,
-      );
+    : environmentBlockedMessage === undefined
+      ? summarizedFailure
+      : [environmentBlockedMessage, summarizedFailure]
+          .filter((value) => value.length > 0)
+          .join(' ');
 
   const result: EvalResult = {
     runId: metadata.runId,
@@ -873,7 +975,9 @@ function buildResult(
     ...(bundlePath === undefined ? {} : { bundlePath }),
     ...(eventLogPath === undefined ? {} : { eventLogPath }),
     normalizedOutput,
-    ...(scored.ok || errorClass === undefined ? {} : { errorClass }),
+    ...(scored.ok || effectiveErrorClass === undefined
+      ? {}
+      : { errorClass: effectiveErrorClass }),
     ...(failureMessage === undefined ? {} : { errorMessage: failureMessage }),
     startedAt,
     completedAt,
@@ -936,6 +1040,7 @@ async function runSingleExecutionCase(
         condition,
         trial,
         runtime,
+        transcript,
         normalizedOutput,
         false,
         invocationStartedAt,
@@ -1008,6 +1113,7 @@ async function runSingleExecutionCase(
         condition,
         trial,
         providerResult.runtime,
+        transcript,
         providerResult.normalized,
         providerResult.ok,
         providerResult.startedAt,
@@ -1049,6 +1155,7 @@ async function runSingleExecutionCase(
         condition,
         trial,
         runtime,
+        transcript,
         normalizedOutput,
         false,
         invocationStartedAt,
@@ -1086,7 +1193,6 @@ async function runSingleExecutionCase(
     }
   }
 }
-
 /** Return all registered execution-lane cases in deterministic order. */
 export function getAllExecutionCases(): ExecutionEvalCase[] {
   assertExecutionCaseInventory(EXECUTION_CASES);