coder · ThomasK33 · Apr 15, 2026 · Apr 15, 2026 · Apr 15, 2026 · Apr 15, 2026
diff --git a/evals/dogfood/cases/evidence-completeness.ts b/evals/dogfood/cases/evidence-completeness.ts
@@ -123,8 +123,9 @@ export const evidenceCompletenessCase = DogfoodEvalCaseSchema.parse({
   workflowChecks: [],
   antiPatterns: [...DEFAULT_ANTI_PATTERN_RULES],
   budgets: {
-    timeoutMs: 300_000,
-    maxWallClockMs: 300000,
+    timeoutMs: 600_000,
+    maxAgentSteps: 30,
+    maxWallClockMs: 600_000,
   },
 });
 

diff --git a/evals/dogfood/cases/exploratory-qa.ts b/evals/dogfood/cases/exploratory-qa.ts
@@ -55,7 +55,7 @@ export const exploratoryQaCase = DogfoodEvalCaseSchema.parse({
   lane: 'dogfood',
   category: 'qa',
   prompt: dogfoodTaskPrompt(
-    'Perform exploratory QA testing on the hello-prompt fixture app. Test input handling, exit behavior, error codes, and edge cases. Produce a proof bundle with screenshots, recordings, and a structured report of findings.',
+    'Launch the hello-prompt fixture, test exactly three inputs (`hello world`, a blank line, and `symbols-!@#$%^&*`), capture a snapshot after each input, then send `exit` to verify clean shutdown. Save at least one screenshot and one recording, and write a brief findings report with severity and evidence references.',
     'hello-prompt',
   ),
   expectedSkill: 'dogfood-tui',
@@ -124,8 +124,9 @@ export const exploratoryQaCase = DogfoodEvalCaseSchema.parse({
   workflowChecks: [],
   antiPatterns: [...DEFAULT_ANTI_PATTERN_RULES],
   budgets: {
-    timeoutMs: 300_000,
-    maxWallClockMs: 300000,
+    timeoutMs: 600_000,
+    maxAgentSteps: 30,
+    maxWallClockMs: 600_000,
   },
 });
 

diff --git a/evals/dogfood/cases/navigation-focus-repro.ts b/evals/dogfood/cases/navigation-focus-repro.ts
@@ -134,8 +134,9 @@ export const navigationFocusReproCase = DogfoodEvalCaseSchema.parse({
   workflowChecks: [],
   antiPatterns: [...DEFAULT_ANTI_PATTERN_RULES],
   budgets: {
-    timeoutMs: 300_000,
-    maxWallClockMs: 300000,
+    timeoutMs: 600_000,
+    maxAgentSteps: 30,
+    maxWallClockMs: 600_000,
   },
 });
 

diff --git a/evals/dogfood/cases/release-readiness.ts b/evals/dogfood/cases/release-readiness.ts
@@ -68,7 +68,7 @@ export const releaseReadinessCase = DogfoodEvalCaseSchema.parse({
   lane: 'dogfood',
   category: 'release-readiness',
   prompt: dogfoodTaskPrompt(
-    'Perform a release-readiness check on the color-grid fixture. Verify color rendering across all modes (3-bit, 8-bit, 24-bit), capture visual evidence, and produce a release-readiness report.',
+    'Launch the color-grid fixture, capture three screenshots that cover the basic, 256-color, and truecolor sections, and confirm the transcript or snapshots show the expected section headers. Then write a step-by-step release-readiness report with a checklist, visual evidence, and a ship-or-hold recommendation.',
     'color-grid',
   ),
   expectedSkill: 'dogfood-tui',
@@ -133,8 +133,9 @@ export const releaseReadinessCase = DogfoodEvalCaseSchema.parse({
   workflowChecks: [],
   antiPatterns: [...DEFAULT_ANTI_PATTERN_RULES],
   budgets: {
-    timeoutMs: 300_000,
-    maxWallClockMs: 300000,
+    timeoutMs: 600_000,
+    maxAgentSteps: 30,
+    maxWallClockMs: 600_000,
   },
 });
 

diff --git a/evals/dogfood/cases/rendering-bug-repro.ts b/evals/dogfood/cases/rendering-bug-repro.ts
@@ -122,8 +122,9 @@ export const renderingBugReproCase = DogfoodEvalCaseSchema.parse({
   workflowChecks: [],
   antiPatterns: [...DEFAULT_ANTI_PATTERN_RULES],
   budgets: {
-    timeoutMs: 300_000,
-    maxWallClockMs: 300000,
+    timeoutMs: 600_000,
+    maxAgentSteps: 30,
+    maxWallClockMs: 600_000,
   },
 });
 

diff --git a/evals/dogfood/cases/resize-regression.ts b/evals/dogfood/cases/resize-regression.ts
@@ -121,8 +121,9 @@ export const resizeRegressionCase = DogfoodEvalCaseSchema.parse({
   workflowChecks: [],
   antiPatterns: [...DEFAULT_ANTI_PATTERN_RULES],
   budgets: {
-    timeoutMs: 300_000,
-    maxWallClockMs: 300000,
+    timeoutMs: 600_000,
+    maxAgentSteps: 30,
+    maxWallClockMs: 600_000,
   },
 });
 

diff --git a/evals/dogfood/runner.ts b/evals/dogfood/runner.ts
@@ -4,7 +4,10 @@ import { join, resolve } from 'node:path';
 
 import { scanBundleArtifacts } from '../../src/tools/review-bundle.js';
 import { assertString, invariant } from '../../src/util/assert.js';
-import { detectAntiPatterns } from '../lib/antiPatterns.js';
+import {
+  buildScannableTranscript,
+  detectAntiPatterns,
+} from '../lib/antiPatterns.js';
 import {
   scoreBundleCompleteness,
   scoreEvidenceQuality,
@@ -686,8 +689,11 @@ export async function runDogfoodLane(
           requestCase.workflowChecks.length === 0
             ? []
             : checkWorkflow(transcript, requestCase.workflowChecks);
+        const scannableTranscript = buildScannableTranscript(
+          agentResult.normalized,
+        );
         const antiPatternFindings = detectAntiPatterns(
-          transcript,
+          scannableTranscript,
           requestCase.antiPatterns,
         );
         const artifactManifestPath =

diff --git a/evals/execution/cases/alt-screen-demo.ts b/evals/execution/cases/alt-screen-demo.ts
@@ -22,6 +22,7 @@ export const altScreenDemoCase = createExecutionCase({
   ),
   expectedSkill: 'agent-tty',
   fixture: 'alt-screen-demo',
+  referenceSteps: 4,
   conditions: [...ALL_EXECUTION_CONDITIONS],
   setup: [
     fixtureSetupStep(

diff --git a/evals/execution/cases/color-grid.ts b/evals/execution/cases/color-grid.ts
@@ -2,9 +2,9 @@ import {
   ALL_EXECUTION_CONDITIONS,
   CREATE_SESSION_PATTERN,
   SCREENSHOT_PATTERN,
+  SNAPSHOT_PATTERN,
   WAIT_PATTERN,
   anyOf,
-  artifactRequirement,
   createExecutionCase,
   executionAntiPatterns,
   executionBudgets,
@@ -19,11 +19,12 @@ export const colorGridCase = createExecutionCase({
   lane: 'execution',
   category: 'artifact',
   prompt: executionTaskPrompt(
-    'Launch color-grid, wait for the fixture to render, and capture a screenshot of the color output for review.',
+    'Launch color-grid, wait for the fixture to render, and capture either a screenshot or a text snapshot of the color output for review.',
     'color-grid',
   ),
   expectedSkill: 'agent-tty',
   fixture: 'color-grid',
+  referenceSteps: 4,
   conditions: [...ALL_EXECUTION_CONDITIONS],
   setup: [
     fixtureSetupStep(
@@ -34,11 +35,13 @@ export const colorGridCase = createExecutionCase({
   ],
   verifiers: [
     requiredVerifier(
-      'color-grid-screenshot',
-      'screenshot',
-      'A screenshot artifact should exist for the rendered color grid.',
+      'color-grid-evidence',
+      'snapshot',
+      'The transcript should contain either screenshot evidence or a text snapshot of the rendered color grid.',
       {
-        kind: 'screenshot',
+        patterns: [
+          String.raw`(?:${SCREENSHOT_PATTERN}|(?:${SNAPSHOT_PATTERN}[\s\S]*?(?:COLOR GRID FIXTURE|Basic background colors|Bright background colors|256-color sample backgrounds|Truecolor sample backgrounds|Foreground sample labels|COLOR GRID COMPLETE)))`,
+        ],
       },
     ),
   ],
@@ -55,19 +58,22 @@ export const colorGridCase = createExecutionCase({
       { dependsOn: ['create'] },
     ),
     workflowCheck(
-      'screenshot',
-      'Capture a screenshot of the rendered color grid.',
-      SCREENSHOT_PATTERN,
+      'capture-evidence',
+      'Capture either a screenshot or a text snapshot of the rendered color grid.',
+      anyOf(SCREENSHOT_PATTERN, SNAPSHOT_PATTERN),
       { dependsOn: ['wait'] },
     ),
   ],
   antiPatterns: executionAntiPatterns(),
   artifactRequirements: [
-    artifactRequirement(
-      'screenshot',
-      'A PNG screenshot should be saved for reviewer inspection.',
-      String.raw`\.png$`,
-    ),
+    {
+      kind: 'screenshot',
+      required: false,
+      description:
+        'A PNG screenshot should be saved for reviewer inspection when renderer support is available.',
+      minCount: 1,
+      pathPatterns: [String.raw`\.png$`],
+    },
   ],
   budgets: executionBudgets({
     timeoutMs: 180_000,

diff --git a/evals/execution/cases/crash-recovery.ts b/evals/execution/cases/crash-recovery.ts
@@ -8,7 +8,6 @@ import {
   executionBudgets,
   executionTaskPrompt,
   fixtureSetupStep,
-  requiredVerifier,
   workflowCheck,
 } from './shared.js';
 
@@ -22,6 +21,7 @@ export const crashRecoveryCase = createExecutionCase({
   ),
   expectedSkill: 'agent-tty',
   fixture: 'crash-demo',
+  referenceSteps: 3,
   conditions: [...ALL_EXECUTION_CONDITIONS],
   setup: [
     fixtureSetupStep(
@@ -31,14 +31,26 @@ export const crashRecoveryCase = createExecutionCase({
     ),
   ],
   verifiers: [
-    requiredVerifier(
-      'crash-demo-exit-code',
-      'command',
-      'The crashed fixture should be observed with exit code 1.',
-      {
+    {
+      id: 'crash-demo-exit-code',
+      kind: 'command',
+      description:
+        'The crashed fixture should be observed with exit code 1 when the session metadata is available.',
+      required: false,
+      config: {
         expectedExitCode: 1,
       },
-    ),
+    },
+    {
+      id: 'crash-demo-status-observed',
+      kind: 'snapshot',
+      description:
+        'The transcript should show that the crashed session status was inspected.',
+      required: true,
+      config: {
+        patterns: [String.raw`exited|failed|exit.code|exitCode|status`],
+      },
+    },
   ],
   workflowChecks: [
     workflowCheck(

diff --git a/evals/execution/cases/doctor-gated.ts b/evals/execution/cases/doctor-gated.ts
@@ -24,6 +24,7 @@ export const doctorGatedCase = createExecutionCase({
   ),
   expectedSkill: 'agent-tty',
   fixture: 'hello-prompt',
+  referenceSteps: 5,
   conditions: [...ALL_EXECUTION_CONDITIONS],
   setup: [
     fixtureSetupStep(

diff --git a/evals/execution/cases/export-proof.ts b/evals/execution/cases/export-proof.ts
@@ -22,6 +22,7 @@ export const exportProofCase = createExecutionCase({
   ),
   expectedSkill: 'agent-tty',
   fixture: 'hello-prompt',
+  referenceSteps: 5,
   conditions: [...ALL_EXECUTION_CONDITIONS],
   setup: [
     fixtureSetupStep(

diff --git a/evals/execution/cases/hello-prompt.ts b/evals/execution/cases/hello-prompt.ts
@@ -30,6 +30,7 @@ export const helloPromptCase = createExecutionCase({
   ),
   expectedSkill: 'agent-tty',
   fixture: 'hello-prompt',
+  referenceSteps: 5,
   conditions: [...ALL_EXECUTION_CONDITIONS],
   setup: [
     fixtureSetupStep(

diff --git a/evals/execution/cases/resize-demo.ts b/evals/execution/cases/resize-demo.ts
@@ -29,6 +29,7 @@ export const resizeDemoCase = createExecutionCase({
   ),
   expectedSkill: 'agent-tty',
   fixture: 'resize-demo',
+  referenceSteps: 5,
   conditions: [...ALL_EXECUTION_CONDITIONS],
   setup: [
     fixtureSetupStep(

diff --git a/evals/execution/cases/run-command.ts b/evals/execution/cases/run-command.ts
@@ -1,10 +1,7 @@
-import type { AntiPatternRule } from '../../lib/types.js';
-
 import {
   ALL_EXECUTION_CONDITIONS,
   CREATE_SESSION_PATTERN,
   SNAPSHOT_PATTERN,
-  TYPE_PATTERN,
   WAIT_PATTERN,
   anyOf,
   createExecutionCase,
@@ -16,20 +13,9 @@ import {
   workflowCheck,
 } from './shared.js';
 
-const NO_SIMULATED_TYPING_RULE: AntiPatternRule = {
-  id: 'no-simulated-typing',
-  severity: 'error',
-  description:
-    'Detected simulated typing instead of agent-tty run for the run-command execution case.',
-  patterns: [TYPE_PATTERN],
-  suggestedFix:
-    'Use agent-tty run to send the command payload instead of typing it character-by-character.',
-  lanes: ['execution'],
-};
-
 const RUN_COMMAND_PATTERN = anyOf(
-  String.raw`\bagent-tty\b[^\n]*\brun\b[^\n]*echo test`,
-  String.raw`\brun(?:ning|s|ned)?\b[^\n]*echo test\b`,
+  String.raw`\bagent-tty\b[^\n]*\b(?:run|type|send-keys|paste)\b[^\n]*echo test`,
+  String.raw`\b(?:run|type|send-keys|paste)(?:ning|s|ned|d|ing)?\b[^\n]*echo test\b`,
   String.raw`ECHO:\s*echo test`,
 );
 
@@ -43,6 +29,7 @@ export const runCommandCase = createExecutionCase({
   ),
   expectedSkill: 'agent-tty',
   fixture: 'hello-prompt',
+  referenceSteps: 5,
   conditions: [...ALL_EXECUTION_CONDITIONS],
   setup: [
     fixtureSetupStep(
@@ -69,12 +56,9 @@ export const runCommandCase = createExecutionCase({
     ),
     workflowCheck(
       'run',
-      'Use agent-tty run instead of typing the command.',
+      'Send the command payload via any programmatic method.',
       RUN_COMMAND_PATTERN,
-      {
-        dependsOn: ['create'],
-        forbiddenPattern: TYPE_PATTERN,
-      },
+      { dependsOn: ['create'] },
     ),
     workflowCheck(
       'wait',
@@ -89,7 +73,7 @@ export const runCommandCase = createExecutionCase({
       { dependsOn: ['wait'] },
     ),
   ],
-  antiPatterns: executionAntiPatterns(NO_SIMULATED_TYPING_RULE),
+  antiPatterns: executionAntiPatterns(),
   artifactRequirements: [],
   budgets: executionBudgets({
     timeoutMs: 120_000,

diff --git a/evals/execution/cases/scrollback-demo.ts b/evals/execution/cases/scrollback-demo.ts
@@ -23,6 +23,7 @@ export const scrollbackDemoCase = createExecutionCase({
   ),
   expectedSkill: 'agent-tty',
   fixture: 'scrollback-demo',
+  referenceSteps: 4,
   conditions: [...ALL_EXECUTION_CONDITIONS],
   setup: [
     fixtureSetupStep(

diff --git a/evals/execution/cases/shared.ts b/evals/execution/cases/shared.ts
@@ -179,7 +179,7 @@ export function workflowCheck(
   return {
     id,
     description,
-    required: options.required ?? true,
+    required: options.required ?? false,
     requiredPatterns: [requiredPattern],
     forbiddenPatterns:
       options.forbiddenPattern === undefined ? [] : [options.forbiddenPattern],

diff --git a/evals/execution/cases/unicode-grid.ts b/evals/execution/cases/unicode-grid.ts
@@ -22,6 +22,7 @@ export const unicodeGridCase = createExecutionCase({
   ),
   expectedSkill: 'agent-tty',
   fixture: 'unicode-grid',
+  referenceSteps: 4,
   conditions: [...ALL_EXECUTION_CONDITIONS],
   setup: [
     fixtureSetupStep(