diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 00000000..121c7d03 --- /dev/null +++ b/evals/README.md @@ -0,0 +1,414 @@ +# Evals for `agent-tty` + +## 1. Overview + +The `evals/` tree measures whether an agent or provider uses `agent-tty` the way this repository intends. + +It answers three related questions: + +1. **Routing:** does the model recognize when `agent-tty` or `dogfood-tui` is the right workflow? +2. **Execution:** can it actually drive the fixture apps with the expected terminal workflow? +3. **Dogfooding:** can it produce reviewer-friendly proof bundles with reproducible evidence and structured reports? + +The system is intentionally deterministic. It scores outputs with schemas, regexes, workflow checks, artifact checks, bundle validation, and anti-pattern detection instead of relying on a model judge. + +## 2. Quick start + +Run everything from the repository root. + +```sh +# Stub provider (no external deps) +npx tsx evals/run.ts --provider stub --lane prompt + +# Claude Code +npx tsx evals/run.ts --provider claude --lane prompt + +# Codex +npx tsx evals/run.ts --provider codex --lane all + +# Specific cases +npx tsx evals/run.ts --provider stub --lane execution --case hello-prompt --case resize-demo +``` + +Useful flags: + +- `--condition ` — `none`, `self-load`, `preloaded`, `stale`, or `all` (default: `all`) +- `--output ` — output base directory (default: `evals/reports/{timestamp}`) +- `--dry-run` — list the case/condition matrix without invoking a provider +- `--json` — emit only a JSON summary to stdout +- `--verbose` — write per-lane progress logs to stderr + +Examples: + +```sh +# Preview the matrix without invoking a provider +npx tsx evals/run.ts --provider stub --lane all --dry-run + +# Restrict to one condition +npx tsx evals/run.ts --provider stub --lane prompt --condition self-load + +# Write results under a custom directory +npx tsx evals/run.ts --provider stub --lane execution --output evals/reports/local-smoke +``` + +## 3. Architecture + +### Lane A — prompt + +The prompt lane is a routing and planning eval. It checks whether the provider: + +- picks the right skill (`agent-tty`, `dogfood-tui`, or `none`), +- mentions the expected workflow, +- avoids forbidden patterns, +- and avoids known terminal automation anti-patterns. + +`runPromptLane()` executes plan-mode requests and scores the returned text with deterministic regex and workflow checks. + +### Lane B — execution + +The execution lane is a closed-loop fixture eval. It runs terminal tasks against deterministic fixture apps and checks: + +- provider invocation success, +- required verifiers, +- workflow compliance, +- artifact requirements, +- and anti-pattern avoidance. + +`runExecutionLane()` is where the repo proves that the recommended workflow is not just described, but actually executed. + +### Lane C — dogfood + +The dogfood lane evaluates exploratory QA and proof-bundle quality. It asks the provider to test a fixture like a reviewer would and then scores: + +- bundle completeness, +- report completeness, +- evidence quality, +- taxonomy usage, +- and reproducibility. + +`runDogfoodLane()` is the most workflow-heavy lane because it cares about evidence, reporting, and bundle structure rather than only command correctness. + +### Skill conditions + +Each lane can be run under four skill-loading conditions: + +| Condition | Meaning | Why it matters | +| ----------- | -------------------------------------------------- | --------------------------------------------------------------------------- | +| `none` | No skill text is preloaded. | Baseline behavior without assistance. | +| `self-load` | Only a bootstrap `agent-tty` prompt is preloaded. | Measures whether the model can route itself to the right specialized skill. | +| `preloaded` | The canonical specialized skill is already loaded. | Upper-bound/oracle condition for the intended workflow. | +| `stale` | A stale or mismatched skill is preloaded. | Measures harm from outdated guidance. | + +### Provider abstraction + +Providers implement one shared interface in `evals/providers/base.ts`: + +- `detect()` — runtime discovery and default model info +- `invokePlanMode()` — prompt-lane execution +- `invokeAgentMode()` — execution/dogfood execution +- `parse()` — normalization of raw provider output + +`evals/run.ts` creates one provider, runs the selected lanes, and then emits JSON + Markdown reports from the normalized `EvalResult[]` stream. + +## 4. Directory layout + +```text +evals/ +├── README.md +├── run.ts +├── lib/ +│ ├── antiPatterns.ts +│ ├── artifacts.ts +│ ├── bundleScoring.ts +│ ├── cliHarness.ts +│ ├── matrix.ts +│ ├── reporting.ts +│ ├── schemas.ts +│ ├── scoring.ts +│ └── types.ts +├── prompt/ +│ ├── cases/ +│ └── runner.ts +├── execution/ +│ ├── cases/ +│ ├── verifiers/ +│ └── runner.ts +├── dogfood/ +│ ├── cases/ +│ ├── scorers/ +│ └── runner.ts +└── providers/ + ├── base.ts + ├── claude.ts + ├── codex.ts + └── fixtures.ts +``` + +High-level roles: + +- `lib/` contains schemas, scoring, report generation, matrix math, and artifact helpers. +- `prompt/`, `execution/`, and `dogfood/` each define a lane-specific case inventory plus a runner. +- `providers/` normalizes different model backends into one interface. +- `run.ts` is the top-level orchestrator for CLI usage. + +## 5. Case inventory + +### Prompt lane + +Prompt cases currently cover positive routing, negative routing, and explicit anti-pattern prompts. + +| Case ID | Category | Expected skill | Description | +| --------------------- | -------------- | -------------- | ------------------------------------------------------------------------------- | +| `session-creation` | `trigger` | `agent-tty` | Create a long-lived terminal session and capture build output. | +| `interactive-cli` | `trigger` | `agent-tty` | Automate an interactive installer that needs prompt-aware input. | +| `wait-for-output` | `trigger` | `agent-tty` | Wait for a specific readiness string before continuing. | +| `snapshot-inspection` | `trigger` | `agent-tty` | Inspect terminal state with a semantic snapshot. | +| `screenshot-capture` | `trigger` | `agent-tty` | Capture a reviewable screenshot of a TUI state. | +| `recording-export` | `trigger` | `agent-tty` | Export a terminal session as a shareable recording or video. | +| `cli-workflow-test` | `trigger` | `agent-tty` | Test a CLI by driving multiple commands and checks. | +| `resize-verification` | `trigger` | `agent-tty` | Verify that a TUI handles terminal resize correctly. | +| `exploratory-testing` | `trigger` | `dogfood-tui` | Explore a TUI, find issues, and attach evidence. | +| `bug-hunting` | `trigger` | `dogfood-tui` | Run a broad bug hunt for rendering, input, and edge cases. | +| `release-readiness` | `trigger` | `dogfood-tui` | Produce a release-readiness quality report for a TUI. | +| `ux-review` | `trigger` | `dogfood-tui` | Review navigation, responsiveness, and visual consistency. | +| `issue-reproduction` | `trigger` | `dogfood-tui` | Reproduce a reported TUI crash and capture evidence. | +| `regression-triage` | `trigger` | `dogfood-tui` | Check whether a known regression is still present. | +| `pure-reasoning` | `trigger` | `none` | Linux process-scheduling question that should not trigger tooling. | +| `code-review` | `trigger` | `none` | Source review request that should stay in normal coding mode. | +| `file-editing` | `trigger` | `none` | File-edit request that does not need terminal automation. | +| `web-development` | `trigger` | `none` | React component authoring task that does not need `agent-tty`. | +| `documentation` | `trigger` | `none` | API documentation request that should not route to a skill. | +| `git-operations` | `trigger` | `none` | Git workflow request that should stay outside terminal automation eval routing. | +| `blind-sleep` | `anti-pattern` | `agent-tty` | Tempts the provider to use brittle `sleep` instead of waiting on state. | +| `tmux-usage` | `anti-pattern` | `agent-tty` | Tempts the provider to use `tmux` instead of `agent-tty`. | +| `screen-usage` | `anti-pattern` | `agent-tty` | Tempts the provider to use `screen` instead of `agent-tty`. | +| `adhoc-screenshots` | `anti-pattern` | `agent-tty` | Tempts the provider to use `scrot`/`import`/similar screenshot tools. | + +### Execution lane + +| Case ID | Category | Fixture | Conditions | Description | +| ----------------- | ---------- | ----------------- | ---------- | ---------------------------------------------------------------------------------- | +| `hello-prompt` | `session` | `hello-prompt` | all four | Launch the fixture, send `hello world`, wait for `READY>`, snapshot, and clean up. | +| `resize-demo` | `tui` | `resize-demo` | all four | Resize from the default size to `100x30` and verify the reported dimensions. | +| `alt-screen-demo` | `tui` | `alt-screen-demo` | all four | Capture evidence for alt-screen entry and main-screen restoration. | +| `color-grid` | `artifact` | `color-grid` | all four | Wait for the grid to render and capture a screenshot artifact. | +| `unicode-grid` | `artifact` | `unicode-grid` | all four | Verify Unicode rendering with a semantic snapshot. | +| `scrollback-demo` | `tui` | `scrollback-demo` | all four | Capture scrollback evidence showing early and late buffer lines. | +| `crash-recovery` | `recovery` | `crash-demo` | all four | Observe a crashing fixture, inspect status, and clean up the session. | +| `export-proof` | `artifact` | `hello-prompt` | all four | Export the session as both asciicast and WebM artifacts. | +| `run-command` | `session` | `hello-prompt` | all four | Use `agent-tty run` instead of simulated typing, then snapshot the result. | +| `doctor-gated` | `artifact` | `hello-prompt` | all four | Run `doctor --json` before taking a renderer-dependent screenshot. | + +### Dogfood lane + +| Case ID | Category | Fixture | Conditions | Description | +| ------------------------ | ------------------- | ----------------- | ---------- | ---------------------------------------------------------------------------------------------------- | +| `exploratory-qa` | `qa` | `hello-prompt` | all four | Run exploratory QA, collect screenshots/recordings, and write structured findings. | +| `release-readiness` | `release-readiness` | `color-grid` | all four | Evaluate color rendering across modes and produce a ship-or-hold report. | +| `rendering-bug-repro` | `bug-repro` | `unicode-grid` | all four | Reproduce narrow-width combining-character corruption with before/after evidence. | +| `navigation-focus-repro` | `bug-repro` | `hello-prompt` | all four | Reproduce a suspected paste/focus input issue and classify it with the taxonomy. | +| `resize-regression` | `bug-repro` | `resize-demo` | all four | Triage stale resize output with evidence at initial, resized, and restored states. | +| `evidence-completeness` | `reporting` | `scrollback-demo` | all four | Produce the fullest possible proof bundle: screenshots, cast, WebM, snapshots, notes, and checklist. | + +## 6. Scoring + +All scoring is deterministic. + +### Prompt scoring + +Prompt-lane scoring combines three positive components and then subtracts penalties: + +- expected-pattern coverage: **0.4** +- skill-selection correctness: **0.4** +- workflow compliance: **0.2** +- forbidden-pattern penalty: **-0.1** per violation +- anti-pattern penalty: **-0.05** per finding + +A prompt case only passes when all expected patterns match, no forbidden patterns fire, the inferred skill matches the expected skill, every required workflow check passes, and no anti-patterns are detected. + +### Execution scoring + +Execution-lane scoring builds a breakdown from: + +- provider invocation success, +- required verifier pass rate, +- required workflow-check pass rate, +- anti-pattern avoidance, +- and artifact-requirement pass rate (when a case has required artifacts). + +The execution case passes only when the provider call succeeds, all required verifiers pass, all required workflow checks pass, no error-severity anti-patterns are present, and all required artifacts exist. + +### Dogfood scoring + +Dogfood-lane scoring uses equal **20%** weights for: + +- bundle completeness, +- report completeness, +- evidence quality, +- taxonomy usage, +- reproducibility. + +A dogfood case passes only when the provider invocation succeeds, required report/workflow expectations are satisfied, no blocking anti-patterns are present, and the overall score is at least **0.6**. + +### Anti-pattern detection and workflow compliance + +Across lanes, the system explicitly looks for common workflow mistakes such as: + +- blind `sleep` usage, +- `tmux` or `screen` instead of `agent-tty` sessions, +- ad hoc screenshot tools like `scrot`, `import`, or `xdotool`, +- missing `--json` on `agent-tty` commands, +- and session cleanup problems. + +That means the evals reward not only “getting the answer” but following the repo’s intended automation workflow. + +## 7. Comparison metrics + +`evals/lib/matrix.ts` computes condition-to-condition metrics per provider × lane × case group. + +| Metric | Definition | +| ------------------- | ------------------------------------------------------------------------------- | +| `realizedSkillLift` | Mean(`self-load`) - Mean(`none`) | +| `oracleSkillLift` | Mean(`preloaded`) - Mean(`none`) | +| `routingGap` | `oracleSkillLift - realizedSkillLift` | +| `staleSkillHarm` | Mean(`none`) - Mean(`stale`) | +| `regressionRate` | `1` when `self-load < none`, else `0` | +| `unlockRate` | `1` when `self-load > none`, else `0` | +| `routingEfficiency` | `clamp(realizedSkillLift / oracleSkillLift, 0, 1)` when oracle lift is positive | + +How to read them: + +- **Realized lift** tells you how much self-routing helped in practice. +- **Oracle lift** tells you how much headroom exists when the right skill is already loaded. +- **Routing gap** is the remaining opportunity between current self-routing and the oracle condition. +- **Stale-skill harm** measures how much outdated preload hurts compared with no preload. +- **Regression/unlock rates** summarize whether self-loading regresses or unlocks success case-by-case. +- **Routing efficiency** normalizes realized lift against the oracle upper bound. + +When you run only one condition, comparison sections are intentionally omitted because there is nothing meaningful to compare. + +## 8. Provider support + +### Stub provider + +`stub` is the safest local smoke-test provider. It has no external dependencies and returns deterministic canned outputs. + +### Fixture provider + +`fixture` replays pre-recorded runtime/result payloads from a fixture directory. `evals/run.ts` expects that path in `EVAL_FIXTURE_DIR`. + +### Claude Code adapter + +`evals/providers/claude.ts` shells out to the `claude` CLI, supports runtime detection, plan mode, agent mode, transcript capture, and tool-call normalization when the provider output includes it. + +### Codex adapter + +`evals/providers/codex.ts` shells out to the `codex` CLI and exposes the same normalized surface: detection, plan mode, agent mode, transcript capture, and parsed tool-call output. + +### Choosing a provider + +- Use `stub` for fast local validation and CI smoke tests. +- Use `fixture` for deterministic replay-based experiments. +- Use `claude` or `codex` when you want real routing/execution behavior. + +## 9. Adding cases + +Case authors should follow a few rules: + +1. **Validate with schemas.** New cases should be created through the existing helpers and parsed by the relevant Zod schema. +2. **Prefer fixtures over ambient state.** Execution and dogfood cases should target deterministic fixture apps so failures are attributable. +3. **Balance routing.** Prompt cases should include true positives, true negatives, and anti-pattern bait — not only “always trigger” prompts. +4. **Keep verifiers deterministic.** Prefer snapshots, screenshots, event-log checks, bundle validation, and explicit pattern matching over subjective judgment. +5. **Make workflow expectations explicit.** Add workflow checks for required steps like `wait`, `snapshot`, `doctor --json`, `record export`, or cleanup. +6. **Keep case IDs stable.** Reports, filtering, and comparisons all key off case IDs. + +A practical authoring loop is: + +1. add the case file, +2. register it in the lane runner, +3. run `npx tsx evals/run.ts --provider stub --lane --case --dry-run`, +4. then run the real lane against `stub` or a real provider. + +## 10. CI integration + +A useful rollout pattern is to tier eval coverage by cost and stability. + +### PR tier + +Keep pull-request coverage cheap and deterministic: + +```sh +npx tsx evals/run.ts --provider stub --lane prompt +npx tsx evals/run.ts --provider stub --lane execution --dry-run +``` + +### Nightly tier + +Run broader smoke coverage with either `stub`, `fixture`, or one real provider on a smaller slice: + +```sh +npx tsx evals/run.ts --provider stub --lane all +npx tsx evals/run.ts --provider codex --lane prompt --condition all +``` + +### Weekly tier + +Use the expensive schedule for full condition-matrix comparisons and dogfood coverage: + +```sh +npx tsx evals/run.ts --provider claude --lane all --condition all +npx tsx evals/run.ts --provider codex --lane all --condition all +``` + +Practical advice: + +- gate real-provider jobs behind credentials and budget controls, +- keep `stub` coverage always-on, +- and use `--json` when another CI step needs to parse the summary. + +## 11. Reports + +Each run writes its outputs under the chosen output base directory in a run-specific subdirectory: + +```text +/ +└── / + ├── metadata.json + ├── report.json + └── report.md +``` + +### `metadata.json` + +Contains the normalized `RunMetadata` for the run: + +- run ID and creation time, +- repo root, +- selected provider(s) and detected model(s), +- active lanes and conditions, +- trial count, +- and notes such as provider detection details or per-lane failures. + +### `report.json` + +Contains the machine-readable aggregate report generated by `generateJsonReport()`: + +- top-level metadata, +- aggregate pass/fail and score metrics, +- condition comparison metrics, +- every normalized `EvalResult`, +- and provider-comparison views when relevant. + +### `report.md` + +Contains the reviewer-oriented Markdown report generated by `generateMarkdownReport()`: + +- executive summary, +- lane breakdown, +- provider comparison, +- condition comparison, +- failed cases, +- anti-pattern summary, +- and completeness rollups. + +Use `--json` when you want only the final run summary on stdout; the full report files are still written to disk. diff --git a/evals/dogfood/cases/evidence-completeness.ts b/evals/dogfood/cases/evidence-completeness.ts new file mode 100644 index 00000000..361b9524 --- /dev/null +++ b/evals/dogfood/cases/evidence-completeness.ts @@ -0,0 +1,131 @@ +import { DEFAULT_ANTI_PATTERN_RULES } from '../../lib/antiPatterns.js'; +import { SKILL_CONDITIONS } from '../../lib/matrix.js'; +import { DogfoodEvalCaseSchema } from '../../lib/schemas.js'; +import { dogfoodTaskPrompt } from './shared.js'; +import type { + ArtifactRequirement, + ReportRequirement, + VerifierSpec, +} from '../../lib/types.js'; + +function requiredArtifact( + kind: ArtifactRequirement['kind'], + description: string, + pathPatterns: string[], + minCount = 1, +): ArtifactRequirement { + return { + kind, + required: true, + description, + minCount, + pathPatterns, + }; +} + +function reportRequirement( + id: string, + section: string, + description: string, + requiredPatterns: string[], +): ReportRequirement { + return { + id, + section, + description, + required: true, + requiredPatterns, + forbiddenPatterns: [], + }; +} + +const verifier: VerifierSpec = { + id: 'bundle-valid', + kind: 'bundle', + description: + 'Validate the evidence-completeness bundle with the interactive renderer profile.', + required: true, + config: { + profile: 'interactive-renderer', + }, +}; + +export const evidenceCompletenessCase = DogfoodEvalCaseSchema.parse({ + id: 'evidence-completeness', + lane: 'dogfood', + category: 'reporting', + prompt: dogfoodTaskPrompt( + 'Test the scrollback-demo fixture and produce the most complete evidence bundle possible: screenshots, recordings, WebM exports, snapshots, notes, and a structured report following the full evidence checklist.', + 'scrollback-demo', + ), + expectedSkill: 'dogfood-tui', + fixture: 'scrollback-demo', + bundlePath: 'proof-bundle', + bundleRequirements: [ + 'Produce the most complete proof bundle possible for the scrollback-demo fixture.', + 'Include screenshots, recordings, WebM exports, snapshots, and notes in one reviewable bundle.', + 'Follow the full evidence checklist, including commands, dimensions, and cleanup notes.', + ], + conditions: [...SKILL_CONDITIONS], + validationProfile: 'interactive-renderer', + artifactRequirements: [ + requiredArtifact( + 'screenshot', + 'Capture at least one screenshot artifact.', + [String.raw`\.png$`], + ), + requiredArtifact('video', 'Export at least one WebM review artifact.', [ + String.raw`\.webm$`, + ]), + requiredArtifact( + 'recording', + 'Capture at least one terminal recording artifact.', + [String.raw`\.cast$`], + ), + requiredArtifact( + 'json', + 'Capture at least one snapshot artifact for searchable evidence.', + [String.raw`(?:^|/).*snapshot.*\.json$`], + ), + requiredArtifact( + 'notes', + 'Write the evidence checklist report in markdown.', + [String.raw`(?:^|/)(?:README|NOTES|index|notes)\.md$`], + ), + ], + reportRequirements: [ + reportRequirement('title', 'Title', 'Test report title.', [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Title\b|\*\*Title:?\*\*)/im`, + ]), + reportRequirement('commands', 'Commands', 'All commands executed.', [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Commands\b|\*\*Commands:?\*\*)/im`, + String.raw`/\b(?:agent-tty|npx\s+tsx\s+src\/cli\/main\.ts)\b/i`, + ]), + reportRequirement('dimensions', 'Dimensions', 'Terminal dimensions used.', [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Dimensions\b|\*\*Dimensions:?\*\*)/im`, + String.raw`/\b(?:\d+\s*[x×]\s*\d+|columns|rows|terminal dimensions)\b/i`, + ]), + reportRequirement( + 'evidence-checklist', + 'Evidence checklist', + 'Complete evidence checklist.', + [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Evidence checklist\b|\*\*Evidence checklist:?\*\*)/im`, + String.raw`/\b(?:screenshot|webm|recording|snapshot|notes)\b/i`, + ], + ), + reportRequirement('cleanup', 'Cleanup', 'Session cleanup confirmation.', [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Cleanup\b|\*\*Cleanup:?\*\*)/im`, + String.raw`/\b(?:destroy|cleanup|cleaned up|session cleanup)\b/i`, + ]), + ], + verifiers: [verifier], + workflowChecks: [], + antiPatterns: [...DEFAULT_ANTI_PATTERN_RULES], + budgets: { + timeoutMs: 300_000, + maxWallClockMs: 300000, + }, +}); + +export default evidenceCompletenessCase; diff --git a/evals/dogfood/cases/exploratory-qa.ts b/evals/dogfood/cases/exploratory-qa.ts new file mode 100644 index 00000000..278ef158 --- /dev/null +++ b/evals/dogfood/cases/exploratory-qa.ts @@ -0,0 +1,132 @@ +import { DEFAULT_ANTI_PATTERN_RULES } from '../../lib/antiPatterns.js'; +import { SKILL_CONDITIONS } from '../../lib/matrix.js'; +import { DogfoodEvalCaseSchema } from '../../lib/schemas.js'; +import { dogfoodTaskPrompt } from './shared.js'; +import type { + ArtifactRequirement, + ReportRequirement, + VerifierSpec, +} from '../../lib/types.js'; + +function requiredArtifact( + kind: ArtifactRequirement['kind'], + description: string, + pathPatterns: string[], + minCount = 1, +): ArtifactRequirement { + return { + kind, + required: true, + description, + minCount, + pathPatterns, + }; +} + +function reportRequirement( + id: string, + section: string, + description: string, + requiredPatterns: string[], +): ReportRequirement { + return { + id, + section, + description, + required: true, + requiredPatterns, + forbiddenPatterns: [], + }; +} + +const verifier: VerifierSpec = { + id: 'bundle-valid', + kind: 'bundle', + description: + 'Validate the exploratory QA proof bundle with the interactive renderer profile.', + required: true, + config: { + profile: 'interactive-renderer', + }, +}; + +export const exploratoryQaCase = DogfoodEvalCaseSchema.parse({ + id: 'exploratory-qa', + lane: 'dogfood', + category: 'qa', + prompt: dogfoodTaskPrompt( + 'Perform exploratory QA testing on the hello-prompt fixture app. Test input handling, exit behavior, error codes, and edge cases. Produce a proof bundle with screenshots, recordings, and a structured report of findings.', + 'hello-prompt', + ), + expectedSkill: 'dogfood-tui', + fixture: 'hello-prompt', + bundlePath: 'proof-bundle', + bundleRequirements: [ + 'Produce a reviewable proof bundle for an exploratory QA investigation.', + 'Capture renderer-backed evidence for the tested interactions and edge cases.', + 'Write structured notes that summarize findings, severity, and evidence references.', + ], + conditions: [...SKILL_CONDITIONS], + validationProfile: 'interactive-renderer', + artifactRequirements: [ + requiredArtifact( + 'screenshot', + 'Capture at least one screenshot of a noteworthy state.', + [String.raw`\.png$`], + ), + requiredArtifact( + 'recording', + 'Capture at least one terminal recording artifact.', + [String.raw`\.cast$`], + ), + requiredArtifact( + 'notes', + 'Write exploratory QA notes in a markdown report.', + [String.raw`(?:^|/)(?:README|NOTES|index|notes)\.md$`], + ), + ], + reportRequirements: [ + reportRequirement( + 'title', + 'Title', + 'Report must have a descriptive title.', + [String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Title\b|\*\*Title:?\*\*)/im`], + ), + reportRequirement( + 'repro-steps', + 'Reproduction steps', + 'Include step-by-step reproduction commands.', + [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*(?:Reproduction steps|Repro(?:duction)? steps|Steps)\b|\*\*(?:Reproduction steps|Repro(?:duction)? steps|Steps):?\*\*)/im`, + String.raw`/\b(?:agent-tty|npx\s+tsx\s+src\/cli\/main\.ts)\b/i`, + ], + ), + reportRequirement( + 'findings', + 'Findings', + 'List findings with severity classification.', + [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*(?:Findings|Issues)\b|\*\*(?:Findings|Issues):?\*\*)/im`, + String.raw`/\b(?:severity|critical|high|medium|low|info)\b/i`, + ], + ), + reportRequirement( + 'evidence', + 'Evidence', + 'Reference captured artifacts such as screenshots and recordings.', + [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Evidence\b|\*\*Evidence:?\*\*)/im`, + String.raw`/\.(?:png|cast|webm|json|md)\b/i`, + ], + ), + ], + verifiers: [verifier], + workflowChecks: [], + antiPatterns: [...DEFAULT_ANTI_PATTERN_RULES], + budgets: { + timeoutMs: 300_000, + maxWallClockMs: 300000, + }, +}); + +export default exploratoryQaCase; diff --git a/evals/dogfood/cases/navigation-focus-repro.ts b/evals/dogfood/cases/navigation-focus-repro.ts new file mode 100644 index 00000000..8ff57f1f --- /dev/null +++ b/evals/dogfood/cases/navigation-focus-repro.ts @@ -0,0 +1,142 @@ +import { DEFAULT_ANTI_PATTERN_RULES } from '../../lib/antiPatterns.js'; +import { SKILL_CONDITIONS } from '../../lib/matrix.js'; +import { DogfoodEvalCaseSchema } from '../../lib/schemas.js'; +import { dogfoodTaskPrompt } from './shared.js'; +import type { + ArtifactRequirement, + ReportRequirement, + VerifierSpec, +} from '../../lib/types.js'; + +function requiredArtifact( + kind: ArtifactRequirement['kind'], + description: string, + pathPatterns: string[], + minCount = 1, +): ArtifactRequirement { + return { + kind, + required: true, + description, + minCount, + pathPatterns, + }; +} + +function optionalArtifact( + kind: ArtifactRequirement['kind'], + description: string, + pathPatterns: string[], +): ArtifactRequirement { + return { + kind, + required: false, + description, + pathPatterns, + }; +} + +function reportRequirement( + id: string, + section: string, + description: string, + requiredPatterns: string[], +): ReportRequirement { + return { + id, + section, + description, + required: true, + requiredPatterns, + forbiddenPatterns: [], + }; +} + +const verifier: VerifierSpec = { + id: 'bundle-valid', + kind: 'bundle', + description: + 'Validate the navigation or focus issue bundle with the contract reporting profile.', + required: true, + config: { + profile: 'contract-reporting', + }, +}; + +export const navigationFocusReproCase = DogfoodEvalCaseSchema.parse({ + id: 'navigation-focus-repro', + lane: 'dogfood', + category: 'bug-repro', + prompt: dogfoodTaskPrompt( + 'Investigate a reported focus/input issue: the hello-prompt fixture may not respond to paste input in certain sequences. Reproduce, capture evidence, and write a structured report.', + 'hello-prompt', + ), + expectedSkill: 'dogfood-tui', + fixture: 'hello-prompt', + bundlePath: 'proof-bundle', + bundleRequirements: [ + 'Produce a structured issue-reproduction bundle for the suspected focus or input defect.', + 'Capture searchable evidence and optional motion proof for paste or focus handling.', + 'Classify the issue using the dogfood taxonomy and document exact reproduction steps.', + ], + conditions: [...SKILL_CONDITIONS], + validationProfile: 'contract-reporting', + artifactRequirements: [ + requiredArtifact( + 'json', + 'Capture at least one snapshot artifact showing the focus or paste state.', + [String.raw`(?:^|/).*snapshot.*\.json$`], + ), + optionalArtifact( + 'recording', + 'Optionally capture a terminal recording of the problematic input sequence.', + [String.raw`\.cast$`], + ), + requiredArtifact( + 'notes', + 'Write a structured focus or input report in markdown.', + [String.raw`(?:^|/)(?:README|NOTES|index|notes)\.md$`], + ), + ], + reportRequirements: [ + reportRequirement('title', 'Title', 'Issue title.', [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Title\b|\*\*Title:?\*\*)/im`, + ]), + reportRequirement( + 'repro-steps', + 'Reproduction steps', + 'Step-by-step reproduction.', + [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*(?:Reproduction steps|Repro(?:duction)? steps|Steps)\b|\*\*(?:Reproduction steps|Repro(?:duction)? steps|Steps):?\*\*)/im`, + String.raw`/\b(?:paste|Ctrl\+V|send-keys|type)\b/i`, + ], + ), + reportRequirement( + 'taxonomy', + 'Taxonomy', + 'Classify using issue taxonomy (focus/input).', + [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*(?:Taxonomy|Classification)\b|\*\*(?:Taxonomy|Classification):?\*\*)/im`, + String.raw`/\bfocus\/input\b/i`, + ], + ), + reportRequirement( + 'evidence', + 'Evidence', + 'Snapshot or recording evidence.', + [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Evidence\b|\*\*Evidence:?\*\*)/im`, + String.raw`/\.(?:json|cast|md)\b/i`, + ], + ), + ], + verifiers: [verifier], + workflowChecks: [], + antiPatterns: [...DEFAULT_ANTI_PATTERN_RULES], + budgets: { + timeoutMs: 300_000, + maxWallClockMs: 300000, + }, +}); + +export default navigationFocusReproCase; diff --git a/evals/dogfood/cases/release-readiness.ts b/evals/dogfood/cases/release-readiness.ts new file mode 100644 index 00000000..be6fafce --- /dev/null +++ b/evals/dogfood/cases/release-readiness.ts @@ -0,0 +1,141 @@ +import { DEFAULT_ANTI_PATTERN_RULES } from '../../lib/antiPatterns.js'; +import { SKILL_CONDITIONS } from '../../lib/matrix.js'; +import { DogfoodEvalCaseSchema } from '../../lib/schemas.js'; +import { dogfoodTaskPrompt } from './shared.js'; +import type { + ArtifactRequirement, + ReportRequirement, + VerifierSpec, +} from '../../lib/types.js'; + +function requiredArtifact( + kind: ArtifactRequirement['kind'], + description: string, + pathPatterns: string[], + minCount = 1, +): ArtifactRequirement { + return { + kind, + required: true, + description, + minCount, + pathPatterns, + }; +} + +function optionalArtifact( + kind: ArtifactRequirement['kind'], + description: string, + pathPatterns: string[], +): ArtifactRequirement { + return { + kind, + required: false, + description, + pathPatterns, + }; +} + +function reportRequirement( + id: string, + section: string, + description: string, + requiredPatterns: string[], +): ReportRequirement { + return { + id, + section, + description, + required: true, + requiredPatterns, + forbiddenPatterns: [], + }; +} + +const verifier: VerifierSpec = { + id: 'bundle-valid', + kind: 'bundle', + description: + 'Validate the release-readiness proof bundle with the interactive renderer profile.', + required: true, + config: { + profile: 'interactive-renderer', + }, +}; + +export const releaseReadinessCase = DogfoodEvalCaseSchema.parse({ + id: 'release-readiness', + lane: 'dogfood', + category: 'release-readiness', + prompt: dogfoodTaskPrompt( + 'Perform a release-readiness check on the color-grid fixture. Verify color rendering across all modes (3-bit, 8-bit, 24-bit), capture visual evidence, and produce a release-readiness report.', + 'color-grid', + ), + expectedSkill: 'dogfood-tui', + fixture: 'color-grid', + bundlePath: 'proof-bundle', + bundleRequirements: [ + 'Produce a release-readiness proof bundle for the color-grid fixture.', + 'Capture visual evidence across 3-bit, 8-bit, and 24-bit color modes.', + 'Summarize readiness status with a ship-or-hold recommendation backed by evidence.', + ], + conditions: [...SKILL_CONDITIONS], + validationProfile: 'interactive-renderer', + artifactRequirements: [ + requiredArtifact( + 'screenshot', + 'Capture screenshots for each rendering mode under evaluation.', + [String.raw`\.png$`], + 3, + ), + requiredArtifact('notes', 'Write a release-readiness report in markdown.', [ + String.raw`(?:^|/)(?:README|NOTES|index|notes)\.md$`, + ]), + optionalArtifact( + 'recording', + 'Optionally capture a terminal recording of mode switching or navigation.', + [String.raw`\.cast$`], + ), + ], + reportRequirements: [ + reportRequirement('title', 'Title', 'Release readiness report title.', [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Title\b|\*\*Title:?\*\*)/im`, + ]), + reportRequirement( + 'checklist', + 'Checklist', + 'Readiness checklist with pass/fail items.', + [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Checklist\b|\*\*Checklist:?\*\*)/im`, + String.raw`/\b(?:pass|fail)\b/i`, + ], + ), + reportRequirement( + 'visual-evidence', + 'Visual evidence', + 'Screenshot evidence for each rendering mode.', + [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Visual evidence\b|\*\*Visual evidence:?\*\*)/im`, + String.raw`/\.(?:png|webm|cast)\b/i`, + ], + ), + reportRequirement( + 'recommendation', + 'Recommendation', + 'Ship or hold recommendation with rationale.', + [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Recommendation\b|\*\*Recommendation:?\*\*)/im`, + String.raw`/\b(?:ship|hold)\b/i`, + ], + ), + ], + verifiers: [verifier], + workflowChecks: [], + antiPatterns: [...DEFAULT_ANTI_PATTERN_RULES], + budgets: { + timeoutMs: 300_000, + maxWallClockMs: 300000, + }, +}); + +export default releaseReadinessCase; diff --git a/evals/dogfood/cases/rendering-bug-repro.ts b/evals/dogfood/cases/rendering-bug-repro.ts new file mode 100644 index 00000000..003bf526 --- /dev/null +++ b/evals/dogfood/cases/rendering-bug-repro.ts @@ -0,0 +1,130 @@ +import { DEFAULT_ANTI_PATTERN_RULES } from '../../lib/antiPatterns.js'; +import { SKILL_CONDITIONS } from '../../lib/matrix.js'; +import { DogfoodEvalCaseSchema } from '../../lib/schemas.js'; +import { dogfoodTaskPrompt } from './shared.js'; +import type { + ArtifactRequirement, + ReportRequirement, + VerifierSpec, +} from '../../lib/types.js'; + +function requiredArtifact( + kind: ArtifactRequirement['kind'], + description: string, + pathPatterns: string[], + minCount = 1, +): ArtifactRequirement { + return { + kind, + required: true, + description, + minCount, + pathPatterns, + }; +} + +function reportRequirement( + id: string, + section: string, + description: string, + requiredPatterns: string[], +): ReportRequirement { + return { + id, + section, + description, + required: true, + requiredPatterns, + forbiddenPatterns: [], + }; +} + +const verifier: VerifierSpec = { + id: 'bundle-valid', + kind: 'bundle', + description: + 'Validate the rendering bug reproduction proof bundle with the interactive renderer profile.', + required: true, + config: { + profile: 'interactive-renderer', + }, +}; + +export const renderingBugReproCase = DogfoodEvalCaseSchema.parse({ + id: 'rendering-bug-repro', + lane: 'dogfood', + category: 'bug-repro', + prompt: dogfoodTaskPrompt( + 'Reproduce a reported rendering issue: the unicode-grid fixture displays combining characters incorrectly when the terminal is narrower than 80 columns. Capture before/after evidence and write a bug report.', + 'unicode-grid', + ), + expectedSkill: 'dogfood-tui', + fixture: 'unicode-grid', + bundlePath: 'proof-bundle', + bundleRequirements: [ + 'Produce a reproducible bug-report bundle for the unicode-grid rendering issue.', + 'Capture before-and-after evidence for the narrow-terminal rendering problem.', + 'Document expected versus actual behavior with artifact references.', + ], + conditions: [...SKILL_CONDITIONS], + validationProfile: 'interactive-renderer', + artifactRequirements: [ + requiredArtifact( + 'screenshot', + 'Capture before-and-after screenshots of the rendering issue.', + [String.raw`\.png$`], + 2, + ), + requiredArtifact( + 'json', + 'Capture at least one snapshot artifact for searchable evidence.', + [String.raw`(?:^|/).*snapshot.*\.json$`], + ), + requiredArtifact('notes', 'Write the bug report in markdown notes.', [ + String.raw`(?:^|/)(?:README|NOTES|index|notes)\.md$`, + ]), + ], + reportRequirements: [ + reportRequirement('title', 'Title', 'Bug report title with taxonomy.', [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Title\b|\*\*Title:?\*\*)/im`, + String.raw`/\b(?:rendering corruption|rendering)\b/i`, + ]), + reportRequirement( + 'repro-steps', + 'Reproduction steps', + 'Exact reproduction steps.', + [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*(?:Reproduction steps|Repro(?:duction)? steps|Steps)\b|\*\*(?:Reproduction steps|Repro(?:duction)? steps|Steps):?\*\*)/im`, + String.raw`/\b(?:80\s*[x×]\s*\d+|narrower than 80|80 columns)\b/i`, + ], + ), + reportRequirement( + 'expected-vs-actual', + 'Expected vs actual', + 'Expected vs actual behavior.', + [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Expected vs actual\b|\*\*Expected vs actual:?\*\*)/im`, + String.raw`/\bexpected\b/i`, + String.raw`/\bactual\b/i`, + ], + ), + reportRequirement( + 'evidence', + 'Evidence', + 'Before and after screenshots or snapshots.', + [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Evidence\b|\*\*Evidence:?\*\*)/im`, + String.raw`/\.(?:png|json|md)\b/i`, + ], + ), + ], + verifiers: [verifier], + workflowChecks: [], + antiPatterns: [...DEFAULT_ANTI_PATTERN_RULES], + budgets: { + timeoutMs: 300_000, + maxWallClockMs: 300000, + }, +}); + +export default renderingBugReproCase; diff --git a/evals/dogfood/cases/resize-regression.ts b/evals/dogfood/cases/resize-regression.ts new file mode 100644 index 00000000..8a1af0cf --- /dev/null +++ b/evals/dogfood/cases/resize-regression.ts @@ -0,0 +1,129 @@ +import { DEFAULT_ANTI_PATTERN_RULES } from '../../lib/antiPatterns.js'; +import { SKILL_CONDITIONS } from '../../lib/matrix.js'; +import { DogfoodEvalCaseSchema } from '../../lib/schemas.js'; +import { dogfoodTaskPrompt } from './shared.js'; +import type { + ArtifactRequirement, + ReportRequirement, + VerifierSpec, +} from '../../lib/types.js'; + +function requiredArtifact( + kind: ArtifactRequirement['kind'], + description: string, + pathPatterns: string[], + minCount = 1, +): ArtifactRequirement { + return { + kind, + required: true, + description, + minCount, + pathPatterns, + }; +} + +function reportRequirement( + id: string, + section: string, + description: string, + requiredPatterns: string[], +): ReportRequirement { + return { + id, + section, + description, + required: true, + requiredPatterns, + forbiddenPatterns: [], + }; +} + +const verifier: VerifierSpec = { + id: 'bundle-valid', + kind: 'bundle', + description: + 'Validate the resize regression proof bundle with the interactive renderer profile.', + required: true, + config: { + profile: 'interactive-renderer', + }, +}; + +export const resizeRegressionCase = DogfoodEvalCaseSchema.parse({ + id: 'resize-regression', + lane: 'dogfood', + category: 'bug-repro', + prompt: dogfoodTaskPrompt( + 'Triage a potential resize regression: the resize-demo fixture should update SIZE output after terminal resize, but users report stale values. Test resize from 80x24 to 120x40 and back. Capture evidence at each step.', + 'resize-demo', + ), + expectedSkill: 'dogfood-tui', + fixture: 'resize-demo', + bundlePath: 'proof-bundle', + bundleRequirements: [ + 'Produce a resize-regression proof bundle for the resize-demo fixture.', + 'Capture evidence for the initial state, the 120x40 resize, and the return to 80x24.', + 'Document expected and actual SIZE values observed at each step.', + ], + conditions: [...SKILL_CONDITIONS], + validationProfile: 'interactive-renderer', + artifactRequirements: [ + requiredArtifact( + 'json', + 'Capture snapshots for each resize step.', + [String.raw`(?:^|/).*snapshot.*\.json$`], + 3, + ), + requiredArtifact( + 'screenshot', + 'Capture at least one screenshot of the resize output.', + [String.raw`\.png$`], + ), + requiredArtifact('notes', 'Write regression-triage notes in markdown.', [ + String.raw`(?:^|/)(?:README|NOTES|index|notes)\.md$`, + ]), + ], + reportRequirements: [ + reportRequirement('title', 'Title', 'Regression report title.', [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Title\b|\*\*Title:?\*\*)/im`, + ]), + reportRequirement( + 'repro-steps', + 'Reproduction steps', + 'Resize sequence with expected outputs.', + [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*(?:Reproduction steps|Repro(?:duction)? steps|Steps)\b|\*\*(?:Reproduction steps|Repro(?:duction)? steps|Steps):?\*\*)/im`, + String.raw`/\b(?:80\s*[x×]\s*24|120\s*[x×]\s*40)\b/i`, + ], + ), + reportRequirement( + 'expected-vs-actual', + 'Expected vs actual', + 'Size values before and after each resize.', + [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Expected vs actual\b|\*\*Expected vs actual:?\*\*)/im`, + String.raw`/\bSIZE\b/i`, + String.raw`/\b(?:80\s*[x×]\s*24|120\s*[x×]\s*40)\b/i`, + ], + ), + reportRequirement( + 'evidence', + 'Evidence', + 'Snapshots at each resize step.', + [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*Evidence\b|\*\*Evidence:?\*\*)/im`, + String.raw`/\.(?:json|png|md)\b/i`, + ], + ), + ], + verifiers: [verifier], + workflowChecks: [], + antiPatterns: [...DEFAULT_ANTI_PATTERN_RULES], + budgets: { + timeoutMs: 300_000, + maxWallClockMs: 300000, + }, +}); + +export default resizeRegressionCase; diff --git a/evals/dogfood/cases/shared.ts b/evals/dogfood/cases/shared.ts new file mode 100644 index 00000000..d5ceec27 --- /dev/null +++ b/evals/dogfood/cases/shared.ts @@ -0,0 +1,18 @@ +import { invariant } from '../../../src/util/assert.js'; + +export function dogfoodTaskPrompt(task: string, fixture?: string): string { + const normalizedTask = task.trim(); + invariant(normalizedTask.length > 0, 'dogfood task prompt must not be empty'); + + return [ + 'ACTUALLY PERFORM this dogfood task by running agent-tty CLI commands via `npx tsx src/cli/main.ts`; do not only describe what you would test.', + 'Use the isolated `AGENT_TTY_HOME` provided for this eval so session state and artifacts stay contained.', + fixture === undefined + ? undefined + : `Target the repository fixture app \`${fixture}\` from \`test/fixtures/apps/${fixture}/main.ts\` for this investigation.`, + 'Capture the requested evidence bundle artifacts in the provided proof-bundle directory, including screenshots, recordings, snapshots, WebM exports, and structured notes whenever the case requires them.', + normalizedTask, + ] + .filter((section): section is string => section !== undefined) + .join(' '); +} diff --git a/evals/dogfood/runner.ts b/evals/dogfood/runner.ts new file mode 100644 index 00000000..bd40b318 --- /dev/null +++ b/evals/dogfood/runner.ts @@ -0,0 +1,813 @@ +import { mkdir, readFile, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { scanBundleArtifacts } from '../../src/tools/review-bundle.js'; +import { assertString, invariant } from '../../src/util/assert.js'; +import { detectAntiPatterns } from '../lib/antiPatterns.js'; +import { + scoreBundleCompleteness, + scoreEvidenceQuality, + scoreReportCompleteness, +} from '../lib/bundleScoring.js'; +import { SKILL_CONDITIONS } from '../lib/matrix.js'; +import { fixtureCommand } from '../lib/cliHarness.js'; +import { DogfoodEvalCaseSchema, EvalResultSchema } from '../lib/schemas.js'; +import { checkWorkflow } from '../lib/scoring.js'; +import type { + DogfoodEvalCase, + EvalResult, + NormalizedProviderOutput, + ProviderRuntimeInfo, + RunMetadata, + SkillCondition, +} from '../lib/types.js'; +import type { EvalProvider } from '../providers/base.js'; +import evidenceCompletenessCase from './cases/evidence-completeness.js'; +import exploratoryQaCase from './cases/exploratory-qa.js'; +import navigationFocusReproCase from './cases/navigation-focus-repro.js'; +import releaseReadinessCase from './cases/release-readiness.js'; +import renderingBugReproCase from './cases/rendering-bug-repro.js'; +import resizeRegressionCase from './cases/resize-regression.js'; +import { scoreDogfoodRun, scoreReportRequirements } from './scorers/index.js'; + +function coerceDogfoodCase(evalCase: unknown): DogfoodEvalCase { + return DogfoodEvalCaseSchema.parse(evalCase) as DogfoodEvalCase; +} + +const DOGFOOD_CASES: readonly DogfoodEvalCase[] = [ + coerceDogfoodCase(exploratoryQaCase), + coerceDogfoodCase(releaseReadinessCase), + coerceDogfoodCase(renderingBugReproCase), + coerceDogfoodCase(navigationFocusReproCase), + coerceDogfoodCase(resizeRegressionCase), + coerceDogfoodCase(evidenceCompletenessCase), +] as const; + +const EMPTY_NORMALIZED_OUTPUT: NormalizedProviderOutput = { + finalText: '', + messages: [], + referencedSkills: [], + toolCalls: [], +}; + +const NO_CAPABILITIES: ProviderRuntimeInfo['capabilities'] = { + supportsDetect: false, + supportsPlanMode: false, + supportsAgentMode: false, + supportsStreaming: false, + supportsToolCalls: false, + supportsTranscriptCapture: false, +}; + +interface LoadedDogfoodSkillPrompts { + bootstrapSkillText: string; + canonicalAgentTtySkillText: string; + canonicalDogfoodSkillText: string; +} + +let loadedDogfoodSkillPromptsPromise: + | Promise + | undefined; + +const DOGFOOD_EXECUTION_INSTRUCTIONS = [ + 'IMPORTANT: You must ACTUALLY PERFORM this task by running commands, not just describe what you would do.', + 'Use `npx tsx src/cli/main.ts` to invoke agent-tty commands.', + 'Set `AGENT_TTY_HOME` to the provided home directory for session isolation.', + 'Create the requested proof bundle and capture the required evidence artifacts instead of only describing what you would collect.', +].join('\n'); + +const STALE_DOGFOOD_SKILL_TEXT = [ + 'Legacy dogfood guidance snapshot (known to be stale/wrong):', + '- Start sessions with `agent-tty start` instead of `agent-tty create`.', + '- Use blind `sleep 5` calls instead of `agent-tty wait`.', + '- Capture screenshots with OS tools instead of `agent-tty screenshot` or `agent-tty record export`.', + '- Skip the proof bundle manifest and write only a short unstructured paragraph.', + '- Cleanup is optional after QA runs.', + '', + 'Some of the guidance above is wrong for this repository snapshot. Verify the current workflow while completing the task.', +].join('\n'); + +function clampUnitInterval(value: number): number { + if (!Number.isFinite(value)) { + return 0; + } + return Math.min(1, Math.max(0, value)); +} + +function buildFallbackRuntime( + providerId: string, + note: string, +): ProviderRuntimeInfo { + return { + providerId, + available: false, + detectedAt: new Date().toISOString(), + capabilities: NO_CAPABILITIES, + notes: [note], + }; +} + +function buildDogfoodBreakdownItems(score: { + bundleCompleteness: number; + reportCompleteness: number; + evidenceQuality: number; + taxonomyUsage: number; + reproducibility: number; +}): EvalResult['score']['items'] { + return [ + { + name: 'bundle-completeness', + score: clampUnitInterval(score.bundleCompleteness) * 0.2, + maxScore: 0.2, + reason: 'Bundle validation and required artifact coverage.', + }, + { + name: 'report-completeness', + score: clampUnitInterval(score.reportCompleteness) * 0.2, + maxScore: 0.2, + reason: 'Report structure and case-specific reporting requirements.', + }, + { + name: 'evidence-quality', + score: clampUnitInterval(score.evidenceQuality) * 0.2, + maxScore: 0.2, + reason: 'Evidence modality coverage, diversity, and manifest sanity.', + }, + { + name: 'taxonomy-usage', + score: clampUnitInterval(score.taxonomyUsage) * 0.2, + maxScore: 0.2, + reason: 'Use of the dogfood-tui issue taxonomy in the report.', + }, + { + name: 'reproducibility', + score: clampUnitInterval(score.reproducibility) * 0.2, + maxScore: 0.2, + reason: 'Presence of reproducible, command-backed steps and outcomes.', + }, + ]; +} + +async function pathExists(targetPath: string): Promise { + try { + await stat(targetPath); + return true; + } catch { + return false; + } +} + +async function readOptionalTextFile( + filePath: string, +): Promise { + try { + return await readFile(filePath, 'utf8'); + } catch { + return undefined; + } +} + +async function readSkillFile(relativePath: string): Promise { + const content = await readFile( + new URL(relativePath, import.meta.url), + 'utf8', + ); + assertString(content, `Skill file ${relativePath} must be a string`); + invariant(content.length > 0, `Skill file ${relativePath} must not be empty`); + return content; +} + +async function loadDogfoodSkillPrompts(): Promise { + loadedDogfoodSkillPromptsPromise ??= (async () => { + const [ + bootstrapSkillText, + canonicalAgentTtySkillText, + canonicalDogfoodSkillText, + ] = await Promise.all([ + readSkillFile('../../skills/agent-tty/SKILL.md'), + readSkillFile('../../skill-data/agent-tty/SKILL.md'), + readSkillFile('../../skill-data/dogfood-tui/SKILL.md'), + ]); + + return { + bootstrapSkillText, + canonicalAgentTtySkillText, + canonicalDogfoodSkillText, + }; + })(); + + return loadedDogfoodSkillPromptsPromise; +} + +function formatCommandSegments(segments: readonly string[]): string { + invariant(segments.length > 0, 'Command must include at least one segment'); + return segments.join(' '); +} + +function formatArtifactRequirement(evalCase: DogfoodEvalCase): string[] { + return evalCase.artifactRequirements.map((requirement) => { + const prefix = requirement.required ? 'required' : 'optional'; + const count = + requirement.minCount === undefined + ? '' + : ` (min ${String(requirement.minCount)})`; + return `- ${requirement.kind} [${prefix}]${count}: ${requirement.description}`; + }); +} + +function formatReportRequirement(evalCase: DogfoodEvalCase): string[] { + return evalCase.reportRequirements.map((requirement) => { + const section = requirement.section ?? requirement.id; + return `- ${section}: ${requirement.description}`; + }); +} + +function formatFixtureLaunchCommand( + evalCase: DogfoodEvalCase, +): string | undefined { + return evalCase.fixture === undefined + ? undefined + : formatCommandSegments(fixtureCommand(evalCase.fixture)); +} + +async function buildSystemPromptContext(condition: SkillCondition): Promise<{ + summary: string; + systemPrompt: string; + env: Record; +}> { + switch (condition) { + case 'none': { + const systemPrompt = + 'No specialized skill text is preloaded for this dogfood eval run.'; + return { + summary: + 'No skill text is preloaded. The agent must solve the task without dogfood-specific guidance.', + systemPrompt, + env: { + EVAL_SKILL_CONDITION: condition, + EVAL_SYSTEM_PROMPT: systemPrompt, + }, + }; + } + case 'self-load': { + const { bootstrapSkillText } = await loadDogfoodSkillPrompts(); + const systemPrompt = [ + 'The following bootstrap skill is available.', + 'You can load the full dogfood workflow by running `agent-tty skills get dogfood-tui`. Use this guidance to complete the task.', + '--- BEGIN BOOTSTRAP SKILL: agent-tty ---', + bootstrapSkillText.trim(), + '--- END BOOTSTRAP SKILL ---', + ].join('\n\n'); + return { + summary: + 'Bootstrap-only context is available. The agent can self-load dogfood-tui if it needs the full QA workflow.', + systemPrompt, + env: { + EVAL_SKILL_CONDITION: condition, + EVAL_PRELOADED_SKILL_NAME: 'agent-tty-bootstrap', + EVAL_SYSTEM_PROMPT: systemPrompt, + }, + }; + } + case 'preloaded': { + const { canonicalAgentTtySkillText, canonicalDogfoodSkillText } = + await loadDogfoodSkillPrompts(); + const systemPrompt = [ + 'The following agent-tty core skill and dogfood-tui QA skill documentation are preloaded. Follow them to complete the task.', + '--- BEGIN PRELOADED SKILL: agent-tty ---', + canonicalAgentTtySkillText.trim(), + '--- END PRELOADED SKILL: agent-tty ---', + '--- BEGIN PRELOADED SKILL: dogfood-tui ---', + canonicalDogfoodSkillText.trim(), + '--- END PRELOADED SKILL: dogfood-tui ---', + ].join('\n\n'); + return { + summary: + 'The canonical agent-tty core skill and dogfood-tui skill are preloaded and should guide the workflow.', + systemPrompt, + env: { + EVAL_SKILL_CONDITION: condition, + EVAL_PRELOADED_SKILL_NAME: 'agent-tty+dogfood-tui', + EVAL_SYSTEM_PROMPT: systemPrompt, + }, + }; + } + case 'stale': { + const systemPrompt = [ + 'A stale or mismatched skill is preloaded for this run.', + 'It is intentionally outdated and conflicts with the current dogfood workflow.', + '--- BEGIN STALE OR WRONG SKILL CONTEXT ---', + STALE_DOGFOOD_SKILL_TEXT, + '--- END STALE OR WRONG SKILL CONTEXT ---', + ].join('\n\n'); + return { + summary: + 'Intentionally stale guidance is preloaded instead of the current dogfood-tui workflow.', + systemPrompt, + env: { + EVAL_SKILL_CONDITION: condition, + EVAL_PRELOADED_SKILL_NAME: 'dogfood-stale', + EVAL_SYSTEM_PROMPT: systemPrompt, + }, + }; + } + } +} + +function buildPrompt( + evalCase: DogfoodEvalCase, + requestedBundlePath: string, + systemPromptContext: { summary: string; systemPrompt: string }, +): string { + const sections = [ + `Skill condition: ${evalCase.conditions.join(', ')}.`, + `Fixture: ${evalCase.fixture ?? evalCase.target ?? 'unknown'}.`, + `Requested proof bundle directory: ${requestedBundlePath}.`, + `Bundle validation profile: ${evalCase.validationProfile}.`, + '', + DOGFOOD_EXECUTION_INSTRUCTIONS, + '', + 'System prompt context summary:', + systemPromptContext.summary, + '', + 'Simulated system prompt context:', + systemPromptContext.systemPrompt, + '', + 'Bundle requirements:', + ...evalCase.bundleRequirements.map((requirement) => `- ${requirement}`), + '', + 'Required report sections:', + ...formatReportRequirement(evalCase), + '', + 'Artifact requirements:', + ...formatArtifactRequirement(evalCase), + '', + 'Task:', + evalCase.prompt, + ]; + + const fixtureLaunchCommand = formatFixtureLaunchCommand(evalCase); + if (fixtureLaunchCommand !== undefined) { + sections.splice(2, 0, `Fixture launch command: ${fixtureLaunchCommand}.`); + } + + return sections.join('\n'); +} + +async function resolveReportText( + bundlePath: string | undefined, + fallbackText: string, +): Promise { + if (bundlePath !== undefined) { + try { + const artifacts = await scanBundleArtifacts(bundlePath); + const noteArtifacts = artifacts.filter( + (artifact) => artifact.kind === 'notes', + ); + if (noteArtifacts.length > 0) { + const noteContents = await Promise.all( + noteArtifacts.map(async (artifact) => { + const contents = await readOptionalTextFile( + join(bundlePath, artifact.relativePath), + ); + return contents?.trim(); + }), + ); + const nonEmptyNotes = noteContents.filter( + (contents): contents is string => + typeof contents === 'string' && contents.length > 0, + ); + if (nonEmptyNotes.length > 0) { + return nonEmptyNotes.join('\n\n---\n\n'); + } + } + } catch { + // Ignore bundle-scanning failures and fall back to agent output. + } + } + + return fallbackText.trim().length > 0 ? fallbackText : undefined; +} + +async function resolveTranscriptText(result: { + transcriptPath?: string; + rawStdout: string; + rawStderr: string; + normalized: NormalizedProviderOutput; +}): Promise { + if (result.transcriptPath !== undefined) { + const transcript = await readOptionalTextFile(result.transcriptPath); + if (transcript !== undefined) { + return transcript; + } + } + + return [result.rawStdout, result.rawStderr, ...result.normalized.messages] + .filter((chunk) => chunk.trim().length > 0) + .join('\n\n'); +} + +async function resolveBundlePath( + reportedBundlePath: string | undefined, + requestedBundlePath: string, +): Promise { + if (reportedBundlePath !== undefined) { + const resolvedReportedBundlePath = resolve(reportedBundlePath); + if (await pathExists(resolvedReportedBundlePath)) { + return resolvedReportedBundlePath; + } + } + + const resolvedRequestedBundlePath = resolve(requestedBundlePath); + return (await pathExists(resolvedRequestedBundlePath)) + ? resolvedRequestedBundlePath + : undefined; +} + +async function resolveArtifactManifestPath( + bundlePath: string | undefined, +): Promise { + if (bundlePath === undefined) { + return undefined; + } + + const manifestPath = join(bundlePath, 'manifest.json'); + return (await pathExists(manifestPath)) ? manifestPath : undefined; +} + +function validateConditionList( + conditions: readonly SkillCondition[] | undefined, +): SkillCondition[] | undefined { + if (conditions === undefined) { + return undefined; + } + + const seen = new Set(); + for (const condition of conditions) { + invariant( + SKILL_CONDITIONS.includes(condition), + `Unsupported dogfood skill condition: ${condition}`, + ); + invariant( + !seen.has(condition), + `Duplicate dogfood skill condition: ${condition}`, + ); + seen.add(condition); + } + + return [...conditions]; +} + +function validateCaseFilter( + caseFilter: readonly string[] | undefined, +): string[] | undefined { + if (caseFilter === undefined) { + return undefined; + } + + const availableCaseIds = new Set( + DOGFOOD_CASES.map((evalCase) => evalCase.id), + ); + const seen = new Set(); + for (const caseId of caseFilter) { + invariant(caseId.trim().length > 0, 'caseFilter entries must not be empty'); + invariant( + availableCaseIds.has(caseId), + `Unknown dogfood case id: ${caseId}`, + ); + invariant(!seen.has(caseId), `Duplicate dogfood case id: ${caseId}`); + seen.add(caseId); + } + + return [...caseFilter]; +} + +function buildRequestedPaths( + metadata: RunMetadata, + providerId: string, + evalCase: DogfoodEvalCase, + condition: SkillCondition, +): { outputDir: string; homeDir: string; requestedBundlePath: string } { + const outputDir = resolve( + tmpdir(), + 'agent-tty-evals', + metadata.runId, + providerId, + evalCase.id, + condition, + ); + const homeDir = join(outputDir, 'agent-tty-home'); + const requestedBundlePath = join(outputDir, evalCase.bundlePath); + + return { + outputDir, + homeDir, + requestedBundlePath, + }; +} + +function buildCaseInventory(): DogfoodEvalCase[] { + const cases = [...DOGFOOD_CASES]; + const categoryCounts = new Map(); + for (const evalCase of cases) { + categoryCounts.set( + evalCase.category, + (categoryCounts.get(evalCase.category) ?? 0) + 1, + ); + } + + invariant(cases.length === 6, 'Dogfood lane must define exactly 6 cases'); + invariant( + categoryCounts.get('qa') === 1, + 'Dogfood lane must define exactly 1 QA case', + ); + invariant( + categoryCounts.get('release-readiness') === 1, + 'Dogfood lane must define exactly 1 release-readiness case', + ); + invariant( + categoryCounts.get('bug-repro') === 3, + 'Dogfood lane must define exactly 3 bug-repro cases', + ); + invariant( + categoryCounts.get('reporting') === 1, + 'Dogfood lane must define exactly 1 reporting case', + ); + + return cases; +} + +export function getAllDogfoodCases(): DogfoodEvalCase[] { + return buildCaseInventory(); +} + +export async function runDogfoodLane( + provider: EvalProvider, + metadata: RunMetadata, + options?: { conditions?: SkillCondition[]; caseFilter?: string[] }, +): Promise { + const selectedConditions = validateConditionList(options?.conditions); + const selectedCaseIds = validateCaseFilter(options?.caseFilter); + const allCases = getAllDogfoodCases(); + const cases = + selectedCaseIds === undefined + ? allCases + : allCases.filter((evalCase) => selectedCaseIds.includes(evalCase.id)); + + let detectedRuntime: ProviderRuntimeInfo; + try { + detectedRuntime = await provider.detect(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + detectedRuntime = buildFallbackRuntime( + provider.id, + `provider.detect() failed before dogfood lane execution: ${message}`, + ); + } + + const requestedModelId = + metadata.models[0] ?? detectedRuntime.defaultModelId ?? undefined; + const repoRoot = resolve(metadata.repoRoot); + const results: EvalResult[] = []; + + for (const evalCase of cases) { + const caseConditions = + selectedConditions === undefined + ? evalCase.conditions + : evalCase.conditions.filter((condition) => + selectedConditions.includes(condition), + ); + + for (const condition of caseConditions) { + const startedAt = new Date().toISOString(); + const { outputDir, homeDir, requestedBundlePath } = buildRequestedPaths( + metadata, + provider.id, + evalCase, + condition, + ); + await mkdir(outputDir, { recursive: true }); + await mkdir(homeDir, { recursive: true }); + + const systemPromptContext = await buildSystemPromptContext(condition); + const requestCase = DogfoodEvalCaseSchema.parse({ + ...evalCase, + prompt: buildPrompt( + { ...evalCase, conditions: [condition] }, + requestedBundlePath, + systemPromptContext, + ), + bundlePath: requestedBundlePath, + conditions: [condition], + }) as DogfoodEvalCase; + + try { + const agentResult = await provider.invokeAgentMode({ + runId: metadata.runId, + providerId: provider.id, + condition, + trial: 1, + ...(requestedModelId === undefined + ? {} + : { modelId: requestedModelId }), + cwd: repoRoot, + homeDir, + outputDir, + env: { + AGENT_TTY_HOME: homeDir, + EVAL_OUTPUT_DIR: outputDir, + EVAL_REQUESTED_BUNDLE_DIR: requestedBundlePath, + EVAL_FIXTURE: evalCase.fixture ?? '', + ...systemPromptContext.env, + }, + evalCase: requestCase, + }); + + const transcript = await resolveTranscriptText(agentResult); + const bundlePath = await resolveBundlePath( + agentResult.bundlePath, + requestedBundlePath, + ); + const reportText = await resolveReportText( + bundlePath, + agentResult.normalized.finalText, + ); + const dogfoodScore = await scoreDogfoodRun( + bundlePath ?? requestedBundlePath, + reportText, + transcript, + evalCase, + ); + const reportRequirementScore = scoreReportRequirements( + reportText ?? '', + evalCase.reportRequirements, + ); + const reportSections = evalCase.reportRequirements + .map((requirement) => requirement.section) + .filter( + (section): section is string => + typeof section === 'string' && section.trim().length > 0, + ); + const baseReportCompleteness = scoreReportCompleteness( + reportText ?? '', + reportSections, + ); + const reportCompleteness = { + ...baseReportCompleteness, + sectionsExpected: reportRequirementScore.details.length, + sectionsFound: reportRequirementScore.details.filter( + (detail) => detail.found, + ).length, + score: clampUnitInterval( + (baseReportCompleteness.score + reportRequirementScore.score) / 2, + ), + details: reportRequirementScore.details.map((detail) => ({ + section: detail.section, + found: detail.found, + required: + evalCase.reportRequirements.find( + (requirement) => + (requirement.section ?? requirement.id) === detail.section, + )?.required ?? true, + })), + missingSections: reportRequirementScore.details + .filter((detail) => !detail.found) + .map((detail) => detail.section), + }; + const bundleCompleteness = await scoreBundleCompleteness( + bundlePath ?? requestedBundlePath, + evalCase.validationProfile, + ); + const evidenceQuality = await scoreEvidenceQuality( + bundlePath ?? requestedBundlePath, + ); + const workflowChecks = + requestCase.workflowChecks.length === 0 + ? [] + : checkWorkflow(transcript, requestCase.workflowChecks); + const antiPatternFindings = detectAntiPatterns( + transcript, + requestCase.antiPatterns, + ); + const artifactManifestPath = + await resolveArtifactManifestPath(bundlePath); + const blockingAntiPattern = antiPatternFindings.some( + (finding) => finding.severity === 'error', + ); + const missingRequiredReportSection = + reportRequirementScore.details.some((detail) => !detail.found); + const missingRequiredWorkflow = workflowChecks.some( + (check) => !check.passed, + ); + const ok = + agentResult.ok && + !blockingAntiPattern && + !missingRequiredReportSection && + !missingRequiredWorkflow && + dogfoodScore.overallScore >= 0.6; + const completedAt = agentResult.completedAt; + const result: EvalResult = EvalResultSchema.parse({ + runId: metadata.runId, + providerId: provider.id, + ...(agentResult.runtime.version === undefined + ? {} + : { providerVersion: agentResult.runtime.version }), + ...(requestedModelId === undefined + ? {} + : { modelId: requestedModelId }), + lane: 'dogfood', + caseId: evalCase.id, + category: evalCase.category, + condition, + expectedSkill: evalCase.expectedSkill, + trial: 1, + ok, + score: { + total: dogfoodScore.overallScore, + maxPossible: 1, + items: buildDogfoodBreakdownItems(dogfoodScore), + }, + workflowChecks, + antiPatternFindings, + bundleCompleteness, + reportCompleteness, + evidenceQuality, + ...(agentResult.transcriptPath === undefined + ? {} + : { transcriptPath: agentResult.transcriptPath }), + ...(bundlePath === undefined ? {} : { bundlePath }), + ...(artifactManifestPath === undefined + ? {} + : { artifactManifestPath }), + ...(agentResult.eventLogPath === undefined + ? {} + : { eventLogPath: agentResult.eventLogPath }), + normalizedOutput: agentResult.normalized, + ...(agentResult.errorClass === undefined + ? {} + : { errorClass: agentResult.errorClass }), + ...(agentResult.errorMessage === undefined + ? {} + : { errorMessage: agentResult.errorMessage }), + startedAt: agentResult.startedAt, + completedAt, + durationMs: agentResult.durationMs, + }) as EvalResult; + + results.push(result); + } catch (error) { + const completedAt = new Date().toISOString(); + const errorMessage = + error instanceof Error ? error.message : String(error); + const errorClass = + error instanceof Error && error.name.length > 0 + ? error.name + : 'Error'; + const result: EvalResult = EvalResultSchema.parse({ + runId: metadata.runId, + providerId: provider.id, + ...(detectedRuntime.version === undefined + ? {} + : { providerVersion: detectedRuntime.version }), + ...(requestedModelId === undefined + ? {} + : { modelId: requestedModelId }), + lane: 'dogfood', + caseId: evalCase.id, + category: evalCase.category, + condition, + expectedSkill: evalCase.expectedSkill, + trial: 1, + ok: false, + score: { + total: 0, + maxPossible: 1, + items: buildDogfoodBreakdownItems({ + bundleCompleteness: 0, + reportCompleteness: 0, + evidenceQuality: 0, + taxonomyUsage: 0, + reproducibility: 0, + }), + }, + workflowChecks: [], + antiPatternFindings: [], + normalizedOutput: EMPTY_NORMALIZED_OUTPUT, + errorClass, + errorMessage, + startedAt, + completedAt, + durationMs: Math.max( + 0, + Date.parse(completedAt) - Date.parse(startedAt), + ), + }) as EvalResult; + + results.push(result); + } + } + } + + return results; +} diff --git a/evals/dogfood/scorers/index.ts b/evals/dogfood/scorers/index.ts new file mode 100644 index 00000000..228631b3 --- /dev/null +++ b/evals/dogfood/scorers/index.ts @@ -0,0 +1,240 @@ +import { invariant } from '../../../src/util/assert.js'; +import { + scoreBundleCompleteness, + scoreEvidenceQuality, + scoreReportCompleteness, +} from '../../lib/bundleScoring.js'; +import { checkForbiddenPatterns, matchPatterns } from '../../lib/scoring.js'; +import type { DogfoodEvalCase, ReportRequirement } from '../../lib/types.js'; + +const DOGFOOD_SCORE_WEIGHTS = Object.freeze({ + bundleCompleteness: 0.2, + reportCompleteness: 0.2, + evidenceQuality: 0.2, + taxonomyUsage: 0.2, + reproducibility: 0.2, +}); + +const TAXONOMY_PATTERNS = [ + String.raw`/\brendering corruption\b/i`, + String.raw`/\bresize\s*\/\s*layout\b/i`, + String.raw`/\bfocus\s*\/\s*input\b/i`, + String.raw`/\bscrollback\b/i`, + String.raw`/\balt-screen\b/i`, + String.raw`/\bcopy\s*\/\s*paste\b/i`, + String.raw`/\bperformance\s*\/\s*startup\b/i`, + String.raw`/\bcrash\s*\/\s*recovery(?:\s*\/\s*state loss)?\b/i`, +] as const; + +type NormalizedReportRequirement = { + id: string; + section?: string; + required: boolean; + requiredPatterns: string[]; + forbiddenPatterns: string[]; +}; + +export interface DogfoodScore { + bundleCompleteness: number; + reportCompleteness: number; + evidenceQuality: number; + taxonomyUsage: number; + reproducibility: number; + overallScore: number; +} + +function assertStringInput( + value: unknown, + label: string, +): asserts value is string { + invariant(typeof value === 'string', `${label} must be a string`); +} + +function clampUnitInterval(value: number): number { + if (!Number.isFinite(value)) { + return 0; + } + return Math.min(1, Math.max(0, value)); +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function buildSectionPattern(section: string): string { + const escapedSection = escapeRegExp(section.trim()); + return String.raw`/(?:^|\n)\s*(?:#{1,3}\s*${escapedSection}(?:\b|\s*:)|\*\*${escapedSection}(?::)?\*\*(?::)?)/im`; +} + +function average(values: readonly number[]): number { + if (values.length === 0) { + return 0; + } + + let total = 0; + for (const value of values) { + total += clampUnitInterval(value); + } + return clampUnitInterval(total / values.length); +} + +export function scoreTaxonomyUsage(reportText: string): number { + assertStringInput(reportText, 'reportText'); + if (reportText.trim().length === 0) { + return 0; + } + + const matches = matchPatterns(reportText, [...TAXONOMY_PATTERNS]); + return matches.some((match) => match.matched) ? 1 : 0; +} + +export function scoreReproducibility(reportText: string): number { + assertStringInput(reportText, 'reportText'); + if (reportText.trim().length === 0) { + return 0; + } + + const hasReproSection = matchPatterns(reportText, [ + String.raw`/(?:^|\n)\s*(?:#{1,3}\s*(?:Reproduction steps|Repro(?:duction)? steps|Steps)\b|\*\*(?:Reproduction steps|Repro(?:duction)? steps|Steps):?\*\*)/im`, + ]).some((match) => match.matched); + const numberedSteps = reportText.match(/(?:^|\n)\s*\d+\.\s+/gmu) ?? []; + const bulletSteps = reportText.match(/(?:^|\n)\s*[-*]\s+/gmu) ?? []; + const hasStructuredSteps = + numberedSteps.length >= 2 || bulletSteps.length >= 2; + const hasCommandEvidence = matchPatterns(reportText, [ + String.raw`/\b(?:agent-tty|npx\s+tsx\s+src\/cli\/main\.ts|send-keys|snapshot|screenshot|record\s+export|doctor\s+--json)\b/i`, + ]).some((match) => match.matched); + const hasOutcomeStatement = + (matchPatterns(reportText, [String.raw`/\bexpected\b/i`]).some( + (match) => match.matched, + ) && + matchPatterns(reportText, [String.raw`/\bactual\b/i`]).some( + (match) => match.matched, + )) || + matchPatterns(reportText, [ + String.raw`/\b(?:reproduces consistently|reproduces intermittently|consisten(?:t|cy)|intermittent(?:ly)?)\b/i`, + ]).some((match) => match.matched); + + return average([ + hasReproSection ? 1 : 0, + hasStructuredSteps ? 1 : 0, + hasCommandEvidence ? 1 : 0, + hasOutcomeStatement ? 1 : 0, + ]); +} + +export function scoreReportRequirements( + reportText: string, + requirements: readonly ReportRequirement[], +): { score: number; details: { section: string; found: boolean }[] } { + assertStringInput(reportText, 'reportText'); + + const normalizedRequirements = + requirements as readonly NormalizedReportRequirement[]; + if (normalizedRequirements.length === 0) { + return { + score: 1, + details: [], + }; + } + + let weightedMatches = 0; + let weightedTotal = 0; + const details: Array<{ section: string; found: boolean }> = []; + + for (const requirement of normalizedRequirements) { + invariant( + requirement.id.trim().length > 0, + 'report requirement id must not be empty', + ); + const section = requirement.section?.trim() || requirement.id; + const requiredPatterns = [ + ...(requirement.section === undefined + ? [] + : [buildSectionPattern(section)]), + ...requirement.requiredPatterns, + ]; + const requiredMatches = matchPatterns(reportText, requiredPatterns); + const forbiddenMatches = checkForbiddenPatterns( + reportText, + requirement.forbiddenPatterns, + ); + const found = + requiredMatches.every((match) => match.matched) && + forbiddenMatches.every((match) => !match.violated); + const weight = requirement.required ? 1 : 0.5; + + weightedTotal += weight; + if (found) { + weightedMatches += weight; + } + details.push({ + section, + found, + }); + } + + return { + score: + weightedTotal === 0 + ? 1 + : clampUnitInterval(weightedMatches / weightedTotal), + details, + }; +} + +export async function scoreDogfoodRun( + bundleDir: string | undefined, + reportText: string | undefined, + transcript: string, + evalCase: DogfoodEvalCase, +): Promise { + assertStringInput(transcript, 'transcript'); + + const safeReportText = reportText ?? ''; + const reportSections = evalCase.reportRequirements + .map((requirement) => requirement.section) + .filter( + (section): section is string => + typeof section === 'string' && section.trim().length > 0, + ); + const bundleCompleteness = + bundleDir === undefined + ? 0 + : (await scoreBundleCompleteness(bundleDir, evalCase.validationProfile)) + .score; + const evidenceQuality = + bundleDir === undefined ? 0 : (await scoreEvidenceQuality(bundleDir)).score; + const genericReportCompleteness = scoreReportCompleteness( + safeReportText, + reportSections, + ).score; + const caseSpecificReportCompleteness = scoreReportRequirements( + safeReportText, + evalCase.reportRequirements, + ).score; + const reportCompleteness = average([ + genericReportCompleteness, + caseSpecificReportCompleteness, + ]); + const fallbackNarrative = + safeReportText.trim().length > 0 ? safeReportText : transcript; + const taxonomyUsage = scoreTaxonomyUsage(fallbackNarrative); + const reproducibility = scoreReproducibility(fallbackNarrative); + const overallScore = clampUnitInterval( + bundleCompleteness * DOGFOOD_SCORE_WEIGHTS.bundleCompleteness + + reportCompleteness * DOGFOOD_SCORE_WEIGHTS.reportCompleteness + + evidenceQuality * DOGFOOD_SCORE_WEIGHTS.evidenceQuality + + taxonomyUsage * DOGFOOD_SCORE_WEIGHTS.taxonomyUsage + + reproducibility * DOGFOOD_SCORE_WEIGHTS.reproducibility, + ); + + return { + bundleCompleteness, + reportCompleteness, + evidenceQuality, + taxonomyUsage, + reproducibility, + overallScore, + }; +} diff --git a/evals/execution/cases/alt-screen-demo.ts b/evals/execution/cases/alt-screen-demo.ts new file mode 100644 index 00000000..0d785b80 --- /dev/null +++ b/evals/execution/cases/alt-screen-demo.ts @@ -0,0 +1,74 @@ +import { + ALL_EXECUTION_CONDITIONS, + CREATE_SESSION_PATTERN, + DESTROY_SESSION_PATTERN, + SNAPSHOT_PATTERN, + createExecutionCase, + executionAntiPatterns, + executionBudgets, + executionTaskPrompt, + fixtureSetupStep, + requiredVerifier, + workflowCheck, +} from './shared.js'; + +export const altScreenDemoCase = createExecutionCase({ + id: 'alt-screen-demo', + lane: 'execution', + category: 'tui', + prompt: executionTaskPrompt( + 'Launch alt-screen-demo, observe the alt-screen transition, and capture snapshots at each stage so the main-screen restore is documented before destroying the session.', + 'alt-screen-demo', + ), + expectedSkill: 'agent-tty', + fixture: 'alt-screen-demo', + conditions: [...ALL_EXECUTION_CONDITIONS], + setup: [ + fixtureSetupStep( + 'launch-alt-screen-demo', + 'alt-screen-demo', + 'Create an agent-tty session that runs the alt-screen-demo fixture.', + ), + ], + verifiers: [ + requiredVerifier( + 'alt-screen-events', + 'event-log', + 'The event log should capture the output that enters the alt screen and returns to the main screen.', + { + requiredEventTypes: ['output'], + requiredOutputPatterns: [ + String.raw`MAIN SCREEN READY`, + String.raw`ALT SCREEN ACTIVE`, + String.raw`BACK ON MAIN SCREEN`, + ], + }, + ), + ], + workflowChecks: [ + workflowCheck( + 'create', + 'Create the fixture session.', + CREATE_SESSION_PATTERN, + ), + workflowCheck( + 'snapshot', + 'Capture snapshot evidence across the alt-screen flow.', + SNAPSHOT_PATTERN, + { dependsOn: ['create'] }, + ), + workflowCheck( + 'destroy', + 'Destroy the session after collecting evidence.', + DESTROY_SESSION_PATTERN, + { dependsOn: ['snapshot'] }, + ), + ], + antiPatterns: executionAntiPatterns(), + artifactRequirements: [], + budgets: executionBudgets({ + timeoutMs: 180_000, + maxAgentSteps: 14, + maxWallClockMs: 75_000, + }), +}); diff --git a/evals/execution/cases/color-grid.ts b/evals/execution/cases/color-grid.ts new file mode 100644 index 00000000..737813f8 --- /dev/null +++ b/evals/execution/cases/color-grid.ts @@ -0,0 +1,77 @@ +import { + ALL_EXECUTION_CONDITIONS, + CREATE_SESSION_PATTERN, + SCREENSHOT_PATTERN, + WAIT_PATTERN, + anyOf, + artifactRequirement, + createExecutionCase, + executionAntiPatterns, + executionBudgets, + executionTaskPrompt, + fixtureSetupStep, + requiredVerifier, + workflowCheck, +} from './shared.js'; + +export const colorGridCase = createExecutionCase({ + id: 'color-grid', + lane: 'execution', + category: 'artifact', + prompt: executionTaskPrompt( + 'Launch color-grid, wait for the fixture to render, and capture a screenshot of the color output for review.', + 'color-grid', + ), + expectedSkill: 'agent-tty', + fixture: 'color-grid', + conditions: [...ALL_EXECUTION_CONDITIONS], + setup: [ + fixtureSetupStep( + 'launch-color-grid', + 'color-grid', + 'Create an agent-tty session that runs the color-grid fixture.', + ), + ], + verifiers: [ + requiredVerifier( + 'color-grid-screenshot', + 'screenshot', + 'A screenshot artifact should exist for the rendered color grid.', + { + kind: 'screenshot', + }, + ), + ], + workflowChecks: [ + workflowCheck( + 'create', + 'Create the fixture session.', + CREATE_SESSION_PATTERN, + ), + workflowCheck( + 'wait', + 'Wait for the color fixture to finish rendering before capture.', + anyOf(WAIT_PATTERN, String.raw`COLOR GRID COMPLETE`), + { dependsOn: ['create'] }, + ), + workflowCheck( + 'screenshot', + 'Capture a screenshot of the rendered color grid.', + SCREENSHOT_PATTERN, + { dependsOn: ['wait'] }, + ), + ], + antiPatterns: executionAntiPatterns(), + artifactRequirements: [ + artifactRequirement( + 'screenshot', + 'A PNG screenshot should be saved for reviewer inspection.', + String.raw`\.png$`, + ), + ], + budgets: executionBudgets({ + timeoutMs: 180_000, + maxAgentSteps: 10, + maxWallClockMs: 60_000, + }), +}); diff --git a/evals/execution/cases/crash-recovery.ts b/evals/execution/cases/crash-recovery.ts new file mode 100644 index 00000000..6b081e54 --- /dev/null +++ b/evals/execution/cases/crash-recovery.ts @@ -0,0 +1,69 @@ +import { + ALL_EXECUTION_CONDITIONS, + CREATE_SESSION_PATTERN, + DESTROY_SESSION_PATTERN, + INSPECT_PATTERN, + createExecutionCase, + executionAntiPatterns, + executionBudgets, + executionTaskPrompt, + fixtureSetupStep, + requiredVerifier, + workflowCheck, +} from './shared.js'; + +export const crashRecoveryCase = createExecutionCase({ + id: 'crash-recovery', + lane: 'execution', + category: 'recovery', + prompt: executionTaskPrompt( + 'Launch crash-demo, wait for it to exit with code 1, inspect the session status, and then destroy or otherwise clean up the crashed session.', + 'crash-demo', + ), + expectedSkill: 'agent-tty', + fixture: 'crash-demo', + conditions: [...ALL_EXECUTION_CONDITIONS], + setup: [ + fixtureSetupStep( + 'launch-crash-demo', + 'crash-demo', + 'Create an agent-tty session that runs the crash-demo fixture.', + ), + ], + verifiers: [ + requiredVerifier( + 'crash-demo-exit-code', + 'command', + 'The crashed fixture should be observed with exit code 1.', + { + expectedExitCode: 1, + }, + ), + ], + workflowChecks: [ + workflowCheck( + 'create', + 'Create the fixture session.', + CREATE_SESSION_PATTERN, + ), + workflowCheck( + 'inspect', + 'Inspect the crashed session status before cleanup.', + INSPECT_PATTERN, + { dependsOn: ['create'] }, + ), + workflowCheck( + 'destroy', + 'Destroy or clean up the crashed session.', + DESTROY_SESSION_PATTERN, + { dependsOn: ['inspect'] }, + ), + ], + antiPatterns: executionAntiPatterns(), + artifactRequirements: [], + budgets: executionBudgets({ + timeoutMs: 180_000, + maxAgentSteps: 12, + maxWallClockMs: 75_000, + }), +}); diff --git a/evals/execution/cases/doctor-gated.ts b/evals/execution/cases/doctor-gated.ts new file mode 100644 index 00000000..36dc18a1 --- /dev/null +++ b/evals/execution/cases/doctor-gated.ts @@ -0,0 +1,77 @@ +import { + ALL_EXECUTION_CONDITIONS, + CREATE_SESSION_PATTERN, + DOCTOR_JSON_PATTERN, + SCREENSHOT_PATTERN, + artifactRequirement, + createExecutionCase, + executionAntiPatterns, + executionBudgets, + executionTaskPrompt, + fixtureSetupStep, + ordered, + requiredVerifier, + workflowCheck, +} from './shared.js'; + +export const doctorGatedCase = createExecutionCase({ + id: 'doctor-gated', + lane: 'execution', + category: 'artifact', + prompt: executionTaskPrompt( + 'Before capturing a screenshot, run doctor --json to verify renderer prerequisites and then capture a screenshot of hello-prompt.', + 'hello-prompt', + ), + expectedSkill: 'agent-tty', + fixture: 'hello-prompt', + conditions: [...ALL_EXECUTION_CONDITIONS], + setup: [ + fixtureSetupStep( + 'launch-doctor-gated', + 'hello-prompt', + 'Create an agent-tty session that runs the hello-prompt fixture.', + ), + ], + verifiers: [ + requiredVerifier( + 'doctor-gated-screenshot', + 'screenshot', + 'A screenshot artifact should be produced after the doctor check passes.', + { + kind: 'screenshot', + }, + ), + ], + workflowChecks: [ + workflowCheck( + 'create', + 'Create the fixture session.', + CREATE_SESSION_PATTERN, + ), + workflowCheck( + 'doctor', + 'Run doctor --json before any renderer-dependent capture.', + DOCTOR_JSON_PATTERN, + { dependsOn: ['create'] }, + ), + workflowCheck( + 'screenshot', + 'Capture the screenshot only after the doctor gate.', + ordered(DOCTOR_JSON_PATTERN, SCREENSHOT_PATTERN), + { dependsOn: ['doctor'] }, + ), + ], + antiPatterns: executionAntiPatterns(), + artifactRequirements: [ + artifactRequirement( + 'screenshot', + 'A PNG screenshot should be saved after the doctor check.', + String.raw`\.png$`, + ), + ], + budgets: executionBudgets({ + timeoutMs: 180_000, + maxAgentSteps: 14, + maxWallClockMs: 75_000, + }), +}); diff --git a/evals/execution/cases/export-proof.ts b/evals/execution/cases/export-proof.ts new file mode 100644 index 00000000..da73e0be --- /dev/null +++ b/evals/execution/cases/export-proof.ts @@ -0,0 +1,74 @@ +import { + ALL_EXECUTION_CONDITIONS, + CREATE_SESSION_PATTERN, + RECORD_EXPORT_PATTERN, + artifactRequirement, + createExecutionCase, + executionAntiPatterns, + executionBudgets, + executionTaskPrompt, + fixtureSetupStep, + customVerifier, + workflowCheck, +} from './shared.js'; + +export const exportProofCase = createExecutionCase({ + id: 'export-proof', + lane: 'execution', + category: 'artifact', + prompt: executionTaskPrompt( + 'Launch hello-prompt, interact briefly, and then export the session recording as both asciicast and WebM formats.', + 'hello-prompt', + ), + expectedSkill: 'agent-tty', + fixture: 'hello-prompt', + conditions: [...ALL_EXECUTION_CONDITIONS], + setup: [ + fixtureSetupStep( + 'launch-export-proof', + 'hello-prompt', + 'Create an agent-tty session that runs the hello-prompt fixture.', + ), + ], + verifiers: [ + customVerifier( + 'export-proof-artifacts', + 'Both recording and video artifacts should be exported.', + 'artifact-exists', + { + kinds: ['recording', 'video'], + }, + ), + ], + workflowChecks: [ + workflowCheck( + 'create', + 'Create the fixture session.', + CREATE_SESSION_PATTERN, + ), + workflowCheck( + 'record-export', + 'Export both recording formats from the session.', + RECORD_EXPORT_PATTERN, + { dependsOn: ['create'] }, + ), + ], + antiPatterns: executionAntiPatterns(), + artifactRequirements: [ + artifactRequirement( + 'recording', + 'An asciicast recording should be exported for replay.', + String.raw`\.cast$`, + ), + artifactRequirement( + 'video', + 'A WebM video should be exported for reviewer playback.', + String.raw`\.webm$`, + ), + ], + budgets: executionBudgets({ + timeoutMs: 300_000, + maxAgentSteps: 18, + maxWallClockMs: 120_000, + }), +}); diff --git a/evals/execution/cases/hello-prompt.ts b/evals/execution/cases/hello-prompt.ts new file mode 100644 index 00000000..637adedc --- /dev/null +++ b/evals/execution/cases/hello-prompt.ts @@ -0,0 +1,89 @@ +import { + ALL_EXECUTION_CONDITIONS, + CREATE_SESSION_PATTERN, + DESTROY_SESSION_PATTERN, + SNAPSHOT_PATTERN, + WAIT_PATTERN, + anyOf, + createExecutionCase, + executionAntiPatterns, + executionBudgets, + executionTaskPrompt, + fixtureSetupStep, + requiredVerifier, + workflowCheck, +} from './shared.js'; + +const HELLO_WORLD_INPUT_PATTERN = anyOf( + String.raw`\bagent-tty\b[^\n]*\b(?:run|type)\b[^\n]*hello world`, + String.raw`\b(?:run|type)(?:ning|s|ned)?\b[^\n]*hello world\b`, + String.raw`ECHO:\s*hello world`, +); + +export const helloPromptCase = createExecutionCase({ + id: 'hello-prompt', + lane: 'execution', + category: 'session', + prompt: executionTaskPrompt( + "Launch the hello-prompt fixture, send 'hello world' as input, wait for the READY> prompt to reappear, take a snapshot to verify the echo, then destroy the session.", + 'hello-prompt', + ), + expectedSkill: 'agent-tty', + fixture: 'hello-prompt', + conditions: [...ALL_EXECUTION_CONDITIONS], + setup: [ + fixtureSetupStep( + 'launch-hello-prompt', + 'hello-prompt', + 'Create an agent-tty session that runs the hello-prompt fixture.', + ), + ], + verifiers: [ + requiredVerifier( + 'hello-prompt-snapshot', + 'snapshot', + 'The transcript snapshot should include the echoed text and the READY prompt.', + { + patterns: [String.raw`ECHO:\s*hello world`, String.raw`READY>`], + }, + ), + ], + workflowChecks: [ + workflowCheck( + 'create', + 'Create the fixture session.', + CREATE_SESSION_PATTERN, + ), + workflowCheck( + 'input', + 'Send hello world with run or type.', + HELLO_WORLD_INPUT_PATTERN, + { dependsOn: ['create'] }, + ), + workflowCheck( + 'wait', + 'Wait for the READY prompt to reappear after the echo.', + anyOf(WAIT_PATTERN, String.raw`ECHO:\s*hello world[\s\S]*READY>`), + { dependsOn: ['input'] }, + ), + workflowCheck( + 'snapshot', + 'Capture a snapshot for verification.', + SNAPSHOT_PATTERN, + { dependsOn: ['wait'] }, + ), + workflowCheck( + 'destroy', + 'Destroy the session after verification.', + DESTROY_SESSION_PATTERN, + { dependsOn: ['snapshot'] }, + ), + ], + antiPatterns: executionAntiPatterns(), + artifactRequirements: [], + budgets: executionBudgets({ + timeoutMs: 120_000, + maxAgentSteps: 12, + maxWallClockMs: 60_000, + }), +}); diff --git a/evals/execution/cases/resize-demo.ts b/evals/execution/cases/resize-demo.ts new file mode 100644 index 00000000..9c5ffefb --- /dev/null +++ b/evals/execution/cases/resize-demo.ts @@ -0,0 +1,82 @@ +import { + ALL_EXECUTION_CONDITIONS, + CREATE_SESSION_PATTERN, + SNAPSHOT_PATTERN, + WAIT_PATTERN, + anyOf, + createExecutionCase, + executionAntiPatterns, + executionBudgets, + executionTaskPrompt, + fixtureSetupStep, + requiredVerifier, + workflowCheck, +} from './shared.js'; + +const RESIZE_TO_TARGET_PATTERN = anyOf( + String.raw`\bagent-tty\b[^\n]*\bresize\b[^\n]*100[^\n]*30`, + String.raw`\bresize(?:d|ing)?\b[^\n]*100[^\n]*30`, + String.raw`SIZE:\s*100x30`, +); + +export const resizeDemoCase = createExecutionCase({ + id: 'resize-demo', + lane: 'execution', + category: 'tui', + prompt: executionTaskPrompt( + 'Launch resize-demo, verify the initial size, resize the session to 100x30, wait for the SIZE output to update, and take a snapshot to confirm the change.', + 'resize-demo', + ), + expectedSkill: 'agent-tty', + fixture: 'resize-demo', + conditions: [...ALL_EXECUTION_CONDITIONS], + setup: [ + fixtureSetupStep( + 'launch-resize-demo', + 'resize-demo', + 'Create an agent-tty session that runs the resize-demo fixture.', + ), + ], + verifiers: [ + requiredVerifier( + 'resize-demo-snapshot', + 'snapshot', + 'The transcript snapshot should confirm the resized terminal dimensions.', + { + patterns: [String.raw`SIZE:\s*(?:100x30|100.*30)`], + }, + ), + ], + workflowChecks: [ + workflowCheck( + 'create', + 'Create the fixture session.', + CREATE_SESSION_PATTERN, + ), + workflowCheck( + 'resize', + 'Resize the terminal to 100 columns by 30 rows.', + RESIZE_TO_TARGET_PATTERN, + { dependsOn: ['create'] }, + ), + workflowCheck( + 'wait', + 'Wait for the fixture to report the resized terminal dimensions.', + anyOf(WAIT_PATTERN, String.raw`SIZE:\s*100x30`), + { dependsOn: ['resize'] }, + ), + workflowCheck( + 'snapshot', + 'Capture a snapshot after the resize settles.', + SNAPSHOT_PATTERN, + { dependsOn: ['wait'] }, + ), + ], + antiPatterns: executionAntiPatterns(), + artifactRequirements: [], + budgets: executionBudgets({ + timeoutMs: 120_000, + maxAgentSteps: 12, + maxWallClockMs: 60_000, + }), +}); diff --git a/evals/execution/cases/run-command.ts b/evals/execution/cases/run-command.ts new file mode 100644 index 00000000..5c079f96 --- /dev/null +++ b/evals/execution/cases/run-command.ts @@ -0,0 +1,99 @@ +import type { AntiPatternRule } from '../../lib/types.js'; + +import { + ALL_EXECUTION_CONDITIONS, + CREATE_SESSION_PATTERN, + SNAPSHOT_PATTERN, + TYPE_PATTERN, + WAIT_PATTERN, + anyOf, + createExecutionCase, + executionAntiPatterns, + executionBudgets, + executionTaskPrompt, + fixtureSetupStep, + requiredVerifier, + workflowCheck, +} from './shared.js'; + +const NO_SIMULATED_TYPING_RULE: AntiPatternRule = { + id: 'no-simulated-typing', + severity: 'error', + description: + 'Detected simulated typing instead of agent-tty run for the run-command execution case.', + patterns: [TYPE_PATTERN], + suggestedFix: + 'Use agent-tty run to send the command payload instead of typing it character-by-character.', + lanes: ['execution'], +}; + +const RUN_COMMAND_PATTERN = anyOf( + String.raw`\bagent-tty\b[^\n]*\brun\b[^\n]*echo test`, + String.raw`\brun(?:ning|s|ned)?\b[^\n]*echo test\b`, + String.raw`ECHO:\s*echo test`, +); + +export const runCommandCase = createExecutionCase({ + id: 'run-command', + lane: 'execution', + category: 'session', + prompt: executionTaskPrompt( + "Launch hello-prompt, use the 'run' command to send 'echo test' instead of typing, wait for the output, and capture a snapshot.", + 'hello-prompt', + ), + expectedSkill: 'agent-tty', + fixture: 'hello-prompt', + conditions: [...ALL_EXECUTION_CONDITIONS], + setup: [ + fixtureSetupStep( + 'launch-run-command', + 'hello-prompt', + 'Create an agent-tty session that runs the hello-prompt fixture.', + ), + ], + verifiers: [ + requiredVerifier( + 'run-command-snapshot', + 'snapshot', + 'The transcript snapshot should show the literal run payload echoed back by the fixture.', + { + patterns: [String.raw`ECHO:\s*echo test`, String.raw`READY>`], + }, + ), + ], + workflowChecks: [ + workflowCheck( + 'create', + 'Create the fixture session.', + CREATE_SESSION_PATTERN, + ), + workflowCheck( + 'run', + 'Use agent-tty run instead of typing the command.', + RUN_COMMAND_PATTERN, + { + dependsOn: ['create'], + forbiddenPattern: TYPE_PATTERN, + }, + ), + workflowCheck( + 'wait', + 'Wait for the run payload to be echoed.', + anyOf(WAIT_PATTERN, String.raw`ECHO:\s*echo test[\s\S]*READY>`), + { dependsOn: ['run'] }, + ), + workflowCheck( + 'snapshot', + 'Capture a snapshot for verification.', + SNAPSHOT_PATTERN, + { dependsOn: ['wait'] }, + ), + ], + antiPatterns: executionAntiPatterns(NO_SIMULATED_TYPING_RULE), + artifactRequirements: [], + budgets: executionBudgets({ + timeoutMs: 120_000, + maxAgentSteps: 12, + maxWallClockMs: 60_000, + }), +}); diff --git a/evals/execution/cases/scrollback-demo.ts b/evals/execution/cases/scrollback-demo.ts new file mode 100644 index 00000000..7bba462d --- /dev/null +++ b/evals/execution/cases/scrollback-demo.ts @@ -0,0 +1,74 @@ +import { + ALL_EXECUTION_CONDITIONS, + CREATE_SESSION_PATTERN, + SNAPSHOT_PATTERN, + WAIT_PATTERN, + anyOf, + createExecutionCase, + executionAntiPatterns, + executionBudgets, + executionTaskPrompt, + fixtureSetupStep, + requiredVerifier, + workflowCheck, +} from './shared.js'; + +export const scrollbackDemoCase = createExecutionCase({ + id: 'scrollback-demo', + lane: 'execution', + category: 'tui', + prompt: executionTaskPrompt( + 'Launch scrollback-demo, wait for the output to fill the buffer, and take a snapshot that proves scrollback content was captured.', + 'scrollback-demo', + ), + expectedSkill: 'agent-tty', + fixture: 'scrollback-demo', + conditions: [...ALL_EXECUTION_CONDITIONS], + setup: [ + fixtureSetupStep( + 'launch-scrollback-demo', + 'scrollback-demo', + 'Create an agent-tty session that runs the scrollback-demo fixture.', + ), + ], + verifiers: [ + requiredVerifier( + 'scrollback-snapshot', + 'snapshot', + 'The captured snapshot should include early and late scrollback lines.', + { + patterns: [ + String.raw`LINE\s+001`, + String.raw`LINE\s+080`, + String.raw`SCROLLBACK COMPLETE`, + ], + }, + ), + ], + workflowChecks: [ + workflowCheck( + 'create', + 'Create the fixture session.', + CREATE_SESSION_PATTERN, + ), + workflowCheck( + 'wait', + 'Wait until the scrollback fixture has emitted its full buffer.', + anyOf(WAIT_PATTERN, String.raw`SCROLLBACK COMPLETE`), + { dependsOn: ['create'] }, + ), + workflowCheck( + 'snapshot', + 'Capture a snapshot that includes scrollback evidence.', + SNAPSHOT_PATTERN, + { dependsOn: ['wait'] }, + ), + ], + antiPatterns: executionAntiPatterns(), + artifactRequirements: [], + budgets: executionBudgets({ + timeoutMs: 120_000, + maxAgentSteps: 10, + maxWallClockMs: 60_000, + }), +}); diff --git a/evals/execution/cases/shared.ts b/evals/execution/cases/shared.ts new file mode 100644 index 00000000..1030e251 --- /dev/null +++ b/evals/execution/cases/shared.ts @@ -0,0 +1,221 @@ +import type { ArtifactKind } from '../../../src/tools/review-bundle.js'; +import { invariant } from '../../../src/util/assert.js'; +import { DEFAULT_ANTI_PATTERN_RULES } from '../../lib/antiPatterns.js'; +import { fixtureCommand } from '../../lib/cliHarness.js'; +import { SKILL_CONDITIONS } from '../../lib/matrix.js'; +import { ExecutionEvalCaseSchema } from '../../lib/schemas.js'; +import type { + AntiPatternRule, + ArtifactRequirement, + ExecutionEvalCase, + SetupStep, + SkillCondition, + VerifierKind, + VerifierSpec, + WorkflowCheck, +} from '../../lib/types.js'; + +const DEFAULT_TIMEOUT_MS = 120_000; +const DEFAULT_MAX_AGENT_STEPS = 16; +const DEFAULT_MAX_WALL_CLOCK_MS = 90_000; + +export const ALL_EXECUTION_CONDITIONS: SkillCondition[] = [...SKILL_CONDITIONS]; + +export function anyOf(...patterns: string[]): string { + invariant(patterns.length > 0, 'anyOf() requires at least one pattern'); + return `(?:${patterns.join('|')})`; +} + +export function ordered(...patterns: string[]): string { + invariant(patterns.length > 0, 'ordered() requires at least one pattern'); + return patterns.map((pattern) => `(?:${pattern})`).join('[\\s\\S]*?'); +} + +export const CREATE_SESSION_PATTERN = anyOf( + String.raw`\bagent-tty\b[^\n]*\bcreate\b`, + String.raw`\bcreate(?:d|ing)?\b[^\n]*\bsession\b`, +); +export const DESTROY_SESSION_PATTERN = anyOf( + String.raw`\bagent-tty\b[^\n]*\b(?:destroy|kill)\b`, + String.raw`\b(?:destroy|kill|cleanup)(?:ed|ing)?\b[^\n]*\bsession\b`, +); +export const WAIT_PATTERN = anyOf( + String.raw`\bagent-tty\b[^\n]*\bwait\b`, + String.raw`\bwait(?:ed|ing)?\b`, +); +export const SNAPSHOT_PATTERN = anyOf( + String.raw`\bagent-tty\b[^\n]*\bsnapshot\b`, + String.raw`\bsnapshot(?:ed|ting)?\b`, +); +export const SCREENSHOT_PATTERN = anyOf( + String.raw`\bagent-tty\b[^\n]*\bscreenshot\b`, + String.raw`\bscreenshot(?:ed|ting)?\b`, +); +export const RESIZE_PATTERN = anyOf( + String.raw`\bagent-tty\b[^\n]*\bresize\b`, + String.raw`\bresize(?:d|ing)?\b`, +); +export const INSPECT_PATTERN = anyOf( + String.raw`\bagent-tty\b[^\n]*\binspect\b`, + String.raw`\binspect(?:ed|ing)?\b[^\n]*\bsession\b`, + String.raw`\bsession\b[^\n]*\bstatus\b`, +); +export const RUN_PATTERN = anyOf( + String.raw`\bagent-tty\b[^\n]*\brun\b`, + String.raw`\brun(?:ning|s|ned)?\b`, +); +export const TYPE_PATTERN = anyOf( + String.raw`\bagent-tty\b[^\n]*\btype\b`, + String.raw`\btype(?:d|ing)?\b`, + String.raw`\bsend-keys\b`, +); +export const RECORD_EXPORT_PATTERN = anyOf( + String.raw`\bagent-tty\b[^\n]*\brecord\b[^\n]*\bexport\b`, + String.raw`\brecord(?:ing)?\b[^\n]*\bexport\b`, +); +export const DOCTOR_JSON_PATTERN = anyOf( + String.raw`\bagent-tty\b[^\n]*\bdoctor\b[^\n]*--json\b`, + String.raw`\bdoctor\b[^\n]*--json\b`, +); + +interface WorkflowCheckOptions { + forbiddenPattern?: string; + dependsOn?: string[]; + weight?: number; + required?: boolean; +} + +function cloneAntiPatternRule(rule: AntiPatternRule): AntiPatternRule { + return { + ...rule, + ...(rule.lanes === undefined ? {} : { lanes: [...rule.lanes] }), + }; +} + +export function executionAntiPatterns( + ...extraRules: AntiPatternRule[] +): AntiPatternRule[] { + return [ + ...DEFAULT_ANTI_PATTERN_RULES.map((rule) => cloneAntiPatternRule(rule)), + ...extraRules.map((rule) => cloneAntiPatternRule(rule)), + ]; +} + +export function executionTaskPrompt(task: string, fixture?: string): string { + const normalizedTask = task.trim(); + invariant( + normalizedTask.length > 0, + 'execution task prompt must not be empty', + ); + + return [ + 'ACTUALLY PERFORM this task by running agent-tty CLI commands via `npx tsx src/cli/main.ts`; do not just describe the steps.', + 'Use the isolated `AGENT_TTY_HOME` provided for this eval so session state stays contained.', + fixture === undefined + ? undefined + : `Use the repository fixture app \`${fixture}\` from \`test/fixtures/apps/${fixture}/main.ts\` via the provided setup command.`, + normalizedTask, + ] + .filter((section): section is string => section !== undefined) + .join(' '); +} + +export function fixtureSetupStep( + id: string, + fixture: string, + description: string, + timeoutMs = 30_000, +): SetupStep { + const fixtureCommandSegments = fixtureCommand(fixture); + const [command, ...argv] = fixtureCommandSegments; + + invariant( + command !== undefined, + `fixtureCommand(${fixture}) must include a command`, + ); + + return { + id, + description, + command, + argv, + timeoutMs, + }; +} + +export function requiredVerifier( + id: string, + kind: VerifierKind, + description: string, + config: Record, +): VerifierSpec { + return { + id, + kind, + description, + required: true, + config, + }; +} + +export function customVerifier( + id: string, + description: string, + validator: string, + config: Record, +): VerifierSpec { + return requiredVerifier(id, 'custom', description, { + validator, + ...config, + }); +} + +export function workflowCheck( + id: string, + description: string, + requiredPattern: string, + options: WorkflowCheckOptions = {}, +): WorkflowCheck { + return { + id, + description, + required: options.required ?? true, + requiredPatterns: [requiredPattern], + forbiddenPatterns: + options.forbiddenPattern === undefined ? [] : [options.forbiddenPattern], + dependsOn: [...(options.dependsOn ?? [])], + ...(options.weight === undefined ? {} : { weight: options.weight }), + }; +} + +export function artifactRequirement( + kind: ArtifactKind, + description: string, + pathPattern: string, + minCount = 1, +): ArtifactRequirement { + return { + kind, + required: true, + description, + minCount, + pathPatterns: [pathPattern], + }; +} + +export function executionBudgets( + overrides: Partial = {}, +): ExecutionEvalCase['budgets'] { + return { + timeoutMs: overrides.timeoutMs ?? DEFAULT_TIMEOUT_MS, + maxAgentSteps: overrides.maxAgentSteps ?? DEFAULT_MAX_AGENT_STEPS, + maxWallClockMs: overrides.maxWallClockMs ?? DEFAULT_MAX_WALL_CLOCK_MS, + }; +} + +export function createExecutionCase( + definition: ExecutionEvalCase, +): ExecutionEvalCase { + ExecutionEvalCaseSchema.parse(definition); + return definition; +} diff --git a/evals/execution/cases/unicode-grid.ts b/evals/execution/cases/unicode-grid.ts new file mode 100644 index 00000000..f763e5be --- /dev/null +++ b/evals/execution/cases/unicode-grid.ts @@ -0,0 +1,74 @@ +import { + ALL_EXECUTION_CONDITIONS, + CREATE_SESSION_PATTERN, + DESTROY_SESSION_PATTERN, + SNAPSHOT_PATTERN, + createExecutionCase, + executionAntiPatterns, + executionBudgets, + executionTaskPrompt, + fixtureSetupStep, + requiredVerifier, + workflowCheck, +} from './shared.js'; + +export const unicodeGridCase = createExecutionCase({ + id: 'unicode-grid', + lane: 'execution', + category: 'artifact', + prompt: executionTaskPrompt( + 'Launch unicode-grid, capture a semantic snapshot to verify Unicode rendering, then destroy the session.', + 'unicode-grid', + ), + expectedSkill: 'agent-tty', + fixture: 'unicode-grid', + conditions: [...ALL_EXECUTION_CONDITIONS], + setup: [ + fixtureSetupStep( + 'launch-unicode-grid', + 'unicode-grid', + 'Create an agent-tty session that runs the unicode-grid fixture.', + ), + ], + verifiers: [ + requiredVerifier( + 'unicode-grid-snapshot', + 'snapshot', + 'The semantic snapshot should preserve the Unicode fixture markers and glyphs.', + { + patterns: [ + String.raw`UNICODE GRID FIXTURE`, + String.raw`┌─┐`, + String.raw`漢字`, + String.raw`UNICODE GRID COMPLETE`, + ], + }, + ), + ], + workflowChecks: [ + workflowCheck( + 'create', + 'Create the fixture session.', + CREATE_SESSION_PATTERN, + ), + workflowCheck( + 'snapshot', + 'Capture a semantic snapshot of the Unicode fixture.', + SNAPSHOT_PATTERN, + { dependsOn: ['create'] }, + ), + workflowCheck( + 'destroy', + 'Destroy the session after snapshot verification.', + DESTROY_SESSION_PATTERN, + { dependsOn: ['snapshot'] }, + ), + ], + antiPatterns: executionAntiPatterns(), + artifactRequirements: [], + budgets: executionBudgets({ + timeoutMs: 120_000, + maxAgentSteps: 10, + maxWallClockMs: 60_000, + }), +}); diff --git a/evals/execution/runner.ts b/evals/execution/runner.ts new file mode 100644 index 00000000..56eac71d --- /dev/null +++ b/evals/execution/runner.ts @@ -0,0 +1,1002 @@ +import { mkdtemp, readdir, readFile, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { assertString, invariant } from '../../src/util/assert.js'; +import { detectAntiPatterns } from '../lib/antiPatterns.js'; +import { createIsolatedEvalHome } from '../lib/cliHarness.js'; +import { EvalResultSchema } from '../lib/schemas.js'; +import { checkWorkflow } from '../lib/scoring.js'; +import type { + AntiPatternFinding, + ArtifactRequirement, + EvalResult, + ExecutionEvalCase, + NormalizedProviderOutput, + ProviderRuntimeInfo, + RunMetadata, + ScoreComponent, + SkillCondition, + VerifierSpec, + WorkflowCheckResult, +} from '../lib/types.js'; +import type { EvalProvider } from '../providers/base.js'; +import { altScreenDemoCase } from './cases/alt-screen-demo.js'; +import { colorGridCase } from './cases/color-grid.js'; +import { crashRecoveryCase } from './cases/crash-recovery.js'; +import { doctorGatedCase } from './cases/doctor-gated.js'; +import { exportProofCase } from './cases/export-proof.js'; +import { helloPromptCase } from './cases/hello-prompt.js'; +import { resizeDemoCase } from './cases/resize-demo.js'; +import { runCommandCase } from './cases/run-command.js'; +import { scrollbackDemoCase } from './cases/scrollback-demo.js'; +import { unicodeGridCase } from './cases/unicode-grid.js'; +import type { VerifierResult } from './verifiers/index.js'; +import { verify, verifyArtifactExists } from './verifiers/index.js'; + +const EXECUTION_CASES = [ + helloPromptCase, + resizeDemoCase, + altScreenDemoCase, + colorGridCase, + unicodeGridCase, + scrollbackDemoCase, + crashRecoveryCase, + exportProofCase, + runCommandCase, + doctorGatedCase, +] satisfies readonly ExecutionEvalCase[]; + +const CASE_CATEGORY_EXPECTATIONS = { + session: 2, + tui: 3, + artifact: 4, + recovery: 1, +} as const; +const RUNNER_OUTPUT_PREFIX = 'agent-tty-execution-eval-'; +const DEFAULT_TRIAL = 1; + +type EvaluatedVerifier = { + spec: VerifierSpec; + result: VerifierResult; +}; + +interface LoadedExecutionSkillPrompts { + bootstrapSkillText: string; + canonicalAgentTtySkillText: string; +} + +let loadedExecutionSkillPromptsPromise: + | Promise + | undefined; + +const EXECUTION_INSTRUCTIONS = [ + 'IMPORTANT: You must ACTUALLY PERFORM this task by running commands, not just describe what you would do.', + 'Use `npx tsx src/cli/main.ts` to invoke agent-tty commands.', + 'Set `AGENT_TTY_HOME` to the provided home directory for session isolation.', +].join('\n'); + +const STALE_EXECUTION_SKILL_TEXT = [ + 'Legacy agent-tty guidance snapshot (known to be stale/wrong):', + '- Start sessions with `agent-tty start` instead of `agent-tty create`.', + '- Use `sleep 5` before checking terminal readiness instead of `agent-tty wait`.', + '- Capture screenshots with `scrot` or other OS tools instead of `agent-tty screenshot`.', + '- It is fine to leave sessions running after the task ends.', + '', + 'Some of the guidance above is wrong for this repository snapshot. Verify commands against the current CLI behavior while completing the task.', +].join('\n'); + +async function readSkillFile(relativePath: string): Promise { + const content = await readFile( + new URL(relativePath, import.meta.url), + 'utf8', + ); + assertString(content, `Skill file ${relativePath} must be a string`); + invariant(content.length > 0, `Skill file ${relativePath} must not be empty`); + return content; +} + +async function loadExecutionSkillPrompts(): Promise { + loadedExecutionSkillPromptsPromise ??= (async () => { + const [bootstrapSkillText, canonicalAgentTtySkillText] = await Promise.all([ + readSkillFile('../../skills/agent-tty/SKILL.md'), + readSkillFile('../../skill-data/agent-tty/SKILL.md'), + ]); + + return { + bootstrapSkillText, + canonicalAgentTtySkillText, + }; + })(); + + return loadedExecutionSkillPromptsPromise; +} + +type EvaluatedArtifactRequirement = { + requirement: ArtifactRequirement; + result: VerifierResult; +}; + +assertExecutionCaseInventory(EXECUTION_CASES); + +function assertExecutionCaseInventory( + cases: readonly ExecutionEvalCase[], +): void { + invariant( + cases.length === 10, + 'Execution lane must register exactly 10 cases', + ); + + const seenIds = new Set(); + const categoryCounts = { + session: 0, + tui: 0, + artifact: 0, + recovery: 0, + }; + + for (const evalCase of cases) { + invariant( + !seenIds.has(evalCase.id), + `Duplicate execution case id: ${evalCase.id}`, + ); + seenIds.add(evalCase.id); + categoryCounts[evalCase.category] += 1; + } + + for (const [category, expectedCount] of Object.entries( + CASE_CATEGORY_EXPECTATIONS, + ) as Array<[keyof typeof CASE_CATEGORY_EXPECTATIONS, number]>) { + invariant( + categoryCounts[category] === expectedCount, + `Execution lane category ${category} must contain ${String(expectedCount)} cases`, + ); + } +} + +function safeRatio(numerator: number, denominator: number): number { + if (denominator === 0) { + return 1; + } + + return numerator / denominator; +} + +function countPassedWorkflowChecks( + workflowChecks: readonly WorkflowCheckResult[], + evalCase: ExecutionEvalCase, +): { passed: number; total: number; failed: WorkflowCheckResult[] } { + const requiredCheckIds = new Set( + evalCase.workflowChecks + .filter((check) => check.required) + .map((check) => check.id), + ); + const failed = workflowChecks.filter( + (result) => requiredCheckIds.has(result.checkId) && !result.passed, + ); + + return { + passed: workflowChecks.filter( + (result) => requiredCheckIds.has(result.checkId) && result.passed, + ).length, + total: requiredCheckIds.size, + failed, + }; +} + +function countPassedVerifiers(verifierResults: readonly EvaluatedVerifier[]): { + passed: number; + total: number; + failed: EvaluatedVerifier[]; +} { + const requiredVerifiers = verifierResults.filter(({ spec }) => spec.required); + return { + passed: requiredVerifiers.filter(({ result }) => result.pass).length, + total: requiredVerifiers.length, + failed: requiredVerifiers.filter(({ result }) => !result.pass), + }; +} + +function countPassedArtifactRequirements( + artifactResults: readonly EvaluatedArtifactRequirement[], +): { passed: number; total: number; failed: EvaluatedArtifactRequirement[] } { + const requiredArtifacts = artifactResults.filter( + ({ requirement }) => requirement.required, + ); + return { + passed: requiredArtifacts.filter(({ result }) => result.pass).length, + total: requiredArtifacts.length, + failed: requiredArtifacts.filter(({ result }) => !result.pass), + }; +} + +function antiPatternScore(findings: readonly AntiPatternFinding[]): number { + return Math.max(0, 1 - Math.min(1, findings.length)); +} + +function summarizeAntiPatterns( + findings: readonly AntiPatternFinding[], +): string { + if (findings.length === 0) { + return 'No anti-pattern findings'; + } + + const counts = { + error: findings.filter((finding) => finding.severity === 'error').length, + warning: findings.filter((finding) => finding.severity === 'warning') + .length, + info: findings.filter((finding) => finding.severity === 'info').length, + }; + return `${String(counts.error)} error(s), ${String(counts.warning)} warning(s), ${String(counts.info)} info finding(s)`; +} + +function createFallbackNormalizedOutput( + transcript: string, +): NormalizedProviderOutput { + return { + finalText: transcript, + messages: transcript.length === 0 ? [] : [transcript], + referencedSkills: [], + toolCalls: [], + }; +} + +function safeStringify(value: unknown): string { + try { + return JSON.stringify(value, null, 2); + } catch { + return String(value); + } +} + +function pushSection( + sections: string[], + title: string, + content: string | undefined, +): void { + if (content === undefined || content.length === 0) { + return; + } + + sections.push(`## ${title}\n${content}`); +} + +async function readTextIfExists( + path: string | undefined, +): Promise { + if (path === undefined) { + return undefined; + } + + try { + return await readFile(path, 'utf8'); + } catch { + return undefined; + } +} + +async function buildTranscript(result: { + rawStdout: string; + rawStderr: string; + normalized: NormalizedProviderOutput; + transcriptPath?: string; +}): Promise { + const sections: string[] = []; + pushSection(sections, 'raw-stdout', result.rawStdout); + pushSection(sections, 'raw-stderr', result.rawStderr); + pushSection(sections, 'normalized-final-text', result.normalized.finalText); + pushSection( + sections, + 'normalized-reasoning', + result.normalized.reasoningText, + ); + if (result.normalized.messages.length > 0) { + pushSection( + sections, + 'normalized-messages', + result.normalized.messages.join('\n---\n'), + ); + } + if (result.normalized.toolCalls.length > 0) { + pushSection( + sections, + 'normalized-tool-calls', + result.normalized.toolCalls + .map((toolCall) => safeStringify(toolCall)) + .join('\n'), + ); + } + const providerTranscript = await readTextIfExists(result.transcriptPath); + pushSection(sections, 'provider-transcript', providerTranscript); + return sections.join('\n\n'); +} + +async function persistRunnerArtifacts( + outputDir: string, + transcript: string, + rawStdout: string, + rawStderr: string, +): Promise<{ transcriptPath: string; stdoutPath: string; stderrPath: string }> { + const transcriptPath = join(outputDir, 'transcript.txt'); + const stdoutPath = join(outputDir, 'stdout.txt'); + const stderrPath = join(outputDir, 'stderr.txt'); + + await Promise.all([ + writeFile(transcriptPath, transcript, 'utf8'), + writeFile(stdoutPath, rawStdout, 'utf8'), + writeFile(stderrPath, rawStderr, 'utf8'), + ]); + + return { + transcriptPath, + stdoutPath, + stderrPath, + }; +} + +async function collectFiles( + targetPath: string, + paths: Set, +): Promise { + try { + const targetStats = await stat(targetPath); + if (targetStats.isFile()) { + paths.add(resolve(targetPath)); + return; + } + if (!targetStats.isDirectory()) { + return; + } + } catch { + return; + } + + const entries = await readdir(targetPath, { withFileTypes: true }); + for (const entry of entries) { + const entryPath = join(targetPath, entry.name); + if (entry.isDirectory()) { + await collectFiles(entryPath, paths); + continue; + } + if (entry.isFile()) { + paths.add(resolve(entryPath)); + } + } +} + +async function collectArtifacts(paths: readonly string[]): Promise { + const artifacts = new Set(); + for (const path of paths) { + await collectFiles(path, artifacts); + } + + return [...artifacts].sort((left, right) => left.localeCompare(right)); +} + +function formatCommand(command: string, argv: readonly string[]): string { + return [command, ...argv].join(' '); +} + +async function buildPromptForCondition( + evalCase: ExecutionEvalCase, + condition: SkillCondition, +): Promise { + const sections = [evalCase.prompt, '', EXECUTION_INSTRUCTIONS]; + + switch (condition) { + case 'none': { + sections.push( + '', + 'Skill condition: none.', + 'No agent-tty skill text is preloaded for this run. Complete the task using the current repository context and actual CLI behavior.', + ); + break; + } + case 'self-load': { + const { bootstrapSkillText } = await loadExecutionSkillPrompts(); + sections.push( + '', + 'Skill condition: self-load.', + 'The following bootstrap skill is available. You can load the full skill by running `agent-tty skills get agent-tty`. Use this guidance to complete the task.', + '', + 'Bootstrap skill text:', + bootstrapSkillText.trim(), + ); + break; + } + case 'preloaded': { + const { canonicalAgentTtySkillText } = await loadExecutionSkillPrompts(); + sections.push( + '', + 'Skill condition: preloaded.', + 'The following agent-tty skill documentation is preloaded. Follow it to complete the task.', + '', + 'Canonical core skill text:', + canonicalAgentTtySkillText.trim(), + ); + break; + } + case 'stale': { + sections.push( + '', + 'Skill condition: stale.', + 'The following guidance is intentionally stale or wrong. Verify commands against the current repository behavior while completing the task.', + '', + 'Stale guidance:', + STALE_EXECUTION_SKILL_TEXT, + ); + break; + } + } + + if (evalCase.fixture !== undefined) { + sections.push('', `Fixture: ${evalCase.fixture}`); + } + + if (evalCase.setup.length > 0) { + sections.push( + '', + 'Reference setup command(s):', + ...evalCase.setup.map( + (step) => + `- ${step.description}: ${formatCommand(step.command, step.argv)}`, + ), + ); + } + + const requiredArtifacts = evalCase.artifactRequirements.filter( + (requirement) => requirement.required, + ); + if (requiredArtifacts.length > 0) { + sections.push( + '', + 'Required artifacts:', + ...requiredArtifacts.map((requirement) => `- ${requirement.description}`), + ); + } + + return sections.join('\n'); +} + +function buildVerifierFailures( + evalCase: ExecutionEvalCase, + message: string, +): EvaluatedVerifier[] { + return evalCase.verifiers.map((spec) => ({ + spec, + result: { + pass: false, + message, + }, + })); +} + +function buildArtifactFailures( + evalCase: ExecutionEvalCase, + message: string, +): EvaluatedArtifactRequirement[] { + return evalCase.artifactRequirements.map((requirement) => ({ + requirement, + result: { + pass: false, + message, + }, + })); +} + +async function evaluateArtifactRequirements( + requirements: readonly ArtifactRequirement[], + artifacts: readonly string[], +): Promise { + const ctx = { + home: '', + sessionId: '', + transcript: '', + artifacts: [...artifacts], + }; + + return Promise.all( + requirements.map(async (requirement) => ({ + requirement, + result: await verifyArtifactExists( + { + kind: requirement.kind, + pathPatterns: requirement.pathPatterns, + minCount: requirement.minCount ?? 1, + }, + ctx, + ), + })), + ); +} + +function summarizeFailureReasons( + providerOk: boolean, + verifierResults: readonly EvaluatedVerifier[], + workflowChecks: readonly WorkflowCheckResult[], + antiPatternFindings: readonly AntiPatternFinding[], + artifactResults: readonly EvaluatedArtifactRequirement[], + providerErrorMessage?: string, +): string { + const reasons: string[] = []; + if (!providerOk) { + reasons.push( + providerErrorMessage ?? 'Provider invocation reported failure', + ); + } + + const failedVerifiers = verifierResults + .filter(({ spec, result }) => spec.required && !result.pass) + .map(({ spec }) => spec.id); + if (failedVerifiers.length > 0) { + reasons.push(`Failed verifiers: ${failedVerifiers.join(', ')}`); + } + + const failedWorkflowChecks = workflowChecks + .filter((check) => !check.passed) + .map((check) => check.checkId); + if (failedWorkflowChecks.length > 0) { + reasons.push(`Failed workflow checks: ${failedWorkflowChecks.join(', ')}`); + } + + const errorFindings = antiPatternFindings.filter( + (finding) => finding.severity === 'error', + ); + if (errorFindings.length > 0) { + reasons.push( + `Error-level anti-patterns: ${errorFindings.map((finding) => finding.ruleId).join(', ')}`, + ); + } + + const missingArtifacts = artifactResults + .filter(({ requirement, result }) => requirement.required && !result.pass) + .map(({ requirement }) => requirement.kind); + if (missingArtifacts.length > 0) { + reasons.push(`Missing required artifacts: ${missingArtifacts.join(', ')}`); + } + + return reasons.join('; '); +} + +function buildScoreBreakdown( + providerOk: boolean, + evalCase: ExecutionEvalCase, + verifierResults: readonly EvaluatedVerifier[], + workflowChecks: readonly WorkflowCheckResult[], + antiPatternFindings: readonly AntiPatternFinding[], + artifactResults: readonly EvaluatedArtifactRequirement[], +): { + breakdown: { total: number; maxPossible: number; items: ScoreComponent[] }; + ok: boolean; +} { + const verifierStatus = countPassedVerifiers(verifierResults); + const workflowStatus = countPassedWorkflowChecks(workflowChecks, evalCase); + const artifactStatus = countPassedArtifactRequirements(artifactResults); + const errorFindings = antiPatternFindings.filter( + (finding) => finding.severity === 'error', + ); + + const items: ScoreComponent[] = [ + { + name: 'provider-invocation', + score: providerOk ? 1 : 0, + maxScore: 1, + reason: providerOk + ? 'Provider agent-mode invocation completed successfully' + : 'Provider agent-mode invocation failed', + }, + { + name: 'verifier-pass-rate', + score: safeRatio(verifierStatus.passed, verifierStatus.total), + maxScore: 1, + reason: `Passed ${String(verifierStatus.passed)} of ${String(verifierStatus.total)} required verifier(s)`, + }, + { + name: 'workflow-compliance', + score: safeRatio(workflowStatus.passed, workflowStatus.total), + maxScore: 1, + reason: `Passed ${String(workflowStatus.passed)} of ${String(workflowStatus.total)} required workflow check(s)`, + }, + { + name: 'anti-pattern-avoidance', + score: antiPatternScore(errorFindings), + maxScore: 1, + reason: summarizeAntiPatterns(antiPatternFindings), + }, + ]; + + if (artifactStatus.total > 0) { + items.push({ + name: 'artifact-requirements', + score: safeRatio(artifactStatus.passed, artifactStatus.total), + maxScore: 1, + reason: `Satisfied ${String(artifactStatus.passed)} of ${String(artifactStatus.total)} required artifact expectation(s)`, + }); + } + + const total = items.reduce((sum, item) => sum + item.score, 0); + const maxPossible = items.reduce((sum, item) => sum + item.maxScore, 0); + const ok = + providerOk && + verifierStatus.failed.length === 0 && + workflowStatus.failed.length === 0 && + errorFindings.length === 0 && + artifactStatus.failed.length === 0; + + return { + breakdown: { + total, + maxPossible, + items, + }, + ok, + }; +} + +function normalizeRequestedConditions( + conditions: readonly SkillCondition[] | undefined, +): Set | undefined { + if (conditions === undefined || conditions.length === 0) { + return undefined; + } + + return new Set(conditions); +} + +function resolveModelId( + metadata: RunMetadata, + runtime: ProviderRuntimeInfo | undefined, +): string | undefined { + if (metadata.models.length === 1) { + return metadata.models[0]; + } + + return runtime?.defaultModelId; +} + +async function createEvalRequest( + provider: EvalProvider, + metadata: RunMetadata, + evalCase: ExecutionEvalCase, + condition: SkillCondition, + homeDir: string, + outputDir: string, + runtime: ProviderRuntimeInfo | undefined, +): Promise[0]> { + const prompt = await buildPromptForCondition(evalCase, condition); + const modelId = resolveModelId(metadata, runtime); + + return { + runId: metadata.runId, + providerId: provider.id, + condition, + trial: DEFAULT_TRIAL, + cwd: resolve(metadata.repoRoot), + homeDir, + outputDir, + env: { + AGENT_TTY_HOME: homeDir, + AGENT_TTY_EVAL_OUTPUT_DIR: outputDir, + }, + evalCase: { + ...evalCase, + prompt, + }, + ...(modelId === undefined ? {} : { modelId }), + }; +} + +function buildResult( + metadata: RunMetadata, + provider: EvalProvider, + evalCase: ExecutionEvalCase, + condition: SkillCondition, + runtime: ProviderRuntimeInfo | undefined, + normalizedOutput: NormalizedProviderOutput, + providerOk: boolean, + startedAt: string, + completedAt: string, + durationMs: number, + workflowChecks: WorkflowCheckResult[], + antiPatternFindings: AntiPatternFinding[], + verifierResults: readonly EvaluatedVerifier[], + artifactResults: readonly EvaluatedArtifactRequirement[], + transcriptPath: string, + stdoutPath: string, + stderrPath: string, + bundlePath: string | undefined, + eventLogPath: string | undefined, + errorClass: string | undefined, + providerErrorMessage: string | undefined, +): EvalResult { + const scored = buildScoreBreakdown( + providerOk, + evalCase, + verifierResults, + workflowChecks, + antiPatternFindings, + artifactResults, + ); + const modelId = resolveModelId(metadata, runtime); + const failureMessage = scored.ok + ? undefined + : summarizeFailureReasons( + providerOk, + verifierResults, + workflowChecks, + antiPatternFindings, + artifactResults, + providerErrorMessage, + ); + + const result: EvalResult = { + runId: metadata.runId, + providerId: provider.id, + ...(runtime?.version === undefined + ? {} + : { providerVersion: runtime.version }), + ...(modelId === undefined ? {} : { modelId }), + lane: 'execution', + caseId: evalCase.id, + category: evalCase.category, + condition, + expectedSkill: evalCase.expectedSkill, + trial: DEFAULT_TRIAL, + ok: scored.ok, + score: scored.breakdown, + workflowChecks, + antiPatternFindings, + transcriptPath, + stdoutPath, + stderrPath, + ...(bundlePath === undefined ? {} : { bundlePath }), + ...(eventLogPath === undefined ? {} : { eventLogPath }), + normalizedOutput, + ...(scored.ok || errorClass === undefined ? {} : { errorClass }), + ...(failureMessage === undefined ? {} : { errorMessage: failureMessage }), + startedAt, + completedAt, + durationMs, + }; + + EvalResultSchema.parse(result); + return result; +} + +async function detectRuntime( + provider: EvalProvider, +): Promise { + try { + return await provider.detect(); + } catch { + return undefined; + } +} + +async function runSingleExecutionCase( + provider: EvalProvider, + metadata: RunMetadata, + evalCase: ExecutionEvalCase, + condition: SkillCondition, + runtime: ProviderRuntimeInfo | undefined, +): Promise { + const homeDir = await createIsolatedEvalHome(); + const outputDir = await mkdtemp(join(tmpdir(), RUNNER_OUTPUT_PREFIX)); + const request = await createEvalRequest( + provider, + metadata, + evalCase, + condition, + homeDir, + outputDir, + runtime, + ); + const invocationStartedAt = new Date().toISOString(); + const invocationStartedMs = Date.now(); + + if (runtime !== undefined && !runtime.available) { + const transcript = `Provider ${provider.id} is unavailable.`; + const persisted = await persistRunnerArtifacts( + outputDir, + transcript, + '', + '', + ); + return buildResult( + metadata, + provider, + evalCase, + condition, + runtime, + createFallbackNormalizedOutput(transcript), + false, + invocationStartedAt, + invocationStartedAt, + 0, + checkWorkflow(transcript, evalCase.workflowChecks), + detectAntiPatterns(transcript, evalCase.antiPatterns), + buildVerifierFailures( + evalCase, + 'Provider is unavailable; verifier did not run.', + ), + buildArtifactFailures( + evalCase, + 'Provider is unavailable; artifact verification did not run.', + ), + persisted.transcriptPath, + persisted.stdoutPath, + persisted.stderrPath, + undefined, + undefined, + 'provider-unavailable', + 'Provider detect() reported unavailable.', + ); + } + + try { + const providerResult = await provider.invokeAgentMode(request); + const transcript = await buildTranscript(providerResult); + const persisted = await persistRunnerArtifacts( + outputDir, + transcript, + providerResult.rawStdout, + providerResult.rawStderr, + ); + const artifacts = await collectArtifacts([ + outputDir, + ...(providerResult.bundlePath === undefined + ? [] + : [providerResult.bundlePath]), + ...(providerResult.transcriptPath === undefined + ? [] + : [providerResult.transcriptPath]), + ...(providerResult.eventLogPath === undefined + ? [] + : [providerResult.eventLogPath]), + ]); + const verifierContext = { + home: homeDir, + sessionId: providerResult.sessionId ?? '', + transcript, + artifacts, + }; + const verifierResults = await Promise.all( + evalCase.verifiers.map(async (spec) => ({ + spec, + result: await verify(spec, verifierContext), + })), + ); + const artifactResults = await evaluateArtifactRequirements( + evalCase.artifactRequirements, + artifacts, + ); + return buildResult( + metadata, + provider, + evalCase, + condition, + providerResult.runtime, + providerResult.normalized, + providerResult.ok, + providerResult.startedAt, + providerResult.completedAt, + providerResult.durationMs, + checkWorkflow(transcript, evalCase.workflowChecks), + detectAntiPatterns(transcript, evalCase.antiPatterns), + verifierResults, + artifactResults, + providerResult.transcriptPath ?? persisted.transcriptPath, + persisted.stdoutPath, + persisted.stderrPath, + providerResult.bundlePath, + providerResult.eventLogPath, + providerResult.errorClass, + providerResult.errorMessage, + ); + } catch (error) { + const completedAt = new Date().toISOString(); + const durationMs = Math.max(0, Date.now() - invocationStartedMs); + const errorMessage = error instanceof Error ? error.message : String(error); + const transcript = + error instanceof Error && error.stack !== undefined + ? `${error.name}: ${errorMessage}\n${error.stack}` + : errorMessage; + const persisted = await persistRunnerArtifacts( + outputDir, + transcript, + '', + errorMessage, + ); + return buildResult( + metadata, + provider, + evalCase, + condition, + runtime, + createFallbackNormalizedOutput(transcript), + false, + invocationStartedAt, + completedAt, + durationMs, + checkWorkflow(transcript, evalCase.workflowChecks), + detectAntiPatterns(transcript, evalCase.antiPatterns), + buildVerifierFailures( + evalCase, + 'Provider invocation failed before verifier execution.', + ), + buildArtifactFailures( + evalCase, + 'Provider invocation failed before artifact verification.', + ), + persisted.transcriptPath, + persisted.stdoutPath, + persisted.stderrPath, + undefined, + undefined, + error instanceof Error ? error.name : 'ProviderInvocationError', + errorMessage, + ); + } +} + +/** Return all registered execution-lane cases in deterministic order. */ +export function getAllExecutionCases(): ExecutionEvalCase[] { + assertExecutionCaseInventory(EXECUTION_CASES); + return [...EXECUTION_CASES]; +} + +/** + * Run the execution eval lane for one provider across the selected case and + * condition matrix. + */ +export async function runExecutionLane( + provider: EvalProvider, + metadata: RunMetadata, + options: { conditions?: SkillCondition[]; caseFilter?: string[] } = {}, +): Promise { + const requestedConditions = normalizeRequestedConditions(options.conditions); + const requestedCaseIds = + options.caseFilter === undefined || options.caseFilter.length === 0 + ? undefined + : new Set(options.caseFilter); + const availableCases = getAllExecutionCases(); + + if (requestedCaseIds !== undefined) { + const availableCaseIds = new Set( + availableCases.map((evalCase) => evalCase.id), + ); + for (const caseId of requestedCaseIds) { + invariant( + availableCaseIds.has(caseId), + `Unknown execution case id: ${caseId}`, + ); + } + } + + const selectedCases = availableCases.filter( + (evalCase) => + requestedCaseIds === undefined || requestedCaseIds.has(evalCase.id), + ); + const runtime = await detectRuntime(provider); + const results: EvalResult[] = []; + + for (const evalCase of selectedCases) { + const selectedConditions = evalCase.conditions.filter( + (condition) => + requestedConditions === undefined || requestedConditions.has(condition), + ); + for (const condition of selectedConditions) { + results.push( + await runSingleExecutionCase( + provider, + metadata, + evalCase, + condition, + runtime, + ), + ); + } + } + + return results; +} diff --git a/evals/execution/verifiers/index.ts b/evals/execution/verifiers/index.ts new file mode 100644 index 00000000..e38ad27e --- /dev/null +++ b/evals/execution/verifiers/index.ts @@ -0,0 +1,577 @@ +import { readFile, stat } from 'node:fs/promises'; +import { resolve } from 'node:path'; + +import { SessionRecordSchema } from '../../../src/protocol/schemas.js'; +import { manifestPath, sessionDir } from '../../../src/storage/sessionPaths.js'; +import type { ArtifactKind } from '../../../src/tools/review-bundle.js'; +import type { BundleValidationProfile } from '../../../src/tools/validate-bundle.js'; +import { validateBundle } from '../../../src/tools/validate-bundle.js'; +import { assertString, unreachable } from '../../../src/util/assert.js'; +import { readEvalEvents } from '../../lib/cliHarness.js'; +import { matchPatterns } from '../../lib/scoring.js'; +import type { VerifierSpec } from '../../lib/types.js'; + +const ARTIFACT_MATCHERS: Record = { + screenshot: [/\.png$/iu], + video: [/\.webm$/iu], + recording: [/\.cast$/iu], + json: [/\.json$/iu], + notes: [/\.md$/iu], + script: [/\.(?:sh|bash|zsh|fish|ps1|cmd|bat)$/iu], + support: [/.+/u], + other: [/.+/u], +}; + +const CUSTOM_VALIDATORS = [ + 'artifact-exists', + 'snapshot-contains', + 'event-log-check', + 'exit-code', + 'bundle-valid', +] as const; + +type CustomValidator = (typeof CUSTOM_VALIDATORS)[number]; + +type ExitCodeLookup = + | { + source: 'event-log' | 'manifest'; + exitCode: number | null; + } + | undefined; + +export interface VerifierContext { + home: string; + sessionId: string; + transcript: string; + artifacts: string[]; +} + +export interface VerifierResult { + pass: boolean; + message: string; + details?: Record; +} + +function success( + message: string, + details?: Record, +): VerifierResult { + return details === undefined + ? { pass: true, message } + : { pass: true, message, details }; +} + +function failure( + message: string, + details?: Record, +): VerifierResult { + return details === undefined + ? { pass: false, message } + : { pass: false, message, details }; +} + +function readRequiredStringArray( + config: Record, + key: string, +): string[] { + const value = config[key]; + if (!Array.isArray(value)) { + throw new Error(`Verifier config.${key} must be an array of strings`); + } + + return value.map((entry, index) => { + assertString( + entry, + `Verifier config.${key}[${String(index)}] must be a string`, + ); + return entry; + }); +} + +function readOptionalStringArray( + config: Record, + key: string, +): string[] { + const value = config[key]; + if (value === undefined) { + return []; + } + + return readRequiredStringArray(config, key); +} + +function readOptionalInteger( + config: Record, + key: string, +): number | undefined { + const value = config[key]; + if (value === undefined) { + return undefined; + } + + if (typeof value !== 'number' || !Number.isInteger(value)) { + throw new Error(`Verifier config.${key} must be an integer`); + } + + return value; +} + +function readRequiredInteger( + config: Record, + key: string, +): number { + const value = readOptionalInteger(config, key); + if (value === undefined) { + throw new Error(`Verifier config.${key} is required`); + } + return value; +} + +function readOptionalString( + config: Record, + key: string, +): string | undefined { + const value = config[key]; + if (value === undefined) { + return undefined; + } + + assertString(value, `Verifier config.${key} must be a string`); + return value; +} + +function readArtifactKind(value: unknown, label: string): ArtifactKind { + assertString(value, `${label} must be a string`); + if (!(value in ARTIFACT_MATCHERS)) { + throw new Error(`${label} must be a supported artifact kind`); + } + + return value as ArtifactKind; +} + +function readRequestedArtifactKinds( + config: Record, +): ArtifactKind[] { + const kinds = new Set(); + + if (config.kind !== undefined) { + kinds.add(readArtifactKind(config.kind, 'Verifier config.kind')); + } + + if (config.kinds !== undefined) { + const requestedKinds = readRequiredStringArray(config, 'kinds'); + for (const kind of requestedKinds) { + kinds.add(readArtifactKind(kind, 'Verifier config.kinds entry')); + } + } + + return [...kinds]; +} + +async function fileExists(targetPath: string): Promise { + try { + return (await stat(targetPath)).isFile(); + } catch { + return false; + } +} + +async function existingArtifacts(paths: readonly string[]): Promise { + const uniquePaths = [...new Set(paths.map((path) => resolve(path)))]; + const existing: string[] = []; + + for (const path of uniquePaths) { + if (await fileExists(path)) { + existing.push(path); + } + } + + return existing; +} + +function matchesAnyPattern(path: string, patterns: readonly string[]): boolean { + if (patterns.length === 0) { + return true; + } + + return matchPatterns(path, [...patterns]).some((result) => result.matched); +} + +async function readSessionExitCode( + home: string, + sessionId: string, +): Promise { + const events = readEvalEvents(home, sessionId); + const exitEvent = [...events] + .reverse() + .find((event) => event.type === 'exit'); + if (exitEvent?.type === 'exit') { + return { + source: 'event-log', + exitCode: exitEvent.payload.exitCode, + }; + } + + const sessionManifestPath = manifestPath(sessionDir(home, sessionId)); + try { + const rawManifest = await readFile(sessionManifestPath, 'utf8'); + const manifestCandidate = JSON.parse(rawManifest) as unknown; + const parsedManifest = SessionRecordSchema.safeParse(manifestCandidate); + if (!parsedManifest.success) { + return undefined; + } + + return { + source: 'manifest', + exitCode: parsedManifest.data.exitCode, + }; + } catch { + return undefined; + } +} + +function resolveCustomValidator( + config: Record, +): CustomValidator | undefined { + const validator = readOptionalString(config, 'validator'); + if (validator === undefined) { + return undefined; + } + + if ((CUSTOM_VALIDATORS as readonly string[]).includes(validator)) { + return validator as CustomValidator; + } + + return undefined; +} + +/** Dispatch a verifier spec against one execution transcript context. */ +export async function verify( + spec: VerifierSpec, + ctx: VerifierContext, +): Promise { + switch (spec.kind) { + case 'snapshot': + return verifySnapshotContains(spec.config, ctx); + case 'event-log': + return verifyEventLogCheck(spec.config, ctx); + case 'screenshot': + return verifyArtifactExists({ kind: 'screenshot', ...spec.config }, ctx); + case 'command': + return verifyExitCode(spec.config, ctx); + case 'bundle': + return verifyBundleValid(spec.config, ctx); + case 'json': + return spec.config.patterns === undefined + ? verifyArtifactExists({ kind: 'json', ...spec.config }, ctx) + : verifySnapshotContains(spec.config, ctx); + case 'custom': { + const validator = resolveCustomValidator(spec.config); + switch (validator) { + case 'artifact-exists': + return verifyArtifactExists(spec.config, ctx); + case 'snapshot-contains': + return verifySnapshotContains(spec.config, ctx); + case 'event-log-check': + return verifyEventLogCheck(spec.config, ctx); + case 'exit-code': + return verifyExitCode(spec.config, ctx); + case 'bundle-valid': + return verifyBundleValid(spec.config, ctx); + case undefined: + return failure( + `Unsupported custom verifier for ${spec.id}: expected one of ${CUSTOM_VALIDATORS.join(', ')}`, + ); + default: + return unreachable(validator, 'unsupported custom validator'); + } + } + default: + return unreachable(spec.kind, 'unsupported verifier kind'); + } +} + +/** Verify that transcript content contains all required snapshot patterns. */ +export function verifySnapshotContains( + config: Record, + ctx: VerifierContext, +): Promise { + try { + const patterns = readRequiredStringArray(config, 'patterns'); + const results = matchPatterns(ctx.transcript, patterns); + const missingPatterns = results.filter((result) => !result.matched); + if (missingPatterns.length > 0) { + return Promise.resolve( + failure( + `Snapshot transcript missed ${String(missingPatterns.length)} required pattern(s).`, + { + missingPatterns: missingPatterns.map((result) => result.pattern), + matches: results, + }, + ), + ); + } + + return Promise.resolve( + success( + `Matched all ${String(results.length)} required snapshot pattern(s).`, + { + matches: results, + }, + ), + ); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return Promise.resolve(failure(`Snapshot verification failed: ${message}`)); + } +} + +/** Verify event-log content, event types, and output patterns for a session. */ +export function verifyEventLogCheck( + config: Record, + ctx: VerifierContext, +): Promise { + if (ctx.sessionId.trim().length === 0) { + return Promise.resolve( + failure( + 'Event-log verification requires a sessionId in the verifier context.', + ), + ); + } + + try { + const requiredEventTypes = readOptionalStringArray( + config, + 'requiredEventTypes', + ); + const forbiddenEventTypes = readOptionalStringArray( + config, + 'forbiddenEventTypes', + ); + const requiredOutputPatterns = readOptionalStringArray( + config, + 'requiredOutputPatterns', + ); + const minEvents = readOptionalInteger(config, 'minEvents') ?? 1; + const events = readEvalEvents(ctx.home, ctx.sessionId); + if (events.length < minEvents) { + return Promise.resolve( + failure( + `Expected at least ${String(minEvents)} event-log record(s), found ${String(events.length)}.`, + { + foundEventTypes: events.map((event) => event.type), + }, + ), + ); + } + + const eventTypes = new Set(events.map((event) => event.type)); + const missingEventTypes = requiredEventTypes.filter( + (eventType) => + !eventTypes.has(eventType as (typeof events)[number]['type']), + ); + if (missingEventTypes.length > 0) { + return Promise.resolve( + failure('Event log missed required event types.', { + missingEventTypes, + foundEventTypes: [...eventTypes], + }), + ); + } + + const presentForbiddenEventTypes = forbiddenEventTypes.filter((eventType) => + eventTypes.has(eventType as (typeof events)[number]['type']), + ); + if (presentForbiddenEventTypes.length > 0) { + return Promise.resolve( + failure('Event log contained forbidden event types.', { + presentForbiddenEventTypes, + foundEventTypes: [...eventTypes], + }), + ); + } + + const outputText = events + .filter((event) => event.type === 'output') + .map((event) => event.payload.data) + .join(''); + const outputMatches = matchPatterns(outputText, requiredOutputPatterns); + const missingOutputPatterns = outputMatches.filter( + (result) => !result.matched, + ); + if (missingOutputPatterns.length > 0) { + return Promise.resolve( + failure('Event log output missed required transcript patterns.', { + missingOutputPatterns: missingOutputPatterns.map( + (result) => result.pattern, + ), + outputMatches, + }), + ); + } + + return Promise.resolve( + success( + `Event log satisfied ${String(requiredEventTypes.length)} required type check(s).`, + { + eventCount: events.length, + foundEventTypes: [...eventTypes], + outputMatches, + }, + ), + ); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return Promise.resolve( + failure(`Event-log verification failed: ${message}`), + ); + } +} + +/** Verify that expected artifact files exist in the collected artifact set. */ +export async function verifyArtifactExists( + config: Record, + ctx: VerifierContext, +): Promise { + try { + const requestedKinds = readRequestedArtifactKinds(config); + const pathPatterns = readOptionalStringArray(config, 'pathPatterns'); + const minCount = readOptionalInteger(config, 'minCount') ?? 1; + const artifacts = await existingArtifacts(ctx.artifacts); + if (artifacts.length === 0) { + return failure('No artifact files were available to validate.'); + } + + if (requestedKinds.length === 0) { + const matchingArtifacts = artifacts.filter((path) => + matchesAnyPattern(path, pathPatterns), + ); + if (matchingArtifacts.length < minCount) { + return failure( + `Expected at least ${String(minCount)} matching artifact(s), found ${String(matchingArtifacts.length)}.`, + { + matchingArtifacts, + artifacts, + }, + ); + } + + return success( + `Found ${String(matchingArtifacts.length)} matching artifact(s).`, + { + matchingArtifacts, + }, + ); + } + + const matchedByKind: Record = {}; + const missingKinds: ArtifactKind[] = []; + for (const kind of requestedKinds) { + const kindMatches = artifacts.filter((path) => { + const kindMatch = ARTIFACT_MATCHERS[kind].some((pattern) => + pattern.test(path), + ); + return kindMatch && matchesAnyPattern(path, pathPatterns); + }); + matchedByKind[kind] = kindMatches; + if (kindMatches.length < minCount) { + missingKinds.push(kind); + } + } + + if (missingKinds.length > 0) { + return failure('Missing one or more required artifact kinds.', { + missingKinds, + matchedByKind, + }); + } + + return success( + `Found required artifacts for ${requestedKinds.join(', ')}.`, + { + matchedByKind, + }, + ); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return failure(`Artifact verification failed: ${message}`); + } +} + +/** Verify the observed session exit code from the event log or manifest. */ +export async function verifyExitCode( + config: Record, + ctx: VerifierContext, +): Promise { + if (ctx.sessionId.trim().length === 0) { + return failure( + 'Exit-code verification requires a sessionId in the verifier context.', + ); + } + + try { + const expectedExitCode = readRequiredInteger(config, 'expectedExitCode'); + const actualExitCode = await readSessionExitCode(ctx.home, ctx.sessionId); + if (actualExitCode === undefined) { + return failure( + 'Could not determine an exit code from the event log or manifest.', + ); + } + + if (actualExitCode.exitCode !== expectedExitCode) { + return failure( + `Expected exit code ${String(expectedExitCode)}, observed ${String(actualExitCode.exitCode)}.`, + { + expectedExitCode, + actualExitCode: actualExitCode.exitCode, + source: actualExitCode.source, + }, + ); + } + + return success( + `Observed expected exit code ${String(expectedExitCode)} from the ${actualExitCode.source}.`, + { + expectedExitCode, + source: actualExitCode.source, + }, + ); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return failure(`Exit-code verification failed: ${message}`); + } +} + +/** Validate a proof bundle directory against one of the built-in profiles. */ +export async function verifyBundleValid( + config: Record, + ctx: VerifierContext, +): Promise { + void ctx; + try { + const bundlePath = readOptionalString(config, 'bundlePath'); + if (bundlePath === undefined) { + return failure('Bundle verification requires config.bundlePath.'); + } + + const profile = + (readOptionalString(config, 'profile') as + | BundleValidationProfile + | undefined) ?? 'interactive-renderer'; + const validationResult = await validateBundle(resolve(bundlePath), profile); + if (!validationResult.ok) { + return failure('Bundle validation failed.', { + profile, + checks: validationResult.checks, + }); + } + + return success('Bundle validation passed.', { + profile, + checks: validationResult.checks, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return failure(`Bundle verification failed: ${message}`); + } +} diff --git a/evals/lib/antiPatterns.ts b/evals/lib/antiPatterns.ts new file mode 100644 index 00000000..f5fd46d7 --- /dev/null +++ b/evals/lib/antiPatterns.ts @@ -0,0 +1,785 @@ +import { assertString, invariant } from '../../src/util/assert.js'; +import { isInNegationContext } from './scoring.js'; +import type { + AntiPatternFinding, + AntiPatternRule, + AntiPatternSeverity, +} from './types.js'; + +const SEVERITY_ORDER: Record = { + info: 0, + warning: 1, + error: 2, +}; + +const COMMENT_ONLY_LINE_PATTERN = /^\s*(?:#|\/\/|\/\*|\*|\*\/|