fix(agent-eval): provide type augmentation for agent-eval/eval

.changeset/eval-type-augmentation.md

@@ -0,0 +1,5 @@
+---
+'@vercel/agent-eval': patch
+---
+
+Ship public types for the `@vercel/agent-eval/eval` judge surface. Importing `environment`/`transcript` now also augments Vitest's `expect` with the `toSatisfyCriterion` and `toScoreAtLeast` matchers — EVAL.ts files type-check with no manual `declare module 'vitest'` boilerplate. The `eval` subpath is now a real package export (resolvable in editors, not just the in-sandbox alias), and `JudgeSubject` / `JudgeVerdict` types are exported for advanced use.

README.md

@@ -202,6 +202,8 @@ You supply only the **criterion** string; the framework owns the judge prompt an
 
 The judge uses the same agent and model as the run under test. Because each assertion is a real agent run, it costs time and tokens — keep criteria focused.
 
+**TypeScript support is built in.** Importing from `@vercel/agent-eval/eval` types the `environment`/`transcript` subjects _and_ registers the `toSatisfyCriterion` / `toScoreAtLeast` matchers on Vitest's `expect` — no manual `declare module 'vitest'` augmentation needed. The package also re-exports the `JudgeSubject` and `JudgeVerdict` types for advanced use.
+
 > **Note**: requires `validation: 'vitest'` (the default). The framework gives the eval process the run's credentials automatically so the judge can call the agent CLI in-sandbox.
 
 ## Configuration Reference

packages/agent-eval/eslint.config.js

@@ -19,6 +19,17 @@ export default tseslint.config(
       globals: { process: 'readonly', console: 'readonly' },
     },
   },
+  {
+    // Hand-authored declaration files (e.g. eval-helper.d.mts) augment upstream
+    // interfaces — `declare module 'vitest'` must mirror Vitest's exact
+    // `Assertion<T = any>` signature for the merge to apply, so the `any` and the
+    // otherwise-unused type parameter are unavoidable, not accidental.
+    files: ['src/**/*.d.mts', 'src/**/*.d.ts'],
+    rules: {
+      '@typescript-eslint/no-explicit-any': 'off',
+      '@typescript-eslint/no-unused-vars': 'off',
+    },
+  },
   {
     ignores: ['dist/**', 'node_modules/**'],
   },

packages/agent-eval/package.json

@@ -9,6 +9,17 @@
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./eval": {
+      "types": "./dist/lib/agents/eval-helper.d.mts",
+      "default": "./dist/lib/agents/eval-helper.mjs"
+    },
+    "./package.json": "./package.json"
+  },
   "bin": {
     "agent-eval": "dist/cli.js"
   },

packages/agent-eval/src/lib/agents/eval-helper.d.mts

@@ -0,0 +1,67 @@
+/** Which artifact a judge assertion evaluates. */
+export type JudgeSubjectKind = 'environment' | 'transcript';
+
+/**
+ * Opaque sentinel you pass to `expect(...)`. The matcher routes on which subject
+ * it receives; the path/cwd resolution is an internal detail. Import the
+ * `environment` and `transcript` values rather than constructing this yourself.
+ */
+export interface JudgeSubject {
+  readonly __judgeSubject: JudgeSubjectKind;
+}
+
+/** A parsed judge verdict. */
+export interface JudgeVerdict {
+  /** Whether the judge decided the criterion is satisfied. */
+  pass: boolean;
+  /** 0–1 score, present only for numeric judgments. */
+  score?: number;
+  /** 1–2 sentences citing concrete evidence. */
+  reason: string;
+}
+
+/** Options for {@link buildJudgePrompt}. */
+export interface BuildJudgePromptOptions {
+  /** Ask the judge for a 0–1 score in addition to the pass/fail verdict. */
+  numeric?: boolean;
+}
+
+export declare const environment: JudgeSubject;
+
+export declare const transcript: JudgeSubject;
+
+export declare function transcriptPath(): string;
+
+export declare function buildJudgePrompt(
+  subject: JudgeSubjectKind,
+  criterion: string,
+  verdictPath: string,
+  opts?: BuildJudgePromptOptions
+): string;
+
+export declare function parseJudgeVerdict(raw: string | null | undefined): JudgeVerdict | null;
+
+declare module 'vitest' {
+  interface Assertion<T = any> {
+    /**
+     * Agentic LLM-judge matcher. Re-invokes the codegen agent in-sandbox to decide
+     * whether `criterion` is satisfied for the `environment` or `transcript` subject.
+     * Pass the subject to `expect(...)`:
+     *
+     *   await expect(environment).toSatisfyCriterion('uses Server Components');
+     */
+    toSatisfyCriterion(criterion: string): Promise<void>;
+    /**
+     * Agentic LLM-judge matcher. Passes when the judge's 0–1 score for `criterion`
+     * is `>= threshold`:
+     *
+     *   await expect(environment).toScoreAtLeast('production-quality errors', 0.8);
+     */
+    toScoreAtLeast(criterion: string, threshold: number): Promise<void>;
+  }
+
+  interface AsymmetricMatchersContaining {
+    toSatisfyCriterion(criterion: string): Promise<void>;
+    toScoreAtLeast(criterion: string, threshold: number): Promise<void>;
+  }
+}