From 1d9429fe3c36b68e088081afeb45a87fd2386aa3 Mon Sep 17 00:00:00 2001 From: Julien Huang Date: Tue, 30 Jun 2026 17:08:40 +0200 Subject: [PATCH] fix(agent-eval): provide type augmentation for agent-eval/eval --- .changeset/eval-type-augmentation.md | 5 ++ README.md | 2 + packages/agent-eval/eslint.config.js | 11 +++ packages/agent-eval/package.json | 11 +++ .../src/lib/agents/eval-helper.d.mts | 67 +++++++++++++++++++ 5 files changed, 96 insertions(+) create mode 100644 .changeset/eval-type-augmentation.md create mode 100644 packages/agent-eval/src/lib/agents/eval-helper.d.mts diff --git a/.changeset/eval-type-augmentation.md b/.changeset/eval-type-augmentation.md new file mode 100644 index 0000000..ad21b8c --- /dev/null +++ b/.changeset/eval-type-augmentation.md @@ -0,0 +1,5 @@ +--- +'@vercel/agent-eval': patch +--- + +Ship public types for the `@vercel/agent-eval/eval` judge surface. Importing `environment`/`transcript` now also augments Vitest's `expect` with the `toSatisfyCriterion` and `toScoreAtLeast` matchers — EVAL.ts files type-check with no manual `declare module 'vitest'` boilerplate. The `eval` subpath is now a real package export (resolvable in editors, not just the in-sandbox alias), and `JudgeSubject` / `JudgeVerdict` types are exported for advanced use. diff --git a/README.md b/README.md index 4674930..d27d4a5 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,8 @@ You supply only the **criterion** string; the framework owns the judge prompt an The judge uses the same agent and model as the run under test. Because each assertion is a real agent run, it costs time and tokens — keep criteria focused. +**TypeScript support is built in.** Importing from `@vercel/agent-eval/eval` types the `environment`/`transcript` subjects _and_ registers the `toSatisfyCriterion` / `toScoreAtLeast` matchers on Vitest's `expect` — no manual `declare module 'vitest'` augmentation needed. The package also re-exports the `JudgeSubject` and `JudgeVerdict` types for advanced use. + > **Note**: requires `validation: 'vitest'` (the default). The framework gives the eval process the run's credentials automatically so the judge can call the agent CLI in-sandbox. ## Configuration Reference diff --git a/packages/agent-eval/eslint.config.js b/packages/agent-eval/eslint.config.js index 6c6633f..bbd8e94 100644 --- a/packages/agent-eval/eslint.config.js +++ b/packages/agent-eval/eslint.config.js @@ -19,6 +19,17 @@ export default tseslint.config( globals: { process: 'readonly', console: 'readonly' }, }, }, + { + // Hand-authored declaration files (e.g. eval-helper.d.mts) augment upstream + // interfaces — `declare module 'vitest'` must mirror Vitest's exact + // `Assertion` signature for the merge to apply, so the `any` and the + // otherwise-unused type parameter are unavoidable, not accidental. + files: ['src/**/*.d.mts', 'src/**/*.d.ts'], + rules: { + '@typescript-eslint/no-explicit-any': 'off', + '@typescript-eslint/no-unused-vars': 'off', + }, + }, { ignores: ['dist/**', 'node_modules/**'], }, diff --git a/packages/agent-eval/package.json b/packages/agent-eval/package.json index b8af00f..a369162 100644 --- a/packages/agent-eval/package.json +++ b/packages/agent-eval/package.json @@ -9,6 +9,17 @@ "type": "module", "main": "dist/index.js", "types": "dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "./eval": { + "types": "./dist/lib/agents/eval-helper.d.mts", + "default": "./dist/lib/agents/eval-helper.mjs" + }, + "./package.json": "./package.json" + }, "bin": { "agent-eval": "dist/cli.js" }, diff --git a/packages/agent-eval/src/lib/agents/eval-helper.d.mts b/packages/agent-eval/src/lib/agents/eval-helper.d.mts new file mode 100644 index 0000000..9045e2b --- /dev/null +++ b/packages/agent-eval/src/lib/agents/eval-helper.d.mts @@ -0,0 +1,67 @@ +/** Which artifact a judge assertion evaluates. */ +export type JudgeSubjectKind = 'environment' | 'transcript'; + +/** + * Opaque sentinel you pass to `expect(...)`. The matcher routes on which subject + * it receives; the path/cwd resolution is an internal detail. Import the + * `environment` and `transcript` values rather than constructing this yourself. + */ +export interface JudgeSubject { + readonly __judgeSubject: JudgeSubjectKind; +} + +/** A parsed judge verdict. */ +export interface JudgeVerdict { + /** Whether the judge decided the criterion is satisfied. */ + pass: boolean; + /** 0–1 score, present only for numeric judgments. */ + score?: number; + /** 1–2 sentences citing concrete evidence. */ + reason: string; +} + +/** Options for {@link buildJudgePrompt}. */ +export interface BuildJudgePromptOptions { + /** Ask the judge for a 0–1 score in addition to the pass/fail verdict. */ + numeric?: boolean; +} + +export declare const environment: JudgeSubject; + +export declare const transcript: JudgeSubject; + +export declare function transcriptPath(): string; + +export declare function buildJudgePrompt( + subject: JudgeSubjectKind, + criterion: string, + verdictPath: string, + opts?: BuildJudgePromptOptions +): string; + +export declare function parseJudgeVerdict(raw: string | null | undefined): JudgeVerdict | null; + +declare module 'vitest' { + interface Assertion { + /** + * Agentic LLM-judge matcher. Re-invokes the codegen agent in-sandbox to decide + * whether `criterion` is satisfied for the `environment` or `transcript` subject. + * Pass the subject to `expect(...)`: + * + * await expect(environment).toSatisfyCriterion('uses Server Components'); + */ + toSatisfyCriterion(criterion: string): Promise; + /** + * Agentic LLM-judge matcher. Passes when the judge's 0–1 score for `criterion` + * is `>= threshold`: + * + * await expect(environment).toScoreAtLeast('production-quality errors', 0.8); + */ + toScoreAtLeast(criterion: string, threshold: number): Promise; + } + + interface AsymmetricMatchersContaining { + toSatisfyCriterion(criterion: string): Promise; + toScoreAtLeast(criterion: string, threshold: number): Promise; + } +}