From e45907c90bd07a1ad087d3b796195815c2817d91 Mon Sep 17 00:00:00 2001 From: Jude Gao Date: Fri, 17 Jul 2026 15:36:30 -0400 Subject: [PATCH] Accept RegExp needles in toContainText --- .changeset/transcript-to-contain-text.md | 2 +- README.md | 5 ++- .../agent-eval/src/lib/agents/eval-helper.mjs | 41 +++++++++++++++---- .../src/lib/agents/eval-helper.test.ts | 41 ++++++++++++++++--- 4 files changed, 72 insertions(+), 17 deletions(-) diff --git a/.changeset/transcript-to-contain-text.md b/.changeset/transcript-to-contain-text.md index 1b50abc..cff9a47 100644 --- a/.changeset/transcript-to-contain-text.md +++ b/.changeset/transcript-to-contain-text.md @@ -2,4 +2,4 @@ '@vercel/agent-eval': minor --- -Add `expect(transcript).toContainText(needle)` — a deterministic, judge-free EVAL.ts matcher over the materialized transcript. Built for `.not` ("the agent never reached for X"): substring absence checks no longer need a judge run or manual `readFileSync(transcriptPath())`. Misuse (wrong subject, empty needle) and a missing/empty transcript throw instead of returning a failed verdict, so `.not` can never invert them into a silent pass. +Add `expect(transcript).toContainText(needle)` — a deterministic, judge-free EVAL.ts matcher over the materialized transcript. `needle` is an exact substring or a RegExp (use `/…/i` for case-insensitive). Built for `.not` ("the agent never reached for X"): absence checks no longer need a judge run or manual `readFileSync(transcriptPath())`. Misuse (wrong subject, empty needle, empty-matching regex) and a missing/empty transcript throw instead of returning a failed verdict, so `.not` can never invert them into a silent pass. diff --git a/README.md b/README.md index c13a04f..df0d8d4 100644 --- a/README.md +++ b/README.md @@ -200,15 +200,16 @@ Two matchers, on either subject: One deterministic matcher, on `transcript` only — no judge run, free and exact: -- `toContainText(needle)` — passes when the raw transcript contains the substring. Built for `.not`, i.e. asserting the agent *never* said or reached for something: +- `toContainText(needle)` — passes when the raw transcript contains the substring, or matches the `RegExp` (use `/…/i` for case-insensitive). Built for `.not`, i.e. asserting the agent *never* said or reached for something: ```typescript test('never suggested the pages router API', () => { expect(transcript).not.toContainText('getServerSideProps'); + expect(transcript).not.toContainText(/getserversideprops/i); // any casing }); ``` -A missing or empty transcript **throws** (fails the test even under `.not`) — an uncaptured transcript is an infra failure, not evidence of absence. Note the transcript is the agent's native format (for `claude-code`, raw session JSONL), so text containing quotes or newlines appears JSON-escaped there; stick to identifier-like needles. For semantic "never did X" checks, phrase the negation *inside* a `toSatisfyCriterion` criterion instead — do not use `.not.toSatisfyCriterion(...)`, which would invert the judge's fail-closed default into a fail-open one. +A missing or empty transcript **throws** (fails the test even under `.not`) — an uncaptured transcript is an infra failure, not evidence of absence. Note the transcript is the agent's native format (for `claude-code`, raw session JSONL), so text containing quotes or newlines appears JSON-escaped there; stick to identifier-like needles or match the escaped form with a `RegExp`. For semantic "never did X" checks, phrase the negation *inside* a `toSatisfyCriterion` criterion instead — do not use `.not.toSatisfyCriterion(...)`, which would invert the judge's fail-closed default into a fail-open one. You supply only the **criterion** string; the framework owns the judge prompt and the verdict contract. On failure the assertion message carries the judge's reasoning, e.g. `[judge:environment] FAIL (score 0.42): product list is a Client Component`, so a failed judge clause is distinguishable from a failed deterministic test or a crash. diff --git a/packages/agent-eval/src/lib/agents/eval-helper.mjs b/packages/agent-eval/src/lib/agents/eval-helper.mjs index ac5c27a..e4dc924 100644 --- a/packages/agent-eval/src/lib/agents/eval-helper.mjs +++ b/packages/agent-eval/src/lib/agents/eval-helper.mjs @@ -227,10 +227,12 @@ expect.extend({ /** * Deterministic transcript assertion — reads the materialized transcript and - * checks for an exact substring. No judge run, so it is free and exact; built - * for `.not` ("the agent never reached for X"): + * checks for an exact substring or a RegExp match (use `/…/i` for + * case-insensitive). No judge run, so it is free and exact; built for `.not` + * ("the agent never reached for X"): * * expect(transcript).not.toContainText('getServerSideProps'); + * expect(transcript).not.toContainText(/getserversideprops/i); * * Misuse and a missing/empty transcript THROW instead of returning pass:false — * a returned failure would invert into a silent pass under `.not`, and an @@ -238,14 +240,15 @@ expect.extend({ * * The transcript is the agent's NATIVE format (e.g. claude-code = raw session * JSONL), so text containing quotes/newlines appears JSON-escaped there; stick - * to identifier-like needles. + * to identifier-like needles or match the escaped form with a RegExp. */ toContainText(received, needle) { if (subjectOf(received) !== 'transcript') { throw new Error('toContainText expects `transcript` from @vercel/agent-eval/eval'); } - if (typeof needle !== 'string' || needle.length === 0) { - throw new Error('toContainText expects a non-empty string to search for'); + const isRegex = needle instanceof RegExp; + if (!isRegex && (typeof needle !== 'string' || needle.length === 0)) { + throw new Error('toContainText expects a non-empty string or a RegExp to search for'); } const text = existsSync(TRANSCRIPT_FILE) ? readFileSync(TRANSCRIPT_FILE, 'utf8') : ''; if (!text) { @@ -254,8 +257,28 @@ expect.extend({ 'transcript capture failed, so nothing can be asserted about it' ); } - const idx = text.indexOf(needle); + let idx = -1; + let matched = ''; + if (isRegex) { + // Strip g/y so exec always searches from the start, regardless of the + // needle's lastIndex state or reuse across assertions. + const re = new RegExp(needle.source, needle.flags.replace(/[gy]/g, '')); + const m = re.exec(text); + if (m) { + if (m[0] === '') { + throw new Error( + `toContainText: ${String(needle)} matches the empty string, which would make the assertion vacuous` + ); + } + idx = m.index; + matched = m[0]; + } + } else { + idx = text.indexOf(needle); + matched = needle; + } const pass = idx !== -1; + const display = isRegex ? String(needle) : JSON.stringify(needle); return { pass, message: () => { @@ -263,11 +286,11 @@ expect.extend({ // Shown when `.not` fails: cite where the needle actually appears. const start = Math.max(0, idx - 60); const context = text - .slice(start, idx + needle.length + 60) + .slice(start, idx + matched.length + 60) .replace(/\n/g, '\\n'); - return `[transcript] expected NOT to contain ${JSON.stringify(needle)}, but found it at char ${idx}: …${context}…`; + return `[transcript] expected NOT to contain ${display}, but found it at char ${idx}: …${context}…`; } - return `[transcript] expected to contain ${JSON.stringify(needle)} (searched ${text.length} chars)`; + return `[transcript] expected to contain ${display} (searched ${text.length} chars)`; }, }; }, diff --git a/packages/agent-eval/src/lib/agents/eval-helper.test.ts b/packages/agent-eval/src/lib/agents/eval-helper.test.ts index 520ec90..28a81c8 100644 --- a/packages/agent-eval/src/lib/agents/eval-helper.test.ts +++ b/packages/agent-eval/src/lib/agents/eval-helper.test.ts @@ -17,7 +17,7 @@ import { declare module 'vitest' { // eslint-disable-next-line @typescript-eslint/no-explicit-any interface Assertion { - toContainText(needle: string): T; + toContainText(needle: string | RegExp): T; } } @@ -161,11 +161,42 @@ describe('toContainText', () => { ); }); - it('throws on an empty or non-string needle', () => { + it('throws on an empty or non-string, non-RegExp needle', () => { writeTranscript('content'); - expect(() => expect(transcript).toContainText('')).toThrowError(/non-empty string/); + expect(() => expect(transcript).toContainText('')).toThrowError( + /non-empty string or a RegExp/ + ); expect(() => - expect(transcript).toContainText(/regex/ as unknown as string) - ).toThrowError(/non-empty string/); + expect(transcript).toContainText(42 as unknown as string) + ).toThrowError(/non-empty string or a RegExp/); + }); + + it('accepts a RegExp needle, e.g. case-insensitive via the i flag', () => { + writeTranscript('the agent added GetServerSideProps to the page'); + expect(transcript).toContainText(/getserversideprops/i); + expect(transcript).not.toContainText(/useEffect/i); + }); + + it('regex .not failure cites the ACTUAL matched text, not the pattern', () => { + writeTranscript('reached for GETSERVERSIDEPROPS here'); + expect(() => + expect(transcript).not.toContainText(/getserversideprops/i) + ).toThrowError( + /expected NOT to contain \/getserversideprops\/i, but found it at char 12: …reached for GETSERVERSIDEPROPS here…/ + ); + }); + + it('is stable across assertions with a g-flagged regex (no lastIndex drift)', () => { + writeTranscript('x marks the spot'); + const re = /x/g; + expect(transcript).toContainText(re); + expect(transcript).toContainText(re); + }); + + it('throws when the regex matches the empty string — the assertion would be vacuous', () => { + writeTranscript('content'); + expect(() => expect(transcript).toContainText(/z*/)).toThrowError( + /matches the empty string/ + ); }); });