refactor(e2e): move cassette filters per-scenario, harden seinfeld defaults

- Replace global cassette-filters.mjs registry with per-scenario cassette-filter.mjs
  files; cassette-preload.mjs now dynamically imports them from the scenario dir
- Default redact to 'paranoid' in seinfeld recorder (was opt-in)
- Gate provider key placeholder injection on replay mode only (not record/passthrough)
- Delete obsolete cassette-filters.mjs and record-cassettes.mjs helper scripts

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: Stephen Belanger

dev-packages/seinfeld/README.md

@@ -5,24 +5,16 @@ Generic VCR/cassette library for Node.js, built on [MSW](https://mswjs.io). Reco
 ## Features
 
 - **Normalizers** (always-on, lossy) transform requests before matching. They strip volatile fields like `Authorization` headers, dynamic IDs (`experimental_generateMessageId`), or query nonces so two structurally-identical requests still match across runs. Their output is internal — never serialized.
-- **Redactors** (opt-in) transform what gets persisted to disk. They mask credentials before the cassette hits version control. Disabled by default; cassettes contain the real on-the-wire bytes unless you opt in.
+- **Redactors** transform what gets persisted to disk. They mask credentials before the cassette hits version control. The `'paranoid'` preset is applied by default; pass `redact: []` to disable.
 
 ## Security note
 
-> **Cassettes contain real request and response bytes by default, including `Authorization` headers.** This is the safer default for fidelity (downstream consumers see real responses) but it means you must either (a) enable redaction, (b) write a custom `RedactionConfig`, or (c) add cassette files to `.gitignore` if they may contain credentials.
-
-Three body-redaction gaps are worth knowing:
+Three body-redaction gaps are worth knowing even with the default `'paranoid'` preset:
 
 1. **Non-canonical content-type** — some servers return JSON with `Content-Type: text/plain`. `redactBodyFields` covers this because seinfeld attempts to parse `text` bodies as JSON before masking.
 2. **SSE event data** — streaming endpoints (OpenAI, Anthropic) emit JSON in `data:` lines. `redactBodyFields` applies to parseable `data:` lines; `redactBodyText` handles non-JSON SSE content.
 3. **Plain-text credentials** — form-encoded bodies, XML, or log-like text are opaque to field-path rules. Use `redactBodyText` with a regex.
 
-For cassettes committed to version control, use the `'paranoid'` preset, which covers all three paths:
-
-```ts
-createCassette({ name: "demo", redact: "paranoid" });
-```
-
 `'paranoid'` redacts credential headers, common credential field names at any JSON depth (`apiKey`, `token`, `secret`, `password`, `authorization`), and Bearer / `sk-` style tokens in text bodies.
 
 To detect misconfigurations at record time, add `strict: true`:

dev-packages/seinfeld/src/recorder.ts

@@ -348,7 +348,7 @@ export interface CassetteOptions {
   store?: CassetteStore;
   /** Filter spec (matching-only normalization). */
   filters?: FilterSpec;
-  /** Redaction spec (applied to persisted bytes). Defaults to none. */
+  /** Redaction spec (applied to persisted bytes). Defaults to `'paranoid'`. Pass `[]` to disable. */
   redact?: RedactionSpec;
   /** Hosts to intercept. Other hosts pass through. Defaults to all hosts. */
   hosts?: Array<string | RegExp>;
@@ -403,7 +403,7 @@ export function createCassette(options: CassetteOptions): Cassette {
       options.store ?? createJsonFileStore({ rootDir: DEFAULT_CASSETTE_DIR }),
     matcher: options.matcher ?? createDefaultMatcher(),
     filters: options.filters,
-    redact: options.redact,
+    redact: options.redact ?? "paranoid",
     hosts: options.hosts,
     passthroughHosts: options.passthroughHosts,
     threshold: resolveThreshold(options.externalBlobThreshold),

e2e/README.md

@@ -178,7 +178,7 @@ unset ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY COHERE_API_KEY GROQ_API_KE
 pnpm --filter=@braintrust/js-e2e-tests run test:e2e:hermetic
 ```
 
-If a scenario records but later replay fails because of volatile fields in the request body (e.g. AI-SDK's generated message ids), add or update the filter for that scenario in `e2e/helpers/cassette-filters.mjs`, then re-record.
+If a scenario records but later replay fails because of volatile fields in the request body (e.g. AI-SDK's generated message ids), add or update `<scenario-dir>/cassette-filter.mjs` for that scenario, then re-record.
 
 ### In-scope scenarios
 

e2e/helpers/cassette-filters.mjs

@@ -8,20 +8,22 @@
  *   BRAINTRUST_E2E_CASSETTE_MODE    — replay | record | passthrough
  *   BRAINTRUST_E2E_CASSETTE_VARIANT — variant key (cassette name, no extension)
  *   BRAINTRUST_E2E_MOCK_HOST        — host:port of the Braintrust mock server (always passthrough)
- *   BRAINTRUST_E2E_CASSETTE_NORMALIZER — name of the request-body filter to use
  *
  * The preload exits silently if the cassette path env var is not set, so
  * it's safe to install for scenarios that haven't migrated yet (the
  * harness only sets the env vars for opted-in scenarios).
+ *
+ * Per-scenario request-body filters live in `<scenario-dir>/cassette-filter.mjs`
+ * (optional). The file should export a named `filter` conforming to the
+ * seinfeld `FilterSpec` type. If absent, the seinfeld `"default"` preset is used.
  */
+import * as path from "node:path";
 import { createCassette, createJsonFileStore } from "@braintrust/seinfeld";
-import { CASSETTE_FILTERS } from "./cassette-filters.mjs";
 
 const CASSETTE_DIR = process.env.BRAINTRUST_E2E_CASSETTE_PATH;
 const MODE_RAW = process.env.BRAINTRUST_E2E_CASSETTE_MODE ?? "replay";
 const VARIANT_KEY = process.env.BRAINTRUST_E2E_CASSETTE_VARIANT ?? "default";
 const MOCK_HOST = process.env.BRAINTRUST_E2E_MOCK_HOST;
-const NORMALIZER_NAME = process.env.BRAINTRUST_E2E_CASSETTE_NORMALIZER;
 
 if (CASSETTE_DIR) {
   await bootCassettePreload(CASSETTE_DIR);
@@ -32,16 +34,14 @@ if (CASSETTE_DIR) {
  */
 async function bootCassettePreload(cassetteDir) {
   const mode = resolveMode(MODE_RAW);
-  const filters =
-    CASSETTE_FILTERS[NORMALIZER_NAME ?? ""] ?? CASSETTE_FILTERS["default"];
+  const filters = await loadScenarioFilter(cassetteDir);
   const passthroughHosts = MOCK_HOST ? [MOCK_HOST] : [];
 
   const cassette = createCassette({
     name: VARIANT_KEY,
     mode,
     store: createJsonFileStore({ rootDir: cassetteDir }),
     filters,
-    redact: "paranoid",
     passthroughHosts,
     onMiss: (req) => {
       process.stderr.write(`[cassette] MISS: ${req.method} ${req.url}\n`);
@@ -62,6 +62,28 @@ async function bootCassettePreload(cassetteDir) {
   });
 }
 
+/**
+ * Try to load a per-scenario cassette filter from `<scenario-dir>/cassette-filter.mjs`.
+ * Falls back to the seinfeld `"default"` preset if the file is absent.
+ *
+ * @param {string} cassetteDir  Absolute path to the __cassettes__ directory.
+ * @returns {Promise<import("@braintrust/seinfeld").FilterSpec>}
+ */
+async function loadScenarioFilter(cassetteDir) {
+  // cassetteDir is <scenario>/__cassettes__ — parent is the scenario root.
+  const scenarioDir = path.resolve(cassetteDir, "..");
+  const filterPath = path.join(scenarioDir, "cassette-filter.mjs");
+  try {
+    const mod = await import(filterPath);
+    if (mod.filter !== undefined) {
+      return mod.filter;
+    }
+  } catch {
+    // File absent or not a valid module — fall through to default.
+  }
+  return "default";
+}
+
 /**
  * @param {string} raw
  * @returns {import('@braintrust/seinfeld').CassetteMode}

e2e/helpers/cassette-preload.mjs

@@ -49,12 +49,6 @@ export interface ScenarioCassetteConfig {
    * Defaults to `runContext.variantKey ?? "default"`.
    */
   variantKey?: string;
-  /**
-   * Name of the request-body normalizer registered in
-   * `e2e/helpers/cassette/normalizers/index.mjs`. Falls back to a
-   * scenario-name based lookup if omitted.
-   */
-  normalizerName?: string;
 }
 
 export interface ScenarioRunContext {
@@ -271,20 +265,15 @@ interface CassetteWiring {
   cassetteDir: string;
   variantKey: string;
   mockHost: string;
-  normalizerName?: string;
 }
 
 function getCassetteEnv(wiring: CassetteWiring): Record<string, string> {
-  const env: Record<string, string> = {
+  return {
     BRAINTRUST_E2E_CASSETTE_PATH: wiring.cassetteDir,
     BRAINTRUST_E2E_CASSETTE_MODE: process.env[CASSETTE_MODE_ENV] ?? "replay",
     BRAINTRUST_E2E_CASSETTE_VARIANT: wiring.variantKey,
     BRAINTRUST_E2E_MOCK_HOST: wiring.mockHost,
   };
-  if (wiring.normalizerName) {
-    env.BRAINTRUST_E2E_CASSETTE_NORMALIZER = wiring.normalizerName;
-  }
-  return env;
 }
 
 /**
@@ -679,15 +668,18 @@ export async function withScenarioHarness(
       return {};
     }
 
-    const normalizerName = config.normalizerName ?? scenarioName;
+    const isReplayMode = !isRecordingMode && cassetteModeRaw !== "passthrough";
 
     return {
-      ...getProviderKeyPlaceholders(),
+      // Only inject placeholder keys in replay mode. In record mode the
+      // subprocess needs the real provider keys to make live API calls;
+      // injecting a fake key causes a confusing "invalid key" error instead
+      // of the clear "missing key" error the SDK would otherwise produce.
+      ...(isReplayMode ? getProviderKeyPlaceholders() : {}),
       ...getCassetteEnv({
         cassetteDir: path.dirname(cassettePath),
         variantKey,
         mockHost: urlToHostHeader(server.url),
-        normalizerName,
       }),
     };
   };

e2e/helpers/scenario-harness.ts

@@ -0,0 +1,47 @@
+// @ts-check
+/** @type {import("@braintrust/seinfeld").FilterSpec} */
+export const filter = [
+  "default",
+  {
+    ignoreBodyFields: [
+      // Ignore all body fields — deterministic call order makes callIndex
+      // the sole discriminator, which is stable across SDK releases.
+      "**",
+      // AI SDK volatile fields (change per-run)
+      "experimental_generateMessageId",
+      "messageId",
+      "messages.*.id",
+      "messages.*.experimental_messageId",
+      "input.*.id",
+      "input.*.experimental_messageId",
+      // OpenAI Responses API fields added as defaults in newer client versions.
+      // These don't affect request semantics but change between SDK releases.
+      "store",
+      "background",
+      "truncation",
+      "instructions",
+      "moderation",
+      "reasoning",
+      "reasoning.effort",
+      "reasoning.summary",
+      "safety_identifier",
+      "service_tier",
+      "text",
+      "text.format",
+      "text.format.type",
+      "text.verbosity",
+      "metadata",
+      "top_logprobs",
+      "top_p",
+      "presence_penalty",
+      "frequency_penalty",
+      "parallel_tool_calls",
+      "max_tool_calls",
+      "prompt_cache_key",
+      "prompt_cache_retention",
+      "previous_response_id",
+      "user",
+      "include",
+    ],
+  },
+];

e2e/scenarios/ai-sdk-instrumentation/cassette-filter.mjs

@@ -0,0 +1,3 @@
+// @ts-check
+// Same AI SDK volatile fields as ai-sdk-instrumentation.
+export { filter } from "../ai-sdk-instrumentation/cassette-filter.mjs";

e2e/scenarios/ai-sdk-otel-export/cassette-filter.mjs

@@ -0,0 +1,39 @@
+// @ts-check
+/** @type {import("@braintrust/seinfeld").FilterSpec} */
+export const filter = [
+  "default",
+  {
+    /**
+     * Mistral's client generates a unique `name` field per session
+     * (e.g. "braintrust-e2e-<uuid>"). Normalize it so cassette matching
+     * isn't broken by the per-run suffix.
+     *
+     * @param {import("@braintrust/seinfeld").RecordedRequest} req
+     * @returns {import("@braintrust/seinfeld").RecordedRequest}
+     */
+    normalizeRequest(req) {
+      if (
+        req.body.kind !== "json" ||
+        req.body.value === null ||
+        typeof req.body.value !== "object" ||
+        Array.isArray(req.body.value)
+      ) {
+        return req;
+      }
+      const value = /** @type {Record<string, unknown>} */ (req.body.value);
+      if (
+        typeof value["name"] === "string" &&
+        /** @type {string} */ (value["name"]).startsWith("braintrust-e2e-")
+      ) {
+        return {
+          ...req,
+          body: {
+            kind: "json",
+            value: { ...value, name: "braintrust-e2e-<placeholder>" },
+          },
+        };
+      }
+      return req;
+    },
+  },
+];

e2e/scenarios/mistral-instrumentation/cassette-filter.mjs

@@ -0,0 +1,3 @@
+// @ts-check
+// Same body-agnostic matching as openrouter-instrumentation.
+export { filter } from "../openrouter-instrumentation/cassette-filter.mjs";