Merge branch 'main' into lforst/add-responses-flavor

Author: lforst

.changeset/fuzzy-evals-warn.md

@@ -0,0 +1,5 @@
+---
+"braintrust": patch
+---
+
+fix: Correct the eval file extension shown in CLI directory warnings

.changeset/short-moles-punch.md

@@ -0,0 +1,5 @@
+---
+"braintrust": minor
+---
+
+feat: Add @cursor/sdk instrumentation

.changeset/yellow-crabs-attend.md

@@ -0,0 +1,5 @@
+---
+"braintrust": minor
+---
+
+(feat) Add experiment dataset filters to experiment metadata

.env.example

@@ -2,6 +2,7 @@ BRAINTRUST_API_KEY=
 OPENAI_API_KEY=
 ANTHROPIC_API_KEY=
 GEMINI_API_KEY=
+CURSOR_API_KEY=
 OPENROUTER_API_KEY=
 MISTRAL_API_KEY=
 HUGGINGFACE_API_KEY=

.github/workflows/checks.yaml

@@ -459,7 +459,6 @@ jobs:
       - js-test
       - js-build
       - e2e-hermetic
-      - js-api-compatibility
       - js-smoke-discover
       - js-smoke-test
       - temporal-js
@@ -492,7 +491,6 @@ jobs:
           check_result "js-test" "${{ needs.js-test.result }}"
           check_result "js-build" "${{ needs.js-build.result }}"
           check_result "e2e-hermetic" "${{ needs.e2e-hermetic.result }}"
-          check_result "js-api-compatibility" "${{ needs.js-api-compatibility.result }}"
           check_result "js-smoke-discover" "${{ needs.js-smoke-discover.result }}"
           check_result "js-smoke-test" "${{ needs.js-smoke-test.result }}"
           check_result "temporal-js" "${{ needs.temporal-js.result }}"

.github/workflows/e2e-canary.yaml

@@ -35,6 +35,7 @@ jobs:
           BRAINTRUST_E2E_PROJECT_NAME: ${{ vars.BRAINTRUST_E2E_PROJECT_NAME }}
           GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
           COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
+          CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }}
           GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}

.github/workflows/integration-tests.yaml

@@ -29,6 +29,7 @@ jobs:
       ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+      CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }}
       OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
       MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
       HUGGINGFACE_API_KEY: ${{ secrets.HUGGINGFACE_API_KEY }}
@@ -58,6 +59,7 @@ jobs:
       ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
       GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
       COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
+      CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }}
       GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
@@ -110,6 +112,7 @@ jobs:
       ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
       GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
       COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
+      CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }}
       GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}

e2e/README.md

@@ -115,6 +115,7 @@ Non-hermetic scenarios require provider credentials in addition to the mock Brai
 - `OPENAI_API_KEY`
 - `ANTHROPIC_API_KEY`
 - `GEMINI_API_KEY` or `GOOGLE_API_KEY`
+- `CURSOR_API_KEY`
 - `OPENROUTER_API_KEY`
 - `MISTRAL_API_KEY`
 - `HUGGINGFACE_API_KEY`

e2e/config/pr-comment-scenarios.json

@@ -133,5 +133,11 @@
         "label": "v0.2.81"
       }
     ]
+  },
+  {
+    "scenarioDirName": "cursor-sdk-instrumentation",
+    "label": "Cursor SDK Instrumentation",
+    "metadataScenario": "cursor-sdk-instrumentation",
+    "variants": [{ "variantKey": "cursor-sdk-v1", "label": "v1" }]
   }
 ]

e2e/helpers/scenario-installer.ts

@@ -22,6 +22,7 @@ const INSTALL_SECRET_ENV_VARS = [
   "ANTHROPIC_API_KEY",
   "BRAINTRUST_API_KEY",
   "COHERE_API_KEY",
+  "CURSOR_API_KEY",
   "GEMINI_API_KEY",
   "GITHUB_TOKEN",
   "GH_TOKEN",
@@ -37,10 +38,9 @@ let cleanupRegistered = false;
 
 type CanaryDependencyRule = {
   packageName: string;
-  query: string;
+  version: string;
 };
 
-const canaryVersionCache = new Map<string, string>();
 const HELPERS_DIR = path.dirname(fileURLToPath(import.meta.url));
 const E2E_ROOT = path.resolve(HELPERS_DIR, "..");
 
@@ -155,32 +155,6 @@ function packageSpecifier(
     : `npm:${packageName}@${version}`;
 }
 
-async function resolveCanaryVersion(
-  rule: CanaryDependencyRule,
-): Promise<string> {
-  const cacheKey = rule.query;
-  const cached = canaryVersionCache.get(cacheKey);
-  if (cached) {
-    return cached;
-  }
-
-  const output = await spawnOrThrow(
-    PNPM_COMMAND,
-    ["view", rule.query, "version", "--json"],
-    process.cwd(),
-    installEnv(),
-  );
-  const parsed = JSON.parse(output) as string | string[];
-  const version = Array.isArray(parsed) ? parsed.at(-1) : parsed;
-
-  if (typeof version !== "string") {
-    throw new Error(`Could not resolve canary version for ${rule.query}`);
-  }
-
-  canaryVersionCache.set(cacheKey, version);
-  return version;
-}
-
 function parseCanaryDependencyRule(
   dependencyName: string,
   rawRule: string,
@@ -195,7 +169,7 @@ function parseCanaryDependencyRule(
   if (rawRule === "latest") {
     return {
       packageName: dependencyName,
-      query: dependencyName,
+      version: "latest",
     };
   }
 
@@ -208,7 +182,7 @@ function parseCanaryDependencyRule(
 
   return {
     packageName: rawRule.slice(0, versionSeparator),
-    query: rawRule,
+    version: rawRule.slice(versionSeparator + 1),
   };
 }
 
@@ -230,11 +204,10 @@ async function rewriteManifestForCanary(scenarioDir: string): Promise<void> {
       rawRule,
       scenarioDir,
     );
-    const version = await resolveCanaryVersion(rule);
     dependencies[dependencyName] = packageSpecifier(
       dependencyName,
       rule.packageName,
-      version,
+      rule.version,
     );
     updated = true;
   }