Skip to content

perf(e2e): parallelize scenario variants within test files #1956

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Stephen Belanger (Qard) wants to merge 5 commits into
base: main
Choose a base branch
from
Open

perf(e2e): parallelize scenario variants within test files #1956

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ef6b9e9
perf(e2e): parallelize scenario variants within test files
May 6, 2026
b476f80
chore: add changeset for e2e parallelization
May 6, 2026
f94b1a7
fix(e2e): use describe.sequential wrappers in claude-agent-sdk test a…
May 6, 2026
27b75d0
fix(e2e): update google-adk snapshots for e.span.ended filter and sor…
May 7, 2026
fe2a4c8
chore(e2e): apply prettier formatting to scenario test files
May 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/parallelize-e2e-scenarios.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"braintrust": patch
---

perf(e2e): parallelize scenario variants within test files
130 changes: 67 additions & 63 deletions e2e/scenarios/ai-sdk-instrumentation/scenario.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,69 +30,73 @@ function parseMajorVersion(version: string): number {
return Number.isFinite(major) ? major : 0;
}

for (const scenario of aiSDKScenarios) {
const sdkMajorVersion = parseMajorVersion(scenario.version);
const supportsRichInputScenarios = sdkMajorVersion >= 5;
const supportsOutputObjectScenario = supportsRichInputScenarios;
const supportsAttachmentScenario = supportsRichInputScenarios;
describe.concurrent("variants", () => {
for (const scenario of aiSDKScenarios) {
const sdkMajorVersion = parseMajorVersion(scenario.version);
const supportsRichInputScenarios = sdkMajorVersion >= 5;
const supportsOutputObjectScenario = supportsRichInputScenarios;
const supportsAttachmentScenario = supportsRichInputScenarios;
const tags = cassetteTagsFor(import.meta.url, scenario.snapshotName);

const tags = cassetteTagsFor(import.meta.url, scenario.snapshotName);
describe(`ai sdk ${scenario.version}`, { tags }, () => {
defineAISDKInstrumentationAssertions({
agentSpanName: scenario.agentSpanName,
name: "wrapped instrumentation",
runScenario: async ({ runScenarioDir }) => {
await runScenarioDir({
entry: scenario.wrapperEntry,
runContext: {
variantKey: scenario.snapshotName,
originalScenarioDir,
},
scenarioDir,
timeoutMs: AI_SDK_SCENARIO_TIMEOUT_MS,
});
},
snapshotName: scenario.snapshotName,
supportsAttachmentScenario,
supportsProviderCacheAssertions: scenario.supportsProviderCacheAssertions,
supportsDenyOutputOverrideScenario: supportsRichInputScenarios,
supportsGenerateObject: scenario.supportsGenerateObject,
supportsOutputObjectScenario,
supportsRerank: scenario.supportsRerank !== false,
supportsStreamObject: scenario.supportsStreamObject,
supportsToolExecution: scenario.supportsToolExecution,
sdkMajorVersion,
testFileUrl: import.meta.url,
timeoutMs: AI_SDK_SCENARIO_TIMEOUT_MS,
});
describe.sequential(`ai sdk ${scenario.version}`, { tags }, () => {
defineAISDKInstrumentationAssertions({
agentSpanName: scenario.agentSpanName,
name: "wrapped instrumentation",
runScenario: async ({ runScenarioDir }) => {
await runScenarioDir({
entry: scenario.wrapperEntry,
runContext: {
variantKey: scenario.snapshotName,
originalScenarioDir,
},
scenarioDir,
timeoutMs: AI_SDK_SCENARIO_TIMEOUT_MS,
});
},
snapshotName: scenario.snapshotName,
supportsAttachmentScenario,
supportsProviderCacheAssertions:
scenario.supportsProviderCacheAssertions,
supportsDenyOutputOverrideScenario: supportsRichInputScenarios,
supportsGenerateObject: scenario.supportsGenerateObject,
supportsOutputObjectScenario,
supportsRerank: scenario.supportsRerank !== false,
supportsStreamObject: scenario.supportsStreamObject,
supportsToolExecution: scenario.supportsToolExecution,
sdkMajorVersion,
testFileUrl: import.meta.url,
timeoutMs: AI_SDK_SCENARIO_TIMEOUT_MS,
});

defineAISDKInstrumentationAssertions({
agentSpanName: scenario.agentSpanName,
name: "auto-hook instrumentation",
runScenario: async ({ runNodeScenarioDir }) => {
await runNodeScenarioDir({
entry: scenario.autoEntry,
nodeArgs: ["--import", "braintrust/hook.mjs"],
runContext: {
variantKey: scenario.snapshotName,
originalScenarioDir,
},
scenarioDir,
timeoutMs: AI_SDK_SCENARIO_TIMEOUT_MS,
});
},
snapshotName: scenario.snapshotName,
supportsAttachmentScenario,
supportsProviderCacheAssertions: scenario.supportsProviderCacheAssertions,
supportsDenyOutputOverrideScenario: supportsRichInputScenarios,
supportsGenerateObject: scenario.supportsGenerateObject,
supportsOutputObjectScenario,
supportsRerank: scenario.supportsRerank !== false,
supportsStreamObject: scenario.supportsStreamObject,
supportsToolExecution: scenario.supportsToolExecution,
sdkMajorVersion,
testFileUrl: import.meta.url,
timeoutMs: AI_SDK_SCENARIO_TIMEOUT_MS,
defineAISDKInstrumentationAssertions({
agentSpanName: scenario.agentSpanName,
name: "auto-hook instrumentation",
runScenario: async ({ runNodeScenarioDir }) => {
await runNodeScenarioDir({
entry: scenario.autoEntry,
nodeArgs: ["--import", "braintrust/hook.mjs"],
runContext: {
variantKey: scenario.snapshotName,
originalScenarioDir,
},
scenarioDir,
timeoutMs: AI_SDK_SCENARIO_TIMEOUT_MS,
});
},
snapshotName: scenario.snapshotName,
supportsAttachmentScenario,
supportsProviderCacheAssertions:
scenario.supportsProviderCacheAssertions,
supportsDenyOutputOverrideScenario: supportsRichInputScenarios,
supportsGenerateObject: scenario.supportsGenerateObject,
supportsOutputObjectScenario,
supportsRerank: scenario.supportsRerank !== false,
supportsStreamObject: scenario.supportsStreamObject,
supportsToolExecution: scenario.supportsToolExecution,
sdkMajorVersion,
testFileUrl: import.meta.url,
timeoutMs: AI_SDK_SCENARIO_TIMEOUT_MS,
});
});
});
}
}
});
96 changes: 49 additions & 47 deletions e2e/scenarios/anthropic-instrumentation/scenario.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,52 +74,54 @@ const anthropicScenarios = await Promise.all(
})),
);

for (const scenario of anthropicScenarios) {
const tags = cassetteTagsFor(import.meta.url, scenario.snapshotName);
describe(`anthropic sdk ${scenario.version}`, { tags }, () => {
defineAnthropicInstrumentationAssertions({
name: "wrapped instrumentation",
runScenario: async ({ runScenarioDir }) => {
await runScenarioDir({
entry: scenario.wrapperEntry,
runContext: {
variantKey: scenario.snapshotName,
originalScenarioDir,
},
scenarioDir,
timeoutMs: TIMEOUT_MS,
});
},
snapshotName: scenario.snapshotName,
supportsBetaMessages: scenario.supportsBetaMessages,
supportsBetaToolRunner: scenario.supportsBetaToolRunner ?? true,
supportsServerToolUse: scenario.supportsServerToolUse ?? true,
supportsThinking: scenario.supportsThinking,
testFileUrl: import.meta.url,
timeoutMs: TIMEOUT_MS,
});
describe.concurrent("variants", () => {
for (const scenario of anthropicScenarios) {
const tags = cassetteTagsFor(import.meta.url, scenario.snapshotName);
describe.sequential(`anthropic sdk ${scenario.version}`, { tags }, () => {
defineAnthropicInstrumentationAssertions({
name: "wrapped instrumentation",
runScenario: async ({ runScenarioDir }) => {
await runScenarioDir({
entry: scenario.wrapperEntry,
runContext: {
variantKey: scenario.snapshotName,
originalScenarioDir,
},
scenarioDir,
timeoutMs: TIMEOUT_MS,
});
},
snapshotName: scenario.snapshotName,
supportsBetaMessages: scenario.supportsBetaMessages,
supportsBetaToolRunner: scenario.supportsBetaToolRunner ?? true,
supportsServerToolUse: scenario.supportsServerToolUse ?? true,
supportsThinking: scenario.supportsThinking,
testFileUrl: import.meta.url,
timeoutMs: TIMEOUT_MS,
});

defineAnthropicInstrumentationAssertions({
name: "auto-hook instrumentation",
runScenario: async ({ runNodeScenarioDir }) => {
await runNodeScenarioDir({
entry: scenario.autoEntry,
nodeArgs: ["--import", "braintrust/hook.mjs"],
runContext: {
variantKey: scenario.snapshotName,
originalScenarioDir,
},
scenarioDir,
timeoutMs: TIMEOUT_MS,
});
},
snapshotName: scenario.snapshotName,
supportsBetaMessages: scenario.supportsBetaMessages,
supportsBetaToolRunner: scenario.supportsBetaToolRunner ?? true,
supportsServerToolUse: scenario.supportsServerToolUse ?? true,
supportsThinking: scenario.supportsThinking,
testFileUrl: import.meta.url,
timeoutMs: TIMEOUT_MS,
defineAnthropicInstrumentationAssertions({
name: "auto-hook instrumentation",
runScenario: async ({ runNodeScenarioDir }) => {
await runNodeScenarioDir({
entry: scenario.autoEntry,
nodeArgs: ["--import", "braintrust/hook.mjs"],
runContext: {
variantKey: scenario.snapshotName,
originalScenarioDir,
},
scenarioDir,
timeoutMs: TIMEOUT_MS,
});
},
snapshotName: scenario.snapshotName,
supportsBetaMessages: scenario.supportsBetaMessages,
supportsBetaToolRunner: scenario.supportsBetaToolRunner ?? true,
supportsServerToolUse: scenario.supportsServerToolUse ?? true,
supportsThinking: scenario.supportsThinking,
testFileUrl: import.meta.url,
timeoutMs: TIMEOUT_MS,
});
});
});
}
}
});
98 changes: 53 additions & 45 deletions e2e/scenarios/claude-agent-sdk-instrumentation/scenario.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,57 +53,65 @@ const claudeAgentSDKScenarios = await Promise.all(
}),
);

describe("wrapped instrumentation", () => {
describe.concurrent("wrapped instrumentation", () => {
for (const scenario of claudeAgentSDKScenarios) {
const tags = cassetteTagsFor(import.meta.url, scenario.snapshotName);
describe(`claude agent sdk ${scenario.version}`, { tags }, () => {
defineClaudeAgentSDKInstrumentationAssertions({
assertLocalToolHandlerParenting: true,
expectTaskLifecycleDetails: scenario.expectTaskLifecycleDetails,
name: "scenario",
runScenario: async ({ runScenarioDir }) => {
await runScenarioDir({
entry: scenario.wrapperEntry,
runContext: {
variantKey: scenario.snapshotName,
originalScenarioDir,
},
scenarioDir,
timeoutMs: TIMEOUT_MS,
});
},
snapshotName: scenario.snapshotName,
testFileUrl: import.meta.url,
timeoutMs: TIMEOUT_MS,
});
});
describe.sequential(
`claude agent sdk ${scenario.version}`,
{ tags },
() => {
defineClaudeAgentSDKInstrumentationAssertions({
assertLocalToolHandlerParenting: true,
expectTaskLifecycleDetails: scenario.expectTaskLifecycleDetails,
name: "scenario",
runScenario: async ({ runScenarioDir }) => {
await runScenarioDir({
entry: scenario.wrapperEntry,
runContext: {
variantKey: scenario.snapshotName,
originalScenarioDir,
},
scenarioDir,
timeoutMs: TIMEOUT_MS,
});
},
snapshotName: scenario.snapshotName,
testFileUrl: import.meta.url,
timeoutMs: TIMEOUT_MS,
});
},
);
}
});

describe("auto-hook instrumentation", () => {
describe.concurrent("auto-hook instrumentation", () => {
for (const scenario of claudeAgentSDKScenarios) {
const tags = cassetteTagsFor(import.meta.url, scenario.snapshotName);
describe(`claude agent sdk ${scenario.version}`, { tags }, () => {
defineClaudeAgentSDKInstrumentationAssertions({
assertLocalToolHandlerParenting: true,
expectTaskLifecycleDetails: scenario.expectTaskLifecycleDetails,
name: "scenario",
runScenario: async ({ runNodeScenarioDir }) => {
await runNodeScenarioDir({
entry: scenario.autoEntry,
nodeArgs: ["--import", "braintrust/hook.mjs"],
runContext: {
variantKey: scenario.snapshotName,
originalScenarioDir,
},
scenarioDir,
timeoutMs: TIMEOUT_MS,
});
},
snapshotName: scenario.snapshotName,
testFileUrl: import.meta.url,
timeoutMs: TIMEOUT_MS,
});
});
describe.sequential(
`claude agent sdk ${scenario.version}`,
{ tags },
() => {
defineClaudeAgentSDKInstrumentationAssertions({
assertLocalToolHandlerParenting: true,
expectTaskLifecycleDetails: scenario.expectTaskLifecycleDetails,
name: "scenario",
runScenario: async ({ runNodeScenarioDir }) => {
await runNodeScenarioDir({
entry: scenario.autoEntry,
nodeArgs: ["--import", "braintrust/hook.mjs"],
runContext: {
variantKey: scenario.snapshotName,
originalScenarioDir,
},
scenarioDir,
timeoutMs: TIMEOUT_MS,
});
},
snapshotName: scenario.snapshotName,
testFileUrl: import.meta.url,
timeoutMs: TIMEOUT_MS,
});
},
);
}
});
Loading
Loading