From 026a595a8606244add96cf1ce9559c33ae132806 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Thu, 7 May 2026 09:44:27 -0700 Subject: [PATCH 1/5] fix(e2e): add COPILOT_API_KEY to canary Docker allowlist The canary Docker runner forwards secrets to the container via an explicit ALLOWED_ENV_KEYS allowlist. COPILOT_API_KEY was missing from that list, so the GitHub Copilot scenario always failed inside Docker even when the key was set in the GHA job env. Co-Authored-By: Claude Sonnet 4.6 --- e2e/scripts/run-canary-tests-docker.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/scripts/run-canary-tests-docker.mjs b/e2e/scripts/run-canary-tests-docker.mjs index 9b75bb175..2ca46b70a 100644 --- a/e2e/scripts/run-canary-tests-docker.mjs +++ b/e2e/scripts/run-canary-tests-docker.mjs @@ -19,6 +19,7 @@ const ALLOWED_ENV_KEYS = [ "GEMINI_API_KEY", "GOOGLE_API_KEY", "COHERE_API_KEY", + "COPILOT_API_KEY", "CURSOR_API_KEY", "GROQ_API_KEY", "OPENAI_API_KEY", From 109c1507720832d83d0ffb2fb39f9f731e4d7744 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Thu, 7 May 2026 10:31:23 -0700 Subject: [PATCH 2/5] fix(turbo): add COPILOT_API_KEY to globalPassThroughEnv and test:e2e env Turbo v2 strict mode (activated by globalPassThroughEnv being defined) strips any env var not explicitly listed. COPILOT_API_KEY was absent from both globalPassThroughEnv and the test:e2e task env list, causing it to be unavailable to the Vitest process and all subprocesses it spawns. Co-Authored-By: Claude Sonnet 4.6 --- turbo.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/turbo.json b/turbo.json index bf4406360..3aaf5b3ed 100644 --- a/turbo.json +++ b/turbo.json @@ -6,6 +6,7 @@ "ANTHROPIC_API_KEY", "GEMINI_API_KEY", "COHERE_API_KEY", + "COPILOT_API_KEY", "CURSOR_API_KEY", "GROQ_API_KEY", "OPENROUTER_API_KEY", @@ -49,6 +50,7 @@ "BRAINTRUST_E2E_RUN_CONTEXT_DIR", "GEMINI_API_KEY", "COHERE_API_KEY", + "COPILOT_API_KEY", "CURSOR_API_KEY", "GROQ_API_KEY", "OPENAI_API_KEY", From e0ebe6cad582fff768505ba5ea802eaf31600850 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Thu, 7 May 2026 10:33:34 -0700 Subject: [PATCH 3/5] fix(turbo): add COPILOT_API_KEY to all provider key env lists Co-Authored-By: Claude Sonnet 4.6 --- turbo.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/turbo.json b/turbo.json index 3aaf5b3ed..eccf031d3 100644 --- a/turbo.json +++ b/turbo.json @@ -29,6 +29,7 @@ "BRAINTRUST_API_KEY", "GEMINI_API_KEY", "COHERE_API_KEY", + "COPILOT_API_KEY", "CURSOR_API_KEY", "GROQ_API_KEY", "OPENAI_API_KEY", @@ -76,6 +77,7 @@ "BRAINTRUST_E2E_RUN_CONTEXT_DIR", "GEMINI_API_KEY", "COHERE_API_KEY", + "COPILOT_API_KEY", "CURSOR_API_KEY", "GROQ_API_KEY", "OPENAI_API_KEY", @@ -96,6 +98,7 @@ "BRAINTRUST_E2E_RUN_CONTEXT_DIR", "GEMINI_API_KEY", "COHERE_API_KEY", + "COPILOT_API_KEY", "CURSOR_API_KEY", "GROQ_API_KEY", "OPENAI_API_KEY", @@ -117,6 +120,7 @@ "BRAINTRUST_E2E_RUN_CONTEXT_DIR", "GEMINI_API_KEY", "COHERE_API_KEY", + "COPILOT_API_KEY", "GROQ_API_KEY", "OPENAI_API_KEY", "OPENAI_BASE_URL", From d64eb624c6c74d1d67c85b0534fe16f26d442a5a Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Thu, 7 May 2026 10:35:37 -0700 Subject: [PATCH 4/5] fix(turbo): add COPILOT_API_KEY to playground tasks, CURSOR_API_KEY to test:e2e:record Co-Authored-By: Claude Sonnet 4.6 --- turbo.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/turbo.json b/turbo.json index eccf031d3..013aa9fa7 100644 --- a/turbo.json +++ b/turbo.json @@ -121,6 +121,7 @@ "GEMINI_API_KEY", "COHERE_API_KEY", "COPILOT_API_KEY", + "CURSOR_API_KEY", "GROQ_API_KEY", "OPENAI_API_KEY", "OPENAI_BASE_URL", @@ -154,6 +155,7 @@ "BRAINTRUST_API_KEY", "GEMINI_API_KEY", "COHERE_API_KEY", + "COPILOT_API_KEY", "CURSOR_API_KEY", "GROQ_API_KEY", "OPENAI_API_KEY", @@ -170,6 +172,7 @@ "BRAINTRUST_API_KEY", "GEMINI_API_KEY", "COHERE_API_KEY", + "COPILOT_API_KEY", "CURSOR_API_KEY", "GROQ_API_KEY", "OPENAI_API_KEY", @@ -186,6 +189,7 @@ "BRAINTRUST_API_KEY", "GEMINI_API_KEY", "COHERE_API_KEY", + "COPILOT_API_KEY", "CURSOR_API_KEY", "GROQ_API_KEY", "OPENAI_API_KEY", From cd94b9b18a14ec31aa1223918dbaca07cf70fa35 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Thu, 7 May 2026 16:28:45 -0700 Subject: [PATCH 5/5] fix(e2e): skip file snapshot comparisons in canary mode Canary tests make real API calls against the latest provider SDK versions. File snapshots were recorded with cassette replay, so real API responses produce output that diverges from the stored snapshots. Co-Authored-By: Claude Sonnet 4.6 --- .../huggingface-instrumentation/assertions.ts | 7 ++-- .../openai-instrumentation/assertions.ts | 33 ++++++++++++------- .../wrap-langchain-js-traces/scenario.test.ts | 23 +++++++------ 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/e2e/scenarios/huggingface-instrumentation/assertions.ts b/e2e/scenarios/huggingface-instrumentation/assertions.ts index 991ba108a..3241337cd 100644 --- a/e2e/scenarios/huggingface-instrumentation/assertions.ts +++ b/e2e/scenarios/huggingface-instrumentation/assertions.ts @@ -5,7 +5,10 @@ import { formatJsonFileSnapshot, resolveFileSnapshotPath, } from "../../helpers/file-snapshot"; -import { withScenarioHarness } from "../../helpers/scenario-harness"; +import { + isCanaryMode, + withScenarioHarness, +} from "../../helpers/scenario-harness"; import { findLatestChildSpan, findLatestSpan, @@ -401,7 +404,7 @@ export function defineHuggingFaceInstrumentationAssertions(options: { }, ); - test( + test.skipIf(isCanaryMode())( "matches the log payload snapshot", { timeout: options.timeoutMs }, async ({ expect }) => { diff --git a/e2e/scenarios/openai-instrumentation/assertions.ts b/e2e/scenarios/openai-instrumentation/assertions.ts index 967d5e800..97f820e79 100644 --- a/e2e/scenarios/openai-instrumentation/assertions.ts +++ b/e2e/scenarios/openai-instrumentation/assertions.ts @@ -8,7 +8,10 @@ import { formatJsonFileSnapshot, resolveFileSnapshotPath, } from "../../helpers/file-snapshot"; -import { withScenarioHarness } from "../../helpers/scenario-harness"; +import { + isCanaryMode, + withScenarioHarness, +} from "../../helpers/scenario-harness"; import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; import { ROOT_NAME, SCENARIO_NAME } from "./scenario.impl.mjs"; @@ -665,16 +668,24 @@ export function defineOpenAIInstrumentationAssertions(options: { }); } - test("matches the shared span snapshot", testConfig, async () => { - await expect( - formatJsonFileSnapshot(buildSpanSummary(events, operationSpecs)), - ).toMatchFileSnapshot(spanSnapshotPath); - }); + test.skipIf(isCanaryMode())( + "matches the shared span snapshot", + testConfig, + async () => { + await expect( + formatJsonFileSnapshot(buildSpanSummary(events, operationSpecs)), + ).toMatchFileSnapshot(spanSnapshotPath); + }, + ); - test("matches the shared payload snapshot", testConfig, async () => { - await expect( - formatJsonFileSnapshot(buildPayloadSummary(events, operationSpecs)), - ).toMatchFileSnapshot(payloadSnapshotPath); - }); + test.skipIf(isCanaryMode())( + "matches the shared payload snapshot", + testConfig, + async () => { + await expect( + formatJsonFileSnapshot(buildPayloadSummary(events, operationSpecs)), + ).toMatchFileSnapshot(payloadSnapshotPath); + }, + ); }); } diff --git a/e2e/scenarios/wrap-langchain-js-traces/scenario.test.ts b/e2e/scenarios/wrap-langchain-js-traces/scenario.test.ts index 7725c479c..77fb12c11 100644 --- a/e2e/scenarios/wrap-langchain-js-traces/scenario.test.ts +++ b/e2e/scenarios/wrap-langchain-js-traces/scenario.test.ts @@ -4,6 +4,7 @@ import { resolveFileSnapshotPath, } from "../../helpers/file-snapshot"; import { + isCanaryMode, prepareScenarioDir, resolveScenarioDir, withScenarioHarness, @@ -43,16 +44,18 @@ test( scenarioName: "wrap-langchain-js-traces", }); - await expect( - formatJsonFileSnapshot(summaries.spanSummary), - ).toMatchFileSnapshot( - resolveFileSnapshotPath(import.meta.url, "span-events.json"), - ); - await expect( - formatJsonFileSnapshot(summaries.payloadSummary), - ).toMatchFileSnapshot( - resolveFileSnapshotPath(import.meta.url, "log-payloads.json"), - ); + if (!isCanaryMode()) { + await expect( + formatJsonFileSnapshot(summaries.spanSummary), + ).toMatchFileSnapshot( + resolveFileSnapshotPath(import.meta.url, "span-events.json"), + ); + await expect( + formatJsonFileSnapshot(summaries.payloadSummary), + ).toMatchFileSnapshot( + resolveFileSnapshotPath(import.meta.url, "log-payloads.json"), + ); + } }); }, );