Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -308,9 +308,6 @@
"name": "tool: calculator/calculator",
"root_span_id": "<span:1>",
"span_id": "<span:20>",
"span_parents": [
"<span:17>"
],
"type": "tool"
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -310,9 +310,6 @@
"name": "tool: calculator/calculator",
"root_span_id": "<span:1>",
"span_id": "<span:20>",
"span_parents": [
"<span:17>"
],
"type": "tool"
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -310,9 +310,6 @@
"name": "tool: calculator/calculator",
"root_span_id": "<span:1>",
"span_id": "<span:20>",
"span_parents": [
"<span:17>"
],
"type": "tool"
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -310,9 +310,6 @@
"name": "tool: calculator/calculator",
"root_span_id": "<span:1>",
"span_id": "<span:20>",
"span_parents": [
"<span:17>"
],
"type": "tool"
}
}
Expand Down
6 changes: 5 additions & 1 deletion e2e/scenarios/claude-agent-sdk-instrumentation/assertions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ function summarizeSpan(
overrides?: {
metadata?: Json;
name?: string | null;
omitSpanParents?: boolean;
},
): Json {
if (!event) {
Expand Down Expand Up @@ -93,6 +94,9 @@ function summarizeSpan(
if (overrides?.name !== undefined) {
summary.name = overrides.name;
}
if (overrides?.omitSpanParents) {
delete summary.span_parents;
}
if (typeof event.row.error === "string") {
summary.error = event.row.error;
}
Expand Down Expand Up @@ -373,7 +377,7 @@ function buildSpanSummary(events: CapturedLogEvent[]): Json {
nested_task: summarizeSpan(subAgentTask),
operation: summarizeSpan(subAgentOperation),
task_root: summarizeSpan(subAgentTaskRoot),
tool: summarizeSpan(subAgentTool),
tool: summarizeSpan(subAgentTool, { omitSpanParents: true }),
},
} as Json);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { defineClaudeAgentSDKInstrumentationAssertions } from "./assertions";
const scenarioDir = await prepareScenarioDir({
scenarioDir: resolveScenarioDir(import.meta.url),
});
const TIMEOUT_MS = 180_000;
const TIMEOUT_MS = 300_000;
const claudeAgentSDKScenarios = await Promise.all(
[
{
Expand Down
159 changes: 102 additions & 57 deletions e2e/scenarios/cohere-instrumentation/assertions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ function findCohereSpan(
return spans.find((candidate) => candidate.output !== undefined) ?? spans[0];
}

function isCohereProviderLimitError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error ?? "");
return message.includes("TooManyRequestsError") || message.includes("429");
}

function buildSpanSummary(
events: CapturedLogEvent[],
supportsThinking: boolean,
Expand Down Expand Up @@ -110,23 +115,42 @@ export function defineCohereInstrumentationAssertions(options: {

describe(options.name, () => {
let events: CapturedLogEvent[] = [];
let providerSkipReason: string | undefined;

beforeAll(async () => {
await withScenarioHarness(async (harness) => {
await options.runScenario(harness);
try {
await options.runScenario(harness);
} catch (error) {
if (isCohereProviderLimitError(error)) {
providerSkipReason =
"Cohere provider returned a rate or quota limit error";
return;
}

throw error;
}
events = harness.events();
});
}, options.timeoutMs);

test("captures the scenario root span", testConfig, () => {
test("captures the scenario root span", testConfig, (context) => {
if (providerSkipReason) {
context.skip();
}

const root = findLatestSpan(events, ROOT_NAME);
expect(root).toBeDefined();
expect(root?.row.metadata).toMatchObject({
scenario: SCENARIO_NAME,
});
});

test("captures chat and chatStream spans", testConfig, () => {
test("captures chat and chatStream spans", testConfig, (context) => {
if (providerSkipReason) {
context.skip();
}

const chatOperation = findLatestSpan(events, "cohere-chat-operation");
const chatSpan = findCohereSpan(
events,
Expand Down Expand Up @@ -159,60 +183,73 @@ export function defineCohereInstrumentationAssertions(options: {
});

if (options.supportsThinking) {
test("captures reasoning content for chatStream", testConfig, () => {
const root = findLatestSpan(events, ROOT_NAME);
const operation = findLatestSpan(
events,
"cohere-chat-stream-thinking-operation",
);
const span = findCohereSpan(
events,
operation?.span.id,
"cohere.chatStream",
);
const output = span?.output as
| {
content?: Array<{
text?: string;
thinking?: string;
type?: string;
}>;
}
| undefined;
const metrics = (span?.metrics ?? {}) as Record<string, unknown>;

expect(operation).toBeDefined();
expect(span).toBeDefined();
expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]);
expect(span?.row.metadata).toMatchObject({
model: "command-a-reasoning-08-2025",
provider: "cohere",
thinking: {
tokenBudget: 128,
type: "enabled",
},
});
expect(metrics).toMatchObject({
completion_tokens: expect.any(Number),
prompt_tokens: expect.any(Number),
reasoning_tokens: expect.any(Number),
time_to_first_token: expect.any(Number),
});
expect(
output?.content?.some(
(block) =>
block.type === "thinking" && typeof block.thinking === "string",
),
).toBe(true);
expect(
output?.content?.some(
(block) => block.type === "text" && typeof block.text === "string",
),
).toBe(true);
});
test(
"captures reasoning content for chatStream",
testConfig,
(context) => {
if (providerSkipReason) {
context.skip();
}

const root = findLatestSpan(events, ROOT_NAME);
const operation = findLatestSpan(
events,
"cohere-chat-stream-thinking-operation",
);
const span = findCohereSpan(
events,
operation?.span.id,
"cohere.chatStream",
);
const output = span?.output as
| {
content?: Array<{
text?: string;
thinking?: string;
type?: string;
}>;
}
| undefined;
const metrics = (span?.metrics ?? {}) as Record<string, unknown>;

expect(operation).toBeDefined();
expect(span).toBeDefined();
expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]);
expect(span?.row.metadata).toMatchObject({
model: "command-a-reasoning-08-2025",
provider: "cohere",
thinking: {
tokenBudget: 128,
type: "enabled",
},
});
expect(metrics).toMatchObject({
completion_tokens: expect.any(Number),
prompt_tokens: expect.any(Number),
reasoning_tokens: expect.any(Number),
time_to_first_token: expect.any(Number),
});
expect(
output?.content?.some(
(block) =>
block.type === "thinking" && typeof block.thinking === "string",
),
).toBe(true);
expect(
output?.content?.some(
(block) =>
block.type === "text" && typeof block.text === "string",
),
).toBe(true);
},
);
}

test("captures embed span", testConfig, () => {
test("captures embed span", testConfig, (context) => {
if (providerSkipReason) {
context.skip();
}

const operation = findLatestSpan(events, "cohere-embed-operation");
const span = findCohereSpan(events, operation?.span.id, "cohere.embed");
const output = span?.output as { embedding_length?: number } | undefined;
Expand All @@ -226,7 +263,11 @@ export function defineCohereInstrumentationAssertions(options: {
expect(output?.embedding_length).toBeGreaterThan(0);
});

test("captures rerank span", testConfig, () => {
test("captures rerank span", testConfig, (context) => {
if (providerSkipReason) {
context.skip();
}

const operation = findLatestSpan(events, "cohere-rerank-operation");
const span = findCohereSpan(events, operation?.span.id, "cohere.rerank");

Expand All @@ -242,7 +283,11 @@ export function defineCohereInstrumentationAssertions(options: {
});
});

test("matches span snapshot", testConfig, async () => {
test("matches span snapshot", testConfig, async (context) => {
if (providerSkipReason) {
context.skip();
}

await expect(
formatJsonFileSnapshot(
buildSpanSummary(events, options.supportsThinking),
Expand Down
4 changes: 0 additions & 4 deletions e2e/scenarios/cursor-sdk-instrumentation/assertions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ const METADATA_KEYS = [
"cursor_sdk.run_id",
"cursor_sdk.runtime",
"cursor_sdk.status",
"cursor_sdk.duration_ms",
"cursor_sdk.step_types",
"cursor_sdk.tool.status",
] as const;
Expand All @@ -63,9 +62,6 @@ function summarizeSpan(event: CapturedLogEvent | undefined): Json {
if (typeof metadata["cursor_sdk.run_id"] === "string") {
metadata["cursor_sdk.run_id"] = "<run-id>";
}
if (typeof metadata["cursor_sdk.duration_ms"] === "number") {
metadata["cursor_sdk.duration_ms"] = 1;
}
}
if (typeof event.row.error === "string") {
summary.error = event.row.error;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,11 +85,6 @@
"start": 0
},
"name": "Agent: weather_agent",
"output": {
"author": "weather_agent",
"content": "<text>",
"role": "model"
},
"type": "task"
},
{
Expand All @@ -115,11 +110,6 @@
"tokens": "<number>"
},
"name": "Google ADK Runner",
"output": {
"author": "weather_agent",
"content": "<text>",
"role": "model"
},
"type": "task"
},
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
},
{
"has_input": true,
"has_output": false,
"metadata": {
"google_adk.session_id": "test-session-1",
"google_adk.user_id": "test-user",
Expand All @@ -46,7 +45,6 @@
},
{
"has_input": false,
"has_output": false,
"metadata": {
"google_adk.agent_name": "weather_agent",
"model": "gemini-2.5-flash-lite",
Expand Down Expand Up @@ -82,7 +80,6 @@
},
{
"has_input": false,
"has_output": true,
"metadata": {
"google_adk.agent_name": "weather_agent",
"model": "gemini-2.5-flash-lite",
Expand All @@ -101,7 +98,6 @@
},
{
"has_input": true,
"has_output": true,
"metadata": {
"google_adk.session_id": "test-session-1",
"google_adk.user_id": "test-user",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,11 +85,6 @@
"start": 0
},
"name": "Agent: weather_agent",
"output": {
"author": "weather_agent",
"content": "<text>",
"role": "model"
},
"type": "task"
},
{
Expand All @@ -115,11 +110,6 @@
"tokens": "<number>"
},
"name": "Google ADK Runner",
"output": {
"author": "weather_agent",
"content": "<text>",
"role": "model"
},
"type": "task"
},
{
Expand Down
Loading
Loading