|
| 1 | +/** |
| 2 | + * @vitest-environment node |
| 3 | + */ |
| 4 | + |
| 5 | +import type { ChatCompletionChunk } from 'openai/resources/chat/completions' |
| 6 | +import { beforeEach, describe, expect, it, vi } from 'vitest' |
| 7 | + |
| 8 | +const { mockCreate, mockExecuteProviderTool } = vi.hoisted(() => ({ |
| 9 | + mockCreate: vi.fn(), |
| 10 | + mockExecuteProviderTool: vi.fn(), |
| 11 | +})) |
| 12 | + |
| 13 | +vi.mock('openai', () => ({ |
| 14 | + default: vi.fn().mockImplementation( |
| 15 | + class { |
| 16 | + chat = { completions: { create: mockCreate } } |
| 17 | + } |
| 18 | + ), |
| 19 | +})) |
| 20 | + |
| 21 | +vi.mock('@/providers', () => ({ MAX_TOOL_ITERATIONS: 10 })) |
| 22 | + |
| 23 | +vi.mock('@/providers/runtime-context', () => ({ |
| 24 | + executeProviderTool: mockExecuteProviderTool, |
| 25 | +})) |
| 26 | + |
| 27 | +import type { StreamingExecution } from '@/executor/types' |
| 28 | +import type { ProviderRequest, ProviderResponse, ProviderToolConfig } from '@/providers/types' |
| 29 | +import { xAIProvider } from '@/providers/xai' |
| 30 | + |
| 31 | +interface XAITestUsage { |
| 32 | + prompt_tokens: number |
| 33 | + completion_tokens: number |
| 34 | + total_tokens: number |
| 35 | + prompt_tokens_details?: { cached_tokens: number } |
| 36 | + completion_tokens_details?: { reasoning_tokens: number } |
| 37 | + cost_in_usd_ticks?: number |
| 38 | +} |
| 39 | + |
| 40 | +function textResponse( |
| 41 | + content: string, |
| 42 | + usage: XAITestUsage, |
| 43 | + serviceTier: 'default' | 'priority' = 'default' |
| 44 | +) { |
| 45 | + return { |
| 46 | + choices: [{ message: { content, tool_calls: undefined }, finish_reason: 'stop' }], |
| 47 | + usage, |
| 48 | + service_tier: serviceTier, |
| 49 | + } |
| 50 | +} |
| 51 | + |
| 52 | +function toolCallResponse(name: string, args: Record<string, unknown>, usage: XAITestUsage) { |
| 53 | + return { |
| 54 | + choices: [ |
| 55 | + { |
| 56 | + message: { |
| 57 | + content: null, |
| 58 | + tool_calls: [ |
| 59 | + { |
| 60 | + id: 'call_1', |
| 61 | + type: 'function', |
| 62 | + function: { name, arguments: JSON.stringify(args) }, |
| 63 | + }, |
| 64 | + ], |
| 65 | + }, |
| 66 | + finish_reason: 'tool_calls', |
| 67 | + }, |
| 68 | + ], |
| 69 | + usage, |
| 70 | + service_tier: 'default', |
| 71 | + } |
| 72 | +} |
| 73 | + |
| 74 | +function tool(id: string): ProviderToolConfig { |
| 75 | + return { |
| 76 | + id, |
| 77 | + name: id, |
| 78 | + description: 'test tool', |
| 79 | + params: {}, |
| 80 | + parameters: { type: 'object', properties: {}, required: [] }, |
| 81 | + } |
| 82 | +} |
| 83 | + |
| 84 | +async function drainStream(stream: ReadableStream<unknown>): Promise<void> { |
| 85 | + const reader = stream.getReader() |
| 86 | + while (!(await reader.read()).done) {} |
| 87 | +} |
| 88 | + |
| 89 | +const baseRequest: ProviderRequest = { |
| 90 | + apiKey: 'xai-test-key', |
| 91 | + model: 'grok-4.5', |
| 92 | + messages: [{ role: 'user', content: 'Hello' }], |
| 93 | +} |
| 94 | + |
| 95 | +describe('xAIProvider usage accounting', () => { |
| 96 | + beforeEach(() => { |
| 97 | + vi.clearAllMocks() |
| 98 | + mockCreate.mockReset() |
| 99 | + mockExecuteProviderTool.mockReset() |
| 100 | + }) |
| 101 | + |
| 102 | + it('returns exact cost and detailed normalized tokens for nonstreaming requests', async () => { |
| 103 | + mockCreate.mockResolvedValueOnce( |
| 104 | + textResponse( |
| 105 | + 'Hello back', |
| 106 | + { |
| 107 | + prompt_tokens: 32, |
| 108 | + completion_tokens: 9, |
| 109 | + total_tokens: 135, |
| 110 | + prompt_tokens_details: { cached_tokens: 6 }, |
| 111 | + completion_tokens_details: { reasoning_tokens: 94 }, |
| 112 | + cost_in_usd_ticks: 12_345_678, |
| 113 | + }, |
| 114 | + 'priority' |
| 115 | + ) |
| 116 | + ) |
| 117 | + |
| 118 | + const response = (await xAIProvider.executeRequest(baseRequest)) as ProviderResponse |
| 119 | + |
| 120 | + expect(response.tokens).toEqual({ |
| 121 | + input: 26, |
| 122 | + output: 103, |
| 123 | + total: 135, |
| 124 | + cacheRead: 6, |
| 125 | + reasoning: 94, |
| 126 | + }) |
| 127 | + expect(response.cost?.total).toBe(0.0012345678) |
| 128 | + expect((response.cost?.input ?? 0) + (response.cost?.output ?? 0)).toBeCloseTo(0.0012345678, 10) |
| 129 | + expect(response.timing?.timeSegments?.[0]).toMatchObject({ |
| 130 | + tokens: response.tokens, |
| 131 | + cost: { total: 0.0012345678 }, |
| 132 | + provider: 'xai', |
| 133 | + }) |
| 134 | + }) |
| 135 | + |
| 136 | + it('accumulates exact cost and detailed tokens across tool-loop turns', async () => { |
| 137 | + mockCreate |
| 138 | + .mockResolvedValueOnce( |
| 139 | + toolCallResponse( |
| 140 | + 'lookup', |
| 141 | + { id: 7 }, |
| 142 | + { |
| 143 | + prompt_tokens: 10, |
| 144 | + completion_tokens: 3, |
| 145 | + total_tokens: 14, |
| 146 | + prompt_tokens_details: { cached_tokens: 2 }, |
| 147 | + completion_tokens_details: { reasoning_tokens: 1 }, |
| 148 | + cost_in_usd_ticks: 1_000_000, |
| 149 | + } |
| 150 | + ) |
| 151 | + ) |
| 152 | + .mockResolvedValueOnce( |
| 153 | + textResponse('Found it', { |
| 154 | + prompt_tokens: 20, |
| 155 | + completion_tokens: 4, |
| 156 | + total_tokens: 26, |
| 157 | + prompt_tokens_details: { cached_tokens: 5 }, |
| 158 | + completion_tokens_details: { reasoning_tokens: 2 }, |
| 159 | + cost_in_usd_ticks: 2_000_000, |
| 160 | + }) |
| 161 | + ) |
| 162 | + mockExecuteProviderTool.mockResolvedValueOnce({ success: true, output: { value: 42 } }) |
| 163 | + |
| 164 | + const response = (await xAIProvider.executeRequest({ |
| 165 | + ...baseRequest, |
| 166 | + tools: [tool('lookup')], |
| 167 | + })) as ProviderResponse |
| 168 | + |
| 169 | + expect(mockExecuteProviderTool).toHaveBeenCalledWith( |
| 170 | + 'lookup', |
| 171 | + expect.objectContaining({ id: 7 }), |
| 172 | + expect.anything() |
| 173 | + ) |
| 174 | + expect(response.content).toBe('Found it') |
| 175 | + expect(response.tokens).toEqual({ |
| 176 | + input: 23, |
| 177 | + output: 10, |
| 178 | + total: 40, |
| 179 | + cacheRead: 7, |
| 180 | + reasoning: 3, |
| 181 | + }) |
| 182 | + expect(response.cost?.total).toBe(0.0003) |
| 183 | + expect(response.toolResults).toEqual([{ value: 42 }]) |
| 184 | + expect( |
| 185 | + response.timing?.timeSegments?.filter((segment) => segment.type === 'model') |
| 186 | + ).toHaveLength(2) |
| 187 | + for (const segment of response.timing?.timeSegments?.filter( |
| 188 | + (candidate) => candidate.type === 'model' |
| 189 | + ) ?? []) { |
| 190 | + expect(segment.tokens).toBeDefined() |
| 191 | + expect(segment.cost?.total).toBeGreaterThan(0) |
| 192 | + } |
| 193 | + }) |
| 194 | + |
| 195 | + it('settles exact cost and detailed tokens after draining a direct stream', async () => { |
| 196 | + const usage: XAITestUsage = { |
| 197 | + prompt_tokens: 32, |
| 198 | + completion_tokens: 9, |
| 199 | + total_tokens: 135, |
| 200 | + prompt_tokens_details: { cached_tokens: 6 }, |
| 201 | + completion_tokens_details: { reasoning_tokens: 94 }, |
| 202 | + cost_in_usd_ticks: 12_345_678, |
| 203 | + } |
| 204 | + const chunks = (async function* (): AsyncGenerator<ChatCompletionChunk> { |
| 205 | + yield { |
| 206 | + id: 'xai-1', |
| 207 | + choices: [{ index: 0, delta: { content: 'Streamed' }, finish_reason: null }], |
| 208 | + created: 0, |
| 209 | + model: 'grok-4.5', |
| 210 | + object: 'chat.completion.chunk', |
| 211 | + } |
| 212 | + yield { |
| 213 | + id: 'xai-1', |
| 214 | + choices: [], |
| 215 | + created: 0, |
| 216 | + model: 'grok-4.5', |
| 217 | + object: 'chat.completion.chunk', |
| 218 | + usage, |
| 219 | + service_tier: 'priority', |
| 220 | + } as unknown as ChatCompletionChunk |
| 221 | + })() |
| 222 | + mockCreate.mockResolvedValueOnce(chunks) |
| 223 | + |
| 224 | + const result = (await xAIProvider.executeRequest({ |
| 225 | + ...baseRequest, |
| 226 | + stream: true, |
| 227 | + })) as StreamingExecution |
| 228 | + await drainStream(result.stream) |
| 229 | + |
| 230 | + expect(result.execution.output.content).toBe('Streamed') |
| 231 | + expect(result.execution.output.tokens).toEqual({ |
| 232 | + input: 26, |
| 233 | + output: 103, |
| 234 | + total: 135, |
| 235 | + cacheRead: 6, |
| 236 | + reasoning: 94, |
| 237 | + }) |
| 238 | + expect(result.execution.output.cost?.total).toBe(0.0012345678) |
| 239 | + expect(result.execution.output.providerTiming?.timeSegments?.[0]).toMatchObject({ |
| 240 | + assistantContent: 'Streamed', |
| 241 | + tokens: result.execution.output.tokens, |
| 242 | + cost: { total: 0.0012345678 }, |
| 243 | + provider: 'xai', |
| 244 | + }) |
| 245 | + expect(mockCreate.mock.calls[0][0]).toMatchObject({ |
| 246 | + stream: true, |
| 247 | + stream_options: { include_usage: true }, |
| 248 | + }) |
| 249 | + }) |
| 250 | +}) |
0 commit comments