diff --git a/packages/gateway/src/gateway-language-model.test.ts b/packages/gateway/src/gateway-language-model.test.ts index 1cd8aa488345..657c24124100 100644 --- a/packages/gateway/src/gateway-language-model.test.ts +++ b/packages/gateway/src/gateway-language-model.test.ts @@ -1482,6 +1482,48 @@ describe('GatewayLanguageModel', () => { }); }); + it('should pass caching option for doGenerate', async () => { + prepareJsonResponse({ + content: { type: 'text', text: 'Test response' }, + }); + + await createTestModel().doGenerate({ + prompt: TEST_PROMPT, + providerOptions: { + gateway: { + caching: 'auto', + }, + }, + }); + + const requestBody = await server.calls[0].requestBodyJson; + expect(requestBody.providerOptions).toEqual({ + gateway: { caching: 'auto' }, + }); + }); + + it('should pass caching option for doStream', async () => { + prepareStreamResponse({ + content: ['Hello', ' world'], + }); + + const { stream } = await createTestModel().doStream({ + prompt: TEST_PROMPT, + providerOptions: { + gateway: { + caching: 'auto', + }, + }, + }); + + await convertReadableStreamToArray(stream); + + const requestBody = await server.calls[0].requestBodyJson; + expect(requestBody.providerOptions).toEqual({ + gateway: { caching: 'auto' }, + }); + }); + it('should pass providerTimeouts for doGenerate', async () => { prepareJsonResponse({ content: { type: 'text', text: 'Test response' }, diff --git a/packages/gateway/src/gateway-provider-options.ts b/packages/gateway/src/gateway-provider-options.ts index 5a808d505b7a..358a4f9ddda2 100644 --- a/packages/gateway/src/gateway-provider-options.ts +++ b/packages/gateway/src/gateway-provider-options.ts @@ -25,6 +25,12 @@ const gatewayProviderOptions = lazySchema(() => * - `'tps'`: highest tokens-per-second first */ sort: z.enum(['cost', 'ttft', 'tps']).optional(), + /** + * Automatically apply cache markers for providers that require them. + * + * Example: `'auto'` + */ + caching: z.literal('auto').optional(), /** * The unique identifier for the end user on behalf of whom the request was made. *