From 8a77d1ee6a2ed611a85e9fd50ededf638aa0e0fc Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 6 Apr 2026 22:05:22 +0000 Subject: [PATCH 1/7] feat(copilot-sdk): support BYOK (Bring Your Own Key) for copilot-sdk target Add optional `byok` block to copilot-sdk target config, allowing users to route requests through their own Azure OpenAI, OpenAI, or Anthropic endpoints instead of GitHub's Copilot infrastructure. Supported BYOK fields: type, base_url, api_key, bearer_token, api_version, wire_api. Secrets (api_key, bearer_token) require ${{ ENV_VAR }} syntax. Closes #955 Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/evaluation/providers/copilot-sdk.ts | 22 ++ .../core/src/evaluation/providers/targets.ts | 76 +++++++ .../core/src/evaluation/providers/types.ts | 2 + .../evaluation/providers/copilot-sdk.test.ts | 138 ++++++++++++ .../test/evaluation/providers/targets.test.ts | 196 ++++++++++++++++++ 5 files changed, 434 insertions(+) diff --git a/packages/core/src/evaluation/providers/copilot-sdk.ts b/packages/core/src/evaluation/providers/copilot-sdk.ts index 1cde066b2..6a6abc24b 100644 --- a/packages/core/src/evaluation/providers/copilot-sdk.ts +++ b/packages/core/src/evaluation/providers/copilot-sdk.ts @@ -115,6 +115,28 @@ export class CopilotSdkProvider implements Provider { }; } + // BYOK — pass a provider block to route requests through a user-provided endpoint + // instead of GitHub's Copilot infrastructure. See copilot-sdk docs/auth/byok.md. + if (this.config.byokBaseUrl) { + // biome-ignore lint/suspicious/noExplicitAny: SDK provider config shape is dynamic + const provider: any = { + type: this.config.byokType ?? 'openai', + baseUrl: this.config.byokBaseUrl, + }; + if (this.config.byokBearerToken) { + provider.bearerToken = this.config.byokBearerToken; + } else if (this.config.byokApiKey) { + provider.apiKey = this.config.byokApiKey; + } + if (this.config.byokWireApi) { + provider.wireApi = this.config.byokWireApi; + } + if (this.config.byokType === 'azure' && this.config.byokApiVersion) { + provider.azure = { apiVersion: this.config.byokApiVersion }; + } + sessionOptions.provider = provider; + } + // biome-ignore lint/suspicious/noExplicitAny: SDK session type is dynamically loaded let session: any; try { diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 10e2dd380..048d83298 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -481,6 +481,18 @@ export interface CopilotSdkResolvedConfig { readonly logDir?: string; readonly logFormat?: 'summary' | 'json'; readonly systemPrompt?: string; + /** BYOK provider type: "azure", "openai", or "anthropic". */ + readonly byokType?: string; + /** BYOK base URL for the provider endpoint. */ + readonly byokBaseUrl?: string; + /** BYOK API key for authenticating with the provider. */ + readonly byokApiKey?: string; + /** BYOK bearer token (takes precedence over apiKey when set). */ + readonly byokBearerToken?: string; + /** BYOK Azure API version (defaults to "2024-10-21"). */ + readonly byokApiVersion?: string; + /** BYOK wire API format: "completions" or "responses". */ + readonly byokWireApi?: string; } export interface CopilotLogResolvedConfig { @@ -1427,6 +1439,64 @@ function resolveCopilotSdkConfig( ? systemPromptSource.trim() : undefined; + // BYOK (Bring Your Own Key) — allows routing through a user-provided endpoint + // instead of GitHub's Copilot infrastructure. The byok block maps to the SDK's + // `provider` option on createSession(). See copilot-sdk docs/auth/byok.md. + const byok = target.byok as Record | undefined; + let byokType: string | undefined; + let byokBaseUrl: string | undefined; + let byokApiKey: string | undefined; + let byokBearerToken: string | undefined; + let byokApiVersion: string | undefined; + let byokWireApi: string | undefined; + + if (byok && typeof byok === 'object') { + byokType = resolveOptionalString(byok.type, env, `${target.name} byok type`, { + allowLiteral: true, + optionalEnv: true, + }); + + byokBaseUrl = resolveOptionalString(byok.base_url, env, `${target.name} byok base URL`, { + allowLiteral: true, + optionalEnv: true, + }); + + byokApiKey = resolveOptionalString(byok.api_key, env, `${target.name} byok API key`, { + allowLiteral: false, + optionalEnv: true, + }); + + byokBearerToken = resolveOptionalString( + byok.bearer_token, + env, + `${target.name} byok bearer token`, + { + allowLiteral: false, + optionalEnv: true, + }, + ); + + byokApiVersion = resolveOptionalString( + byok.api_version, + env, + `${target.name} byok API version`, + { + allowLiteral: true, + optionalEnv: true, + }, + ); + + byokWireApi = resolveOptionalString(byok.wire_api, env, `${target.name} byok wire API`, { + allowLiteral: true, + optionalEnv: true, + }); + + // base_url is required when byok is specified + if (!byokBaseUrl) { + throw new Error(`${target.name}: 'byok.base_url' is required when 'byok' is specified`); + } + } + return { cliUrl, cliPath, @@ -1438,6 +1508,12 @@ function resolveCopilotSdkConfig( logDir, logFormat, systemPrompt, + byokType, + byokBaseUrl, + byokApiKey, + byokBearerToken, + byokApiVersion, + byokWireApi, }; } diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index b24833643..f9ed86758 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -366,6 +366,8 @@ export interface TargetDefinition { readonly cli_url?: string | unknown | undefined; readonly cli_path?: string | unknown | undefined; readonly github_token?: string | unknown | undefined; + // Copilot SDK BYOK (Bring Your Own Key) — routes through a user-provided endpoint + readonly byok?: Record | undefined; // Retry configuration fields readonly max_retries?: number | unknown | undefined; readonly retry_initial_delay_ms?: number | unknown | undefined; diff --git a/packages/core/test/evaluation/providers/copilot-sdk.test.ts b/packages/core/test/evaluation/providers/copilot-sdk.test.ts index f70df4823..f81c2a00b 100644 --- a/packages/core/test/evaluation/providers/copilot-sdk.test.ts +++ b/packages/core/test/evaluation/providers/copilot-sdk.test.ts @@ -320,6 +320,144 @@ describe('CopilotSdkProvider', () => { expect(result.kind).toBe('approved'); }); + it('passes byok provider block to createSession for azure', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + const sdkMock = mockCopilotSdk(client); + + mock.module('@github/copilot-sdk', () => sdkMock); + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + model: 'gpt-4o', + byokType: 'azure', + byokBaseUrl: 'https://my-resource.openai.azure.com', + byokApiKey: 'azure-secret', + byokApiVersion: '2024-10-21', + }); + + await provider.invoke({ question: 'Test' }); + + const sessionOptions = client.createSession.mock.calls[0][0]; + expect(sessionOptions.provider).toBeDefined(); + expect(sessionOptions.provider.type).toBe('azure'); + expect(sessionOptions.provider.baseUrl).toBe('https://my-resource.openai.azure.com'); + expect(sessionOptions.provider.apiKey).toBe('azure-secret'); + expect(sessionOptions.provider.azure).toEqual({ apiVersion: '2024-10-21' }); + }); + + it('passes byok provider block with bearer token', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + const sdkMock = mockCopilotSdk(client); + + mock.module('@github/copilot-sdk', () => sdkMock); + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + byokType: 'openai', + byokBaseUrl: 'https://custom-endpoint.example.com/v1', + byokBearerToken: 'bearer-secret', + }); + + await provider.invoke({ question: 'Test' }); + + const sessionOptions = client.createSession.mock.calls[0][0]; + expect(sessionOptions.provider).toBeDefined(); + expect(sessionOptions.provider.bearerToken).toBe('bearer-secret'); + expect(sessionOptions.provider.apiKey).toBeUndefined(); + }); + + it('passes byok provider block with wireApi', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + const sdkMock = mockCopilotSdk(client); + + mock.module('@github/copilot-sdk', () => sdkMock); + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + byokType: 'openai', + byokBaseUrl: 'https://resource.openai.azure.com/openai/v1/', + byokApiKey: 'key', + byokWireApi: 'responses', + }); + + await provider.invoke({ question: 'Test' }); + + const sessionOptions = client.createSession.mock.calls[0][0]; + expect(sessionOptions.provider.wireApi).toBe('responses'); + }); + + it('does not set provider when byok is not configured', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + const sdkMock = mockCopilotSdk(client); + + mock.module('@github/copilot-sdk', () => sdkMock); + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + model: 'gpt-4o', + }); + + await provider.invoke({ question: 'Test' }); + + const sessionOptions = client.createSession.mock.calls[0][0]; + expect(sessionOptions.provider).toBeUndefined(); + }); + + it('defaults byok type to openai when not specified', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + const sdkMock = mockCopilotSdk(client); + + mock.module('@github/copilot-sdk', () => sdkMock); + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + byokBaseUrl: 'http://localhost:11434/v1', + }); + + await provider.invoke({ question: 'Test' }); + + const sessionOptions = client.createSession.mock.calls[0][0]; + expect(sessionOptions.provider.type).toBe('openai'); + }); + + it('does not set azure block for non-azure byok type', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + const sdkMock = mockCopilotSdk(client); + + mock.module('@github/copilot-sdk', () => sdkMock); + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + byokType: 'openai', + byokBaseUrl: 'https://api.openai.com/v1', + byokApiKey: 'key', + byokApiVersion: '2024-10-21', // should be ignored for non-azure + }); + + await provider.invoke({ question: 'Test' }); + + const sessionOptions = client.createSession.mock.calls[0][0]; + expect(sessionOptions.provider.azure).toBeUndefined(); + }); + it('includes timing information in response', async () => { const session = createMockSession({ events: [{ type: 'assistant.message', data: { content: 'response' } }], diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index aac8d357a..eb7de4076 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -750,6 +750,202 @@ describe('resolveTargetDefinition', () => { expect(target.config.executable).toBe('copilot'); }); + it('resolves copilot-sdk with byok azure config', () => { + const env = { + AZURE_OPENAI_ENDPOINT: 'https://my-resource.openai.azure.com', + AZURE_OPENAI_API_KEY: 'azure-secret', + AZURE_DEPLOYMENT_NAME: 'gpt-4o', + } satisfies Record; + + const target = resolveTargetDefinition( + { + name: 'copilot-sdk-azure', + provider: 'copilot-sdk', + model: '${{ AZURE_DEPLOYMENT_NAME }}', + byok: { + type: 'azure', + base_url: '${{ AZURE_OPENAI_ENDPOINT }}', + api_key: '${{ AZURE_OPENAI_API_KEY }}', + api_version: '2024-10-21', + }, + }, + env, + ); + + expect(target.kind).toBe('copilot-sdk'); + if (target.kind !== 'copilot-sdk') { + throw new Error('expected copilot-sdk target'); + } + + expect(target.config.model).toBe('gpt-4o'); + expect(target.config.byokType).toBe('azure'); + expect(target.config.byokBaseUrl).toBe('https://my-resource.openai.azure.com'); + expect(target.config.byokApiKey).toBe('azure-secret'); + expect(target.config.byokApiVersion).toBe('2024-10-21'); + }); + + it('resolves copilot-sdk with byok openai config', () => { + const env = { + OPENAI_API_KEY: 'openai-secret', + } satisfies Record; + + const target = resolveTargetDefinition( + { + name: 'copilot-sdk-openai', + provider: 'copilot-sdk', + model: 'gpt-5', + byok: { + type: 'openai', + base_url: 'https://api.openai.com/v1', + api_key: '${{ OPENAI_API_KEY }}', + }, + }, + env, + ); + + expect(target.kind).toBe('copilot-sdk'); + if (target.kind !== 'copilot-sdk') { + throw new Error('expected copilot-sdk target'); + } + + expect(target.config.byokType).toBe('openai'); + expect(target.config.byokBaseUrl).toBe('https://api.openai.com/v1'); + expect(target.config.byokApiKey).toBe('openai-secret'); + }); + + it('copilot-sdk byok defaults type to undefined when not specified', () => { + const env = { + MY_KEY: 'secret', + } satisfies Record; + + const target = resolveTargetDefinition( + { + name: 'copilot-sdk-byok-minimal', + provider: 'copilot-sdk', + byok: { + base_url: 'http://localhost:11434/v1', + api_key: '${{ MY_KEY }}', + }, + }, + env, + ); + + expect(target.kind).toBe('copilot-sdk'); + if (target.kind !== 'copilot-sdk') { + throw new Error('expected copilot-sdk target'); + } + + expect(target.config.byokType).toBeUndefined(); + expect(target.config.byokBaseUrl).toBe('http://localhost:11434/v1'); + expect(target.config.byokApiKey).toBe('secret'); + }); + + it('copilot-sdk byok rejects missing base_url', () => { + expect(() => + resolveTargetDefinition( + { + name: 'copilot-sdk-no-url', + provider: 'copilot-sdk', + byok: { + type: 'azure', + api_key: '${{ MY_KEY }}', + }, + }, + { MY_KEY: 'secret' }, + ), + ).toThrow(/byok\.base_url.*required/i); + }); + + it('copilot-sdk byok rejects literal api_key', () => { + expect(() => + resolveTargetDefinition( + { + name: 'copilot-sdk-literal-key', + provider: 'copilot-sdk', + byok: { + base_url: 'https://example.com', + api_key: 'plaintext-secret', + }, + }, + {}, + ), + ).toThrow(/must use.*VARIABLE_NAME/i); + }); + + it('copilot-sdk byok supports bearer_token', () => { + const env = { + MY_TOKEN: 'bearer-secret', + } satisfies Record; + + const target = resolveTargetDefinition( + { + name: 'copilot-sdk-bearer', + provider: 'copilot-sdk', + byok: { + base_url: 'https://custom-endpoint.example.com/v1', + bearer_token: '${{ MY_TOKEN }}', + }, + }, + env, + ); + + expect(target.kind).toBe('copilot-sdk'); + if (target.kind !== 'copilot-sdk') { + throw new Error('expected copilot-sdk target'); + } + + expect(target.config.byokBearerToken).toBe('bearer-secret'); + expect(target.config.byokApiKey).toBeUndefined(); + }); + + it('copilot-sdk byok supports wire_api', () => { + const env = { + FOUNDRY_KEY: 'foundry-secret', + } satisfies Record; + + const target = resolveTargetDefinition( + { + name: 'copilot-sdk-responses', + provider: 'copilot-sdk', + model: 'gpt-5', + byok: { + type: 'openai', + base_url: 'https://resource.openai.azure.com/openai/v1/', + api_key: '${{ FOUNDRY_KEY }}', + wire_api: 'responses', + }, + }, + env, + ); + + expect(target.kind).toBe('copilot-sdk'); + if (target.kind !== 'copilot-sdk') { + throw new Error('expected copilot-sdk target'); + } + + expect(target.config.byokWireApi).toBe('responses'); + }); + + it('copilot-sdk without byok has no byok fields', () => { + const target = resolveTargetDefinition( + { + name: 'copilot-sdk-plain', + provider: 'copilot-sdk', + model: 'gpt-4o', + }, + {}, + ); + + expect(target.kind).toBe('copilot-sdk'); + if (target.kind !== 'copilot-sdk') { + throw new Error('expected copilot-sdk target'); + } + + expect(target.config.byokType).toBeUndefined(); + expect(target.config.byokBaseUrl).toBeUndefined(); + expect(target.config.byokApiKey).toBeUndefined(); + }); + it('rejects removed target-level workspaceTemplate camelCase field', () => { expect(() => resolveTargetDefinition( From 00bc67cfae8496deb076b2a7a8fea926aba8c76b Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 6 Apr 2026 22:10:03 +0000 Subject: [PATCH 2/7] test(copilot-sdk): add literal bearer_token rejection test and fix JSDoc Co-Authored-By: Claude Opus 4.6 (1M context) --- .../core/src/evaluation/providers/targets.ts | 2 +- .../test/evaluation/providers/targets.test.ts | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 048d83298..a33183ce0 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -489,7 +489,7 @@ export interface CopilotSdkResolvedConfig { readonly byokApiKey?: string; /** BYOK bearer token (takes precedence over apiKey when set). */ readonly byokBearerToken?: string; - /** BYOK Azure API version (defaults to "2024-10-21"). */ + /** BYOK Azure API version (e.g. "2024-10-21"). Only used when byokType is "azure". */ readonly byokApiVersion?: string; /** BYOK wire API format: "completions" or "responses". */ readonly byokWireApi?: string; diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index eb7de4076..030c044c0 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -872,6 +872,22 @@ describe('resolveTargetDefinition', () => { ).toThrow(/must use.*VARIABLE_NAME/i); }); + it('copilot-sdk byok rejects literal bearer_token', () => { + expect(() => + resolveTargetDefinition( + { + name: 'copilot-sdk-literal-bearer', + provider: 'copilot-sdk', + byok: { + base_url: 'https://example.com', + bearer_token: 'plaintext-bearer-secret', + }, + }, + {}, + ), + ).toThrow(/must use.*VARIABLE_NAME/i); + }); + it('copilot-sdk byok supports bearer_token', () => { const env = { MY_TOKEN: 'bearer-secret', From 581e4bbce29d7c4a58c74c117b17da91f9b85007 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 6 Apr 2026 22:11:11 +0000 Subject: [PATCH 3/7] docs: clarify e2e verification must precede code review in AGENTS.md Make explicit that manual e2e verification is a prerequisite for the code review step, not a parallel activity. If e2e fails, fix it before investing time in review. Co-Authored-By: Claude Opus 4.6 (1M context) --- AGENTS.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 02bfdd492..b97b917ca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -110,7 +110,7 @@ cd ../agentv.worktrees/- - Subagents for: research, file exploration, running tests, code review. - For complex problems, throw more subagents at it — parallelize where possible. - Name subagents descriptively. -- Before declaring a repo change complete or opening/finalizing a PR, spawn a subagent for a final code review pass unless the user explicitly says not to. +- Before declaring a repo change complete or opening/finalizing a PR, complete manual e2e verification first (see E2E Checklist), **then** spawn a subagent for a final code review pass. E2E must pass before code review — if e2e fails, fix the issue before investing time in review. The user may explicitly skip the review step. ### Autonomous Bug Fixes - When you spot a bug, just fix it. Don't ask for hand-holding. @@ -382,12 +382,12 @@ When working on a GitHub issue, **ALWAYS** follow this workflow: ``` Push incremental commits to the draft PR as you work so progress is visible and recoverable. -6. **Before marking the PR ready for review or merging a low-risk change**, ensure: - - **E2E verification completed** (see "Completing Work — E2E Checklist") - - For CLI or other user-facing changes, run at least one manual end-to-end check of the real user flow, not just unit/integration tests. - - A final subagent code review pass has been run and any findings addressed or called out. - - CI pipeline passes (all checks green) - - No merge conflicts with `main` +6. **Before marking the PR ready for review or merging a low-risk change**, ensure (in this order): + 1. **E2E verification completed** (see "Completing Work — E2E Checklist") — this must pass first. + 2. For CLI or other user-facing changes, run at least one manual end-to-end check of the real user flow, not just unit/integration tests. + 3. **After e2e passes**, spawn a final subagent code review pass and address or call out any findings. Do NOT run the code review before e2e — if e2e fails you'll need to fix it first, which invalidates the review. + 4. CI pipeline passes (all checks green). + 5. No merge conflicts with `main`. 7. **Only after verification is complete**: - Mark the draft PR ready for review, or From 297336ef03efeecfe8a318da1b64179cd299ce15 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 6 Apr 2026 22:20:54 +0000 Subject: [PATCH 4/7] fix(copilot-sdk): add byok to known target validator settings Prevents false "Unknown setting" warning when validating targets with a byok block. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/core/src/evaluation/validation/targets-validator.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index f4cfb7626..f31ddc635 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -123,6 +123,7 @@ const COPILOT_SDK_SETTINGS = new Set([ 'log_format', 'system_prompt', 'workspace_template', + 'byok', ]); const COPILOT_CLI_SETTINGS = new Set([ From fc59d15bc0e86227be89faffd6a5fe30aaf0b8bf Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 6 Apr 2026 22:29:59 +0000 Subject: [PATCH 5/7] chore: add copilot-sdk-azure target and AZURE_OPENAI_BYOK_URL env var Co-Authored-By: Claude Opus 4.6 (1M context) --- .agentv/targets.yaml | 11 +++++++++++ .env.example | 2 ++ apps/cli/src/templates/.env.example | 2 ++ 3 files changed, 15 insertions(+) diff --git a/.agentv/targets.yaml b/.agentv/targets.yaml index e2adfce58..e2cf6a3ff 100644 --- a/.agentv/targets.yaml +++ b/.agentv/targets.yaml @@ -40,6 +40,17 @@ targets: grader_target: grader log_format: json + - name: copilot-sdk-azure + provider: copilot-sdk + model: ${{ AZURE_DEPLOYMENT_NAME }} + byok: + type: azure + base_url: ${{ AZURE_OPENAI_BYOK_URL }} + api_key: ${{ AZURE_OPENAI_API_KEY }} + grader_target: grader + timeout_seconds: 120 + log_format: json + - name: claude provider: claude-cli grader_target: grader diff --git a/.env.example b/.env.example index ee5de1976..0348aa622 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,8 @@ AZURE_OPENAI_API_FORMAT=chat # If omitted, AgentV defaults chat targets to 2024-12-01-preview. # Azure responses targets default to `v1` automatically. AZURE_OPENAI_API_VERSION=2024-12-01-preview +# Full URL for Copilot SDK BYOK (e.g. https://my-resource.openai.azure.com) +AZURE_OPENAI_BYOK_URL=https://your-resource.openai.azure.com # OpenAI OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/ diff --git a/apps/cli/src/templates/.env.example b/apps/cli/src/templates/.env.example index 1f8e22057..3a542b8f7 100644 --- a/apps/cli/src/templates/.env.example +++ b/apps/cli/src/templates/.env.example @@ -5,6 +5,8 @@ AZURE_OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/ AZURE_OPENAI_API_KEY=your-openai-api-key-here AZURE_DEPLOYMENT_NAME=gpt-5-mini AZURE_OPENAI_API_VERSION=2024-12-01-preview +# Full URL for Copilot SDK BYOK (e.g. https://my-resource.openai.azure.com) +AZURE_OPENAI_BYOK_URL=https://your-resource.openai.azure.com # OpenAI OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/ From 1e05762d56abcf380e8ad7501193592e2a125811 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 6 Apr 2026 22:48:24 +0000 Subject: [PATCH 6/7] feat(copilot-sdk): auto-normalize bare Azure resource names in BYOK base_url MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When byok.type is "azure" and base_url is a bare resource name (no https:// prefix), automatically construct the full URL as https://{name}.openai.azure.com. This lets users reuse AZURE_OPENAI_ENDPOINT directly without a separate env var. Also reverts the AZURE_OPENAI_BYOK_URL env var — no longer needed. Co-Authored-By: Claude Opus 4.6 (1M context) --- .agentv/targets.yaml | 2 +- .env.example | 2 - apps/cli/src/templates/.env.example | 2 - .../src/evaluation/providers/copilot-sdk.ts | 22 +++++++++- .../evaluation/providers/copilot-sdk.test.ts | 44 +++++++++++++++++++ 5 files changed, 65 insertions(+), 7 deletions(-) diff --git a/.agentv/targets.yaml b/.agentv/targets.yaml index e2cf6a3ff..bc5d7856b 100644 --- a/.agentv/targets.yaml +++ b/.agentv/targets.yaml @@ -45,7 +45,7 @@ targets: model: ${{ AZURE_DEPLOYMENT_NAME }} byok: type: azure - base_url: ${{ AZURE_OPENAI_BYOK_URL }} + base_url: ${{ AZURE_OPENAI_ENDPOINT }} api_key: ${{ AZURE_OPENAI_API_KEY }} grader_target: grader timeout_seconds: 120 diff --git a/.env.example b/.env.example index 0348aa622..ee5de1976 100644 --- a/.env.example +++ b/.env.example @@ -9,8 +9,6 @@ AZURE_OPENAI_API_FORMAT=chat # If omitted, AgentV defaults chat targets to 2024-12-01-preview. # Azure responses targets default to `v1` automatically. AZURE_OPENAI_API_VERSION=2024-12-01-preview -# Full URL for Copilot SDK BYOK (e.g. https://my-resource.openai.azure.com) -AZURE_OPENAI_BYOK_URL=https://your-resource.openai.azure.com # OpenAI OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/ diff --git a/apps/cli/src/templates/.env.example b/apps/cli/src/templates/.env.example index 3a542b8f7..1f8e22057 100644 --- a/apps/cli/src/templates/.env.example +++ b/apps/cli/src/templates/.env.example @@ -5,8 +5,6 @@ AZURE_OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/ AZURE_OPENAI_API_KEY=your-openai-api-key-here AZURE_DEPLOYMENT_NAME=gpt-5-mini AZURE_OPENAI_API_VERSION=2024-12-01-preview -# Full URL for Copilot SDK BYOK (e.g. https://my-resource.openai.azure.com) -AZURE_OPENAI_BYOK_URL=https://your-resource.openai.azure.com # OpenAI OPENAI_ENDPOINT=https://your-endpoint.openai.azure.com/ diff --git a/packages/core/src/evaluation/providers/copilot-sdk.ts b/packages/core/src/evaluation/providers/copilot-sdk.ts index 6a6abc24b..b18a10a65 100644 --- a/packages/core/src/evaluation/providers/copilot-sdk.ts +++ b/packages/core/src/evaluation/providers/copilot-sdk.ts @@ -118,10 +118,11 @@ export class CopilotSdkProvider implements Provider { // BYOK — pass a provider block to route requests through a user-provided endpoint // instead of GitHub's Copilot infrastructure. See copilot-sdk docs/auth/byok.md. if (this.config.byokBaseUrl) { + const byokType = this.config.byokType ?? 'openai'; // biome-ignore lint/suspicious/noExplicitAny: SDK provider config shape is dynamic const provider: any = { - type: this.config.byokType ?? 'openai', - baseUrl: this.config.byokBaseUrl, + type: byokType, + baseUrl: normalizeByokBaseUrl(this.config.byokBaseUrl, byokType), }; if (this.config.byokBearerToken) { provider.bearerToken = this.config.byokBearerToken; @@ -408,6 +409,23 @@ function resolveSkillDirectories(cwd: string): string[] { return candidates.filter((dir) => existsSync(dir)); } +/** + * Normalize a BYOK base URL for the Copilot SDK. + * For Azure type, if the value is a bare resource name (no https:// prefix), + * construct the full URL: https://{resourceName}.openai.azure.com + * This lets users reuse AZURE_OPENAI_ENDPOINT without a separate env var. + */ +function normalizeByokBaseUrl(baseUrl: string, type: string): string { + const trimmed = baseUrl.trim().replace(/\/+$/, ''); + if (/^https?:\/\//i.test(trimmed)) { + return trimmed; + } + if (type === 'azure') { + return `https://${trimmed}.openai.azure.com`; + } + return trimmed; +} + function summarizeSdkEvent(eventType: string, data: unknown): string | undefined { if (!data || typeof data !== 'object') { return eventType; diff --git a/packages/core/test/evaluation/providers/copilot-sdk.test.ts b/packages/core/test/evaluation/providers/copilot-sdk.test.ts index f81c2a00b..59eb3be13 100644 --- a/packages/core/test/evaluation/providers/copilot-sdk.test.ts +++ b/packages/core/test/evaluation/providers/copilot-sdk.test.ts @@ -348,6 +348,50 @@ describe('CopilotSdkProvider', () => { expect(sessionOptions.provider.azure).toEqual({ apiVersion: '2024-10-21' }); }); + it('normalizes bare azure resource name to full URL', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + const sdkMock = mockCopilotSdk(client); + + mock.module('@github/copilot-sdk', () => sdkMock); + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + byokType: 'azure', + byokBaseUrl: 'my-resource-eastus2', + byokApiKey: 'key', + }); + + await provider.invoke({ question: 'Test' }); + + const sessionOptions = client.createSession.mock.calls[0][0]; + expect(sessionOptions.provider.baseUrl).toBe('https://my-resource-eastus2.openai.azure.com'); + }); + + it('passes full URL through unchanged for azure byok', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + const sdkMock = mockCopilotSdk(client); + + mock.module('@github/copilot-sdk', () => sdkMock); + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + byokType: 'azure', + byokBaseUrl: 'https://my-resource.openai.azure.com', + byokApiKey: 'key', + }); + + await provider.invoke({ question: 'Test' }); + + const sessionOptions = client.createSession.mock.calls[0][0]; + expect(sessionOptions.provider.baseUrl).toBe('https://my-resource.openai.azure.com'); + }); + it('passes byok provider block with bearer token', async () => { const session = createMockSession({ events: [{ type: 'assistant.message', data: { content: 'response' } }], From 07b3712bb2a24764cc1015250e1c9375027d5d96 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 6 Apr 2026 22:50:44 +0000 Subject: [PATCH 7/7] chore: remove timeout_seconds from copilot-sdk-azure target Co-Authored-By: Claude Opus 4.6 (1M context) --- .agentv/targets.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.agentv/targets.yaml b/.agentv/targets.yaml index bc5d7856b..6e9c25a20 100644 --- a/.agentv/targets.yaml +++ b/.agentv/targets.yaml @@ -48,7 +48,6 @@ targets: base_url: ${{ AZURE_OPENAI_ENDPOINT }} api_key: ${{ AZURE_OPENAI_API_KEY }} grader_target: grader - timeout_seconds: 120 log_format: json - name: claude