diff --git a/.agentv/targets.yaml b/.agentv/targets.yaml index e2adfce58..6e9c25a20 100644 --- a/.agentv/targets.yaml +++ b/.agentv/targets.yaml @@ -40,6 +40,16 @@ targets: grader_target: grader log_format: json + - name: copilot-sdk-azure + provider: copilot-sdk + model: ${{ AZURE_DEPLOYMENT_NAME }} + byok: + type: azure + base_url: ${{ AZURE_OPENAI_ENDPOINT }} + api_key: ${{ AZURE_OPENAI_API_KEY }} + grader_target: grader + log_format: json + - name: claude provider: claude-cli grader_target: grader diff --git a/AGENTS.md b/AGENTS.md index 02bfdd492..b97b917ca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -110,7 +110,7 @@ cd ../agentv.worktrees/- - Subagents for: research, file exploration, running tests, code review. - For complex problems, throw more subagents at it — parallelize where possible. - Name subagents descriptively. -- Before declaring a repo change complete or opening/finalizing a PR, spawn a subagent for a final code review pass unless the user explicitly says not to. +- Before declaring a repo change complete or opening/finalizing a PR, complete manual e2e verification first (see E2E Checklist), **then** spawn a subagent for a final code review pass. E2E must pass before code review — if e2e fails, fix the issue before investing time in review. The user may explicitly skip the review step. ### Autonomous Bug Fixes - When you spot a bug, just fix it. Don't ask for hand-holding. @@ -382,12 +382,12 @@ When working on a GitHub issue, **ALWAYS** follow this workflow: ``` Push incremental commits to the draft PR as you work so progress is visible and recoverable. -6. **Before marking the PR ready for review or merging a low-risk change**, ensure: - - **E2E verification completed** (see "Completing Work — E2E Checklist") - - For CLI or other user-facing changes, run at least one manual end-to-end check of the real user flow, not just unit/integration tests. - - A final subagent code review pass has been run and any findings addressed or called out. - - CI pipeline passes (all checks green) - - No merge conflicts with `main` +6. **Before marking the PR ready for review or merging a low-risk change**, ensure (in this order): + 1. **E2E verification completed** (see "Completing Work — E2E Checklist") — this must pass first. + 2. For CLI or other user-facing changes, run at least one manual end-to-end check of the real user flow, not just unit/integration tests. + 3. **After e2e passes**, spawn a final subagent code review pass and address or call out any findings. Do NOT run the code review before e2e — if e2e fails you'll need to fix it first, which invalidates the review. + 4. CI pipeline passes (all checks green). + 5. No merge conflicts with `main`. 7. **Only after verification is complete**: - Mark the draft PR ready for review, or diff --git a/packages/core/src/evaluation/providers/copilot-sdk.ts b/packages/core/src/evaluation/providers/copilot-sdk.ts index 1cde066b2..b18a10a65 100644 --- a/packages/core/src/evaluation/providers/copilot-sdk.ts +++ b/packages/core/src/evaluation/providers/copilot-sdk.ts @@ -115,6 +115,29 @@ export class CopilotSdkProvider implements Provider { }; } + // BYOK — pass a provider block to route requests through a user-provided endpoint + // instead of GitHub's Copilot infrastructure. See copilot-sdk docs/auth/byok.md. + if (this.config.byokBaseUrl) { + const byokType = this.config.byokType ?? 'openai'; + // biome-ignore lint/suspicious/noExplicitAny: SDK provider config shape is dynamic + const provider: any = { + type: byokType, + baseUrl: normalizeByokBaseUrl(this.config.byokBaseUrl, byokType), + }; + if (this.config.byokBearerToken) { + provider.bearerToken = this.config.byokBearerToken; + } else if (this.config.byokApiKey) { + provider.apiKey = this.config.byokApiKey; + } + if (this.config.byokWireApi) { + provider.wireApi = this.config.byokWireApi; + } + if (this.config.byokType === 'azure' && this.config.byokApiVersion) { + provider.azure = { apiVersion: this.config.byokApiVersion }; + } + sessionOptions.provider = provider; + } + // biome-ignore lint/suspicious/noExplicitAny: SDK session type is dynamically loaded let session: any; try { @@ -386,6 +409,23 @@ function resolveSkillDirectories(cwd: string): string[] { return candidates.filter((dir) => existsSync(dir)); } +/** + * Normalize a BYOK base URL for the Copilot SDK. + * For Azure type, if the value is a bare resource name (no https:// prefix), + * construct the full URL: https://{resourceName}.openai.azure.com + * This lets users reuse AZURE_OPENAI_ENDPOINT without a separate env var. + */ +function normalizeByokBaseUrl(baseUrl: string, type: string): string { + const trimmed = baseUrl.trim().replace(/\/+$/, ''); + if (/^https?:\/\//i.test(trimmed)) { + return trimmed; + } + if (type === 'azure') { + return `https://${trimmed}.openai.azure.com`; + } + return trimmed; +} + function summarizeSdkEvent(eventType: string, data: unknown): string | undefined { if (!data || typeof data !== 'object') { return eventType; diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 10e2dd380..a33183ce0 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -481,6 +481,18 @@ export interface CopilotSdkResolvedConfig { readonly logDir?: string; readonly logFormat?: 'summary' | 'json'; readonly systemPrompt?: string; + /** BYOK provider type: "azure", "openai", or "anthropic". */ + readonly byokType?: string; + /** BYOK base URL for the provider endpoint. */ + readonly byokBaseUrl?: string; + /** BYOK API key for authenticating with the provider. */ + readonly byokApiKey?: string; + /** BYOK bearer token (takes precedence over apiKey when set). */ + readonly byokBearerToken?: string; + /** BYOK Azure API version (e.g. "2024-10-21"). Only used when byokType is "azure". */ + readonly byokApiVersion?: string; + /** BYOK wire API format: "completions" or "responses". */ + readonly byokWireApi?: string; } export interface CopilotLogResolvedConfig { @@ -1427,6 +1439,64 @@ function resolveCopilotSdkConfig( ? systemPromptSource.trim() : undefined; + // BYOK (Bring Your Own Key) — allows routing through a user-provided endpoint + // instead of GitHub's Copilot infrastructure. The byok block maps to the SDK's + // `provider` option on createSession(). See copilot-sdk docs/auth/byok.md. + const byok = target.byok as Record | undefined; + let byokType: string | undefined; + let byokBaseUrl: string | undefined; + let byokApiKey: string | undefined; + let byokBearerToken: string | undefined; + let byokApiVersion: string | undefined; + let byokWireApi: string | undefined; + + if (byok && typeof byok === 'object') { + byokType = resolveOptionalString(byok.type, env, `${target.name} byok type`, { + allowLiteral: true, + optionalEnv: true, + }); + + byokBaseUrl = resolveOptionalString(byok.base_url, env, `${target.name} byok base URL`, { + allowLiteral: true, + optionalEnv: true, + }); + + byokApiKey = resolveOptionalString(byok.api_key, env, `${target.name} byok API key`, { + allowLiteral: false, + optionalEnv: true, + }); + + byokBearerToken = resolveOptionalString( + byok.bearer_token, + env, + `${target.name} byok bearer token`, + { + allowLiteral: false, + optionalEnv: true, + }, + ); + + byokApiVersion = resolveOptionalString( + byok.api_version, + env, + `${target.name} byok API version`, + { + allowLiteral: true, + optionalEnv: true, + }, + ); + + byokWireApi = resolveOptionalString(byok.wire_api, env, `${target.name} byok wire API`, { + allowLiteral: true, + optionalEnv: true, + }); + + // base_url is required when byok is specified + if (!byokBaseUrl) { + throw new Error(`${target.name}: 'byok.base_url' is required when 'byok' is specified`); + } + } + return { cliUrl, cliPath, @@ -1438,6 +1508,12 @@ function resolveCopilotSdkConfig( logDir, logFormat, systemPrompt, + byokType, + byokBaseUrl, + byokApiKey, + byokBearerToken, + byokApiVersion, + byokWireApi, }; } diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index b24833643..f9ed86758 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -366,6 +366,8 @@ export interface TargetDefinition { readonly cli_url?: string | unknown | undefined; readonly cli_path?: string | unknown | undefined; readonly github_token?: string | unknown | undefined; + // Copilot SDK BYOK (Bring Your Own Key) — routes through a user-provided endpoint + readonly byok?: Record | undefined; // Retry configuration fields readonly max_retries?: number | unknown | undefined; readonly retry_initial_delay_ms?: number | unknown | undefined; diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index f4cfb7626..f31ddc635 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -123,6 +123,7 @@ const COPILOT_SDK_SETTINGS = new Set([ 'log_format', 'system_prompt', 'workspace_template', + 'byok', ]); const COPILOT_CLI_SETTINGS = new Set([ diff --git a/packages/core/test/evaluation/providers/copilot-sdk.test.ts b/packages/core/test/evaluation/providers/copilot-sdk.test.ts index f70df4823..59eb3be13 100644 --- a/packages/core/test/evaluation/providers/copilot-sdk.test.ts +++ b/packages/core/test/evaluation/providers/copilot-sdk.test.ts @@ -320,6 +320,188 @@ describe('CopilotSdkProvider', () => { expect(result.kind).toBe('approved'); }); + it('passes byok provider block to createSession for azure', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + const sdkMock = mockCopilotSdk(client); + + mock.module('@github/copilot-sdk', () => sdkMock); + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + model: 'gpt-4o', + byokType: 'azure', + byokBaseUrl: 'https://my-resource.openai.azure.com', + byokApiKey: 'azure-secret', + byokApiVersion: '2024-10-21', + }); + + await provider.invoke({ question: 'Test' }); + + const sessionOptions = client.createSession.mock.calls[0][0]; + expect(sessionOptions.provider).toBeDefined(); + expect(sessionOptions.provider.type).toBe('azure'); + expect(sessionOptions.provider.baseUrl).toBe('https://my-resource.openai.azure.com'); + expect(sessionOptions.provider.apiKey).toBe('azure-secret'); + expect(sessionOptions.provider.azure).toEqual({ apiVersion: '2024-10-21' }); + }); + + it('normalizes bare azure resource name to full URL', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + const sdkMock = mockCopilotSdk(client); + + mock.module('@github/copilot-sdk', () => sdkMock); + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + byokType: 'azure', + byokBaseUrl: 'my-resource-eastus2', + byokApiKey: 'key', + }); + + await provider.invoke({ question: 'Test' }); + + const sessionOptions = client.createSession.mock.calls[0][0]; + expect(sessionOptions.provider.baseUrl).toBe('https://my-resource-eastus2.openai.azure.com'); + }); + + it('passes full URL through unchanged for azure byok', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + const sdkMock = mockCopilotSdk(client); + + mock.module('@github/copilot-sdk', () => sdkMock); + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + byokType: 'azure', + byokBaseUrl: 'https://my-resource.openai.azure.com', + byokApiKey: 'key', + }); + + await provider.invoke({ question: 'Test' }); + + const sessionOptions = client.createSession.mock.calls[0][0]; + expect(sessionOptions.provider.baseUrl).toBe('https://my-resource.openai.azure.com'); + }); + + it('passes byok provider block with bearer token', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + const sdkMock = mockCopilotSdk(client); + + mock.module('@github/copilot-sdk', () => sdkMock); + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + byokType: 'openai', + byokBaseUrl: 'https://custom-endpoint.example.com/v1', + byokBearerToken: 'bearer-secret', + }); + + await provider.invoke({ question: 'Test' }); + + const sessionOptions = client.createSession.mock.calls[0][0]; + expect(sessionOptions.provider).toBeDefined(); + expect(sessionOptions.provider.bearerToken).toBe('bearer-secret'); + expect(sessionOptions.provider.apiKey).toBeUndefined(); + }); + + it('passes byok provider block with wireApi', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + const sdkMock = mockCopilotSdk(client); + + mock.module('@github/copilot-sdk', () => sdkMock); + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + byokType: 'openai', + byokBaseUrl: 'https://resource.openai.azure.com/openai/v1/', + byokApiKey: 'key', + byokWireApi: 'responses', + }); + + await provider.invoke({ question: 'Test' }); + + const sessionOptions = client.createSession.mock.calls[0][0]; + expect(sessionOptions.provider.wireApi).toBe('responses'); + }); + + it('does not set provider when byok is not configured', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + const sdkMock = mockCopilotSdk(client); + + mock.module('@github/copilot-sdk', () => sdkMock); + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + model: 'gpt-4o', + }); + + await provider.invoke({ question: 'Test' }); + + const sessionOptions = client.createSession.mock.calls[0][0]; + expect(sessionOptions.provider).toBeUndefined(); + }); + + it('defaults byok type to openai when not specified', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + const sdkMock = mockCopilotSdk(client); + + mock.module('@github/copilot-sdk', () => sdkMock); + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + byokBaseUrl: 'http://localhost:11434/v1', + }); + + await provider.invoke({ question: 'Test' }); + + const sessionOptions = client.createSession.mock.calls[0][0]; + expect(sessionOptions.provider.type).toBe('openai'); + }); + + it('does not set azure block for non-azure byok type', async () => { + const session = createMockSession({ + events: [{ type: 'assistant.message', data: { content: 'response' } }], + }); + const client = createMockClient(session); + const sdkMock = mockCopilotSdk(client); + + mock.module('@github/copilot-sdk', () => sdkMock); + const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); + + const provider = new CopilotSdkProvider('test-target', { + byokType: 'openai', + byokBaseUrl: 'https://api.openai.com/v1', + byokApiKey: 'key', + byokApiVersion: '2024-10-21', // should be ignored for non-azure + }); + + await provider.invoke({ question: 'Test' }); + + const sessionOptions = client.createSession.mock.calls[0][0]; + expect(sessionOptions.provider.azure).toBeUndefined(); + }); + it('includes timing information in response', async () => { const session = createMockSession({ events: [{ type: 'assistant.message', data: { content: 'response' } }], diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index aac8d357a..030c044c0 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -750,6 +750,218 @@ describe('resolveTargetDefinition', () => { expect(target.config.executable).toBe('copilot'); }); + it('resolves copilot-sdk with byok azure config', () => { + const env = { + AZURE_OPENAI_ENDPOINT: 'https://my-resource.openai.azure.com', + AZURE_OPENAI_API_KEY: 'azure-secret', + AZURE_DEPLOYMENT_NAME: 'gpt-4o', + } satisfies Record; + + const target = resolveTargetDefinition( + { + name: 'copilot-sdk-azure', + provider: 'copilot-sdk', + model: '${{ AZURE_DEPLOYMENT_NAME }}', + byok: { + type: 'azure', + base_url: '${{ AZURE_OPENAI_ENDPOINT }}', + api_key: '${{ AZURE_OPENAI_API_KEY }}', + api_version: '2024-10-21', + }, + }, + env, + ); + + expect(target.kind).toBe('copilot-sdk'); + if (target.kind !== 'copilot-sdk') { + throw new Error('expected copilot-sdk target'); + } + + expect(target.config.model).toBe('gpt-4o'); + expect(target.config.byokType).toBe('azure'); + expect(target.config.byokBaseUrl).toBe('https://my-resource.openai.azure.com'); + expect(target.config.byokApiKey).toBe('azure-secret'); + expect(target.config.byokApiVersion).toBe('2024-10-21'); + }); + + it('resolves copilot-sdk with byok openai config', () => { + const env = { + OPENAI_API_KEY: 'openai-secret', + } satisfies Record; + + const target = resolveTargetDefinition( + { + name: 'copilot-sdk-openai', + provider: 'copilot-sdk', + model: 'gpt-5', + byok: { + type: 'openai', + base_url: 'https://api.openai.com/v1', + api_key: '${{ OPENAI_API_KEY }}', + }, + }, + env, + ); + + expect(target.kind).toBe('copilot-sdk'); + if (target.kind !== 'copilot-sdk') { + throw new Error('expected copilot-sdk target'); + } + + expect(target.config.byokType).toBe('openai'); + expect(target.config.byokBaseUrl).toBe('https://api.openai.com/v1'); + expect(target.config.byokApiKey).toBe('openai-secret'); + }); + + it('copilot-sdk byok defaults type to undefined when not specified', () => { + const env = { + MY_KEY: 'secret', + } satisfies Record; + + const target = resolveTargetDefinition( + { + name: 'copilot-sdk-byok-minimal', + provider: 'copilot-sdk', + byok: { + base_url: 'http://localhost:11434/v1', + api_key: '${{ MY_KEY }}', + }, + }, + env, + ); + + expect(target.kind).toBe('copilot-sdk'); + if (target.kind !== 'copilot-sdk') { + throw new Error('expected copilot-sdk target'); + } + + expect(target.config.byokType).toBeUndefined(); + expect(target.config.byokBaseUrl).toBe('http://localhost:11434/v1'); + expect(target.config.byokApiKey).toBe('secret'); + }); + + it('copilot-sdk byok rejects missing base_url', () => { + expect(() => + resolveTargetDefinition( + { + name: 'copilot-sdk-no-url', + provider: 'copilot-sdk', + byok: { + type: 'azure', + api_key: '${{ MY_KEY }}', + }, + }, + { MY_KEY: 'secret' }, + ), + ).toThrow(/byok\.base_url.*required/i); + }); + + it('copilot-sdk byok rejects literal api_key', () => { + expect(() => + resolveTargetDefinition( + { + name: 'copilot-sdk-literal-key', + provider: 'copilot-sdk', + byok: { + base_url: 'https://example.com', + api_key: 'plaintext-secret', + }, + }, + {}, + ), + ).toThrow(/must use.*VARIABLE_NAME/i); + }); + + it('copilot-sdk byok rejects literal bearer_token', () => { + expect(() => + resolveTargetDefinition( + { + name: 'copilot-sdk-literal-bearer', + provider: 'copilot-sdk', + byok: { + base_url: 'https://example.com', + bearer_token: 'plaintext-bearer-secret', + }, + }, + {}, + ), + ).toThrow(/must use.*VARIABLE_NAME/i); + }); + + it('copilot-sdk byok supports bearer_token', () => { + const env = { + MY_TOKEN: 'bearer-secret', + } satisfies Record; + + const target = resolveTargetDefinition( + { + name: 'copilot-sdk-bearer', + provider: 'copilot-sdk', + byok: { + base_url: 'https://custom-endpoint.example.com/v1', + bearer_token: '${{ MY_TOKEN }}', + }, + }, + env, + ); + + expect(target.kind).toBe('copilot-sdk'); + if (target.kind !== 'copilot-sdk') { + throw new Error('expected copilot-sdk target'); + } + + expect(target.config.byokBearerToken).toBe('bearer-secret'); + expect(target.config.byokApiKey).toBeUndefined(); + }); + + it('copilot-sdk byok supports wire_api', () => { + const env = { + FOUNDRY_KEY: 'foundry-secret', + } satisfies Record; + + const target = resolveTargetDefinition( + { + name: 'copilot-sdk-responses', + provider: 'copilot-sdk', + model: 'gpt-5', + byok: { + type: 'openai', + base_url: 'https://resource.openai.azure.com/openai/v1/', + api_key: '${{ FOUNDRY_KEY }}', + wire_api: 'responses', + }, + }, + env, + ); + + expect(target.kind).toBe('copilot-sdk'); + if (target.kind !== 'copilot-sdk') { + throw new Error('expected copilot-sdk target'); + } + + expect(target.config.byokWireApi).toBe('responses'); + }); + + it('copilot-sdk without byok has no byok fields', () => { + const target = resolveTargetDefinition( + { + name: 'copilot-sdk-plain', + provider: 'copilot-sdk', + model: 'gpt-4o', + }, + {}, + ); + + expect(target.kind).toBe('copilot-sdk'); + if (target.kind !== 'copilot-sdk') { + throw new Error('expected copilot-sdk target'); + } + + expect(target.config.byokType).toBeUndefined(); + expect(target.config.byokBaseUrl).toBeUndefined(); + expect(target.config.byokApiKey).toBeUndefined(); + }); + it('rejects removed target-level workspaceTemplate camelCase field', () => { expect(() => resolveTargetDefinition(