From d6e9c86c769f7dcbe5bf78a1f54ad90c2ac230a0 Mon Sep 17 00:00:00 2001 From: molebox Date: Mon, 6 Jul 2026 12:06:21 +0200 Subject: [PATCH] agent-eval: route OpenCode explicit model overrides through the AI Gateway An explicit --model like anthropic/claude-sonnet-5 was passed to the OpenCode CLI verbatim, which reads the first segment as its provider id. The generated opencode.json only configures the vercel (AI Gateway) provider, so every explicit-model run using a canonical gateway id died at session start with "Unexpected server error" and a null observedModel. - resolveOpenCodeModel (host-side): prefix vercel/ unless the caller already targets vercel/... or a configured extraProviders key. - Threaded to the runner via runnerExtra -> input.extra.cliModel, mirroring codex's host-computed model (the judge path ships extra but not agentOptions, so the resolution must happen host-side). - normalizeObservedModel (runner-side): un-apply exactly that prefix on the observation, so observedModel === requestedModel holds for canonical gateway ids and a gateway substitution still surfaces as a clean gateway id. Native-default observations are untouched, and the documented vercel/-prefixed form still passes through verbatim. - README: the OpenCode model format section now documents both forms (it previously stated unprefixed ids fail with provider-not-found). --- .changeset/opencode-gateway-model-prefix.md | 5 + README.md | 15 +-- .../src/lib/agents/opencode.test.ts | 94 ++++++++++++++++++- .../src/lib/agents/opencode/agent.ts | 48 ++++++++++ .../src/lib/agents/opencode/run.mjs | 50 ++++++++-- 5 files changed, 199 insertions(+), 13 deletions(-) create mode 100644 .changeset/opencode-gateway-model-prefix.md diff --git a/.changeset/opencode-gateway-model-prefix.md b/.changeset/opencode-gateway-model-prefix.md new file mode 100644 index 0000000..d08c4bb --- /dev/null +++ b/.changeset/opencode-gateway-model-prefix.md @@ -0,0 +1,5 @@ +--- +"@vercel/agent-eval": patch +--- + +Fix OpenCode explicit model overrides never routing through the AI Gateway. `--model anthropic/claude-sonnet-5` was passed to the OpenCode CLI verbatim, which reads the first segment as its *provider* id — a provider that is not configured in the generated opencode.json (only `vercel`, the AI Gateway, is) — so every explicit-model run died at session start with "Unexpected server error". The model override is now resolved host-side (`vercel/` is prefixed unless the caller already targets `vercel/...` or a configured `extraProviders` key) and handed to the runner via `input.extra`, mirroring codex's `openai/` prefixing. Observed models are reported back in the caller's namespace: when the host added the prefix, the runner strips the leading `vercel/` from the observation, so `observedModel === requestedModel` holds for canonical gateway ids and a gateway substitution still surfaces as a clean gateway id. Native-default observations (e.g. `vercel/google/gemini-3-pro-preview`) and already-prefixed or extra-provider requests are unchanged. diff --git a/README.md b/README.md index 6eade5d..7b45474 100644 --- a/README.md +++ b/README.md @@ -349,16 +349,19 @@ transcript or logs. Provide `model` to force a specific model. ### OpenCode model format -OpenCode uses Vercel AI Gateway exclusively. Models must use the `vercel/{provider}/{model}` format: +OpenCode uses Vercel AI Gateway exclusively. The OpenCode CLI reads models as +`{providerID}/{modelID}`, where the provider is the CLI's own `vercel` (AI +Gateway) provider — so both of these forms work: ```typescript -model: 'vercel/anthropic/claude-sonnet-4' -model: 'vercel/openai/gpt-4o' -model: 'vercel/moonshotai/kimi-k2' -model: 'vercel/minimax/minimax-m2.1' +model: 'anthropic/claude-sonnet-4' // canonical gateway id — vercel/ is added automatically +model: 'vercel/anthropic/claude-sonnet-4' // OpenCode's native form — passed verbatim ``` -The `vercel/` prefix is required. Using `anthropic/claude-sonnet-4` (without `vercel/`) will fail with a "provider not found" error. +When the prefix was added automatically, `observedModel` is reported back in +the request's namespace (`anthropic/claude-sonnet-4`), so requested-vs-observed +comparisons hold. Models targeting a provider configured via +`agentOptions.extraProviders` are passed verbatim. ### Response-only evals diff --git a/packages/agent-eval/src/lib/agents/opencode.test.ts b/packages/agent-eval/src/lib/agents/opencode.test.ts index 7380ae4..308bea9 100644 --- a/packages/agent-eval/src/lib/agents/opencode.test.ts +++ b/packages/agent-eval/src/lib/agents/opencode.test.ts @@ -3,11 +3,13 @@ import { describe, expect, it } from 'vitest'; // tested logic is exactly what the sandbox runs); the pure config generator lives // in the host-side definition. import { + buildOpenCodeCliArgs, extractObservedModelFromOpenCodeOutput, extractObservedModelFromSessionExport, extractSessionIdFromTranscript, + normalizeObservedModel, } from './opencode/run.mjs'; -import { generateOpenCodeConfig } from './opencode/agent.js'; +import { createOpenCodeDefinition, generateOpenCodeConfig, resolveOpenCodeModel } from './opencode/agent.js'; describe('generateOpenCodeConfig', () => { it('grants only the default tool permissions when webResearch is off', () => { @@ -33,6 +35,96 @@ describe('generateOpenCodeConfig', () => { }); }); +describe('resolveOpenCodeModel', () => { + it('prefixes canonical gateway ids with the vercel provider', () => { + expect(resolveOpenCodeModel('anthropic/claude-sonnet-5')).toBe('vercel/anthropic/claude-sonnet-5'); + expect(resolveOpenCodeModel('openai/gpt-5.5')).toBe('vercel/openai/gpt-5.5'); + }); + + it('keeps already-prefixed ids verbatim', () => { + expect(resolveOpenCodeModel('vercel/anthropic/claude-sonnet-5')).toBe('vercel/anthropic/claude-sonnet-5'); + }); + + it('keeps ids targeting an extra provider verbatim', () => { + const extraProviders = { 'anthropic-preview': { npm: '@ai-sdk/anthropic' } }; + expect(resolveOpenCodeModel('anthropic-preview/claude-x', extraProviders)).toBe('anthropic-preview/claude-x'); + // A gateway id still gets prefixed when extra providers are configured. + expect(resolveOpenCodeModel('anthropic/claude-sonnet-5', extraProviders)).toBe('vercel/anthropic/claude-sonnet-5'); + }); + + it('prefixes bare ids so the gateway rejects them with a model error, not a provider error', () => { + expect(resolveOpenCodeModel('claude-sonnet-5')).toBe('vercel/claude-sonnet-5'); + }); +}); + +describe('OpenCode definition runnerExtra', () => { + const definition = createOpenCodeDefinition(); + + const baseOptions = { prompt: 'p', apiKey: 'k', timeout: 1000 }; + + it('resolves the model override against extraProviders from agentOptions', () => { + const extra = definition.runnerExtra!({ + ...baseOptions, + model: 'anthropic/claude-sonnet-5', + }); + expect(extra).toEqual({ cliModel: 'vercel/anthropic/claude-sonnet-5' }); + + const withProvider = definition.runnerExtra!({ + ...baseOptions, + model: 'anthropic-preview/claude-x', + agentOptions: { extraProviders: { 'anthropic-preview': {} } }, + }); + expect(withProvider).toEqual({ cliModel: 'anthropic-preview/claude-x' }); + }); + + it('is null on native-default runs (no model override)', () => { + expect(definition.runnerExtra!(baseOptions)).toEqual({ cliModel: null }); + }); +}); + +describe('buildOpenCodeCliArgs', () => { + it('passes the host-resolved model from extra.cliModel', () => { + const args = buildOpenCodeCliArgs({ + prompt: 'do the thing', + model: 'anthropic/claude-sonnet-5', + extra: { cliModel: 'vercel/anthropic/claude-sonnet-5' }, + }); + expect(args).toEqual(['run', 'do the thing', '--format', 'json', '--model', 'vercel/anthropic/claude-sonnet-5']); + }); + + it('never passes input.model verbatim — extra.cliModel is the only model source', () => { + // Verbatim pass-through is the mis-route this fixes (opencode would read + // `anthropic` as its provider id). Without extra there is no --model at all. + const args = buildOpenCodeCliArgs({ prompt: 'p', model: 'anthropic/claude-sonnet-5' }); + expect(args).toEqual(['run', 'p', '--format', 'json']); + }); + + it('enables log printing instead of --model on native-default runs', () => { + const args = buildOpenCodeCliArgs({ prompt: 'p', modelPolicy: 'native-default', extra: { cliModel: null } }); + expect(args).toEqual(['run', 'p', '--format', 'json', '--print-logs', '--log-level', 'INFO']); + }); +}); + +describe('normalizeObservedModel', () => { + it('un-applies the host vercel/ prefix so observed matches the requested gateway id', () => { + const input = { model: 'anthropic/claude-sonnet-5', extra: { cliModel: 'vercel/anthropic/claude-sonnet-5' } }; + expect(normalizeObservedModel('vercel/anthropic/claude-sonnet-5', input)).toBe('anthropic/claude-sonnet-5'); + // A gateway substitution still surfaces as a clean gateway id. + expect(normalizeObservedModel('vercel/anthropic/claude-haiku-4', input)).toBe('anthropic/claude-haiku-4'); + }); + + it('passes observations through when the caller already spoke the OpenCode namespace', () => { + const input = { model: 'vercel/openai/gpt-5.5', extra: { cliModel: 'vercel/openai/gpt-5.5' } }; + expect(normalizeObservedModel('vercel/openai/gpt-5.5', input)).toBe('vercel/openai/gpt-5.5'); + }); + + it('passes native-default observations through untouched', () => { + const input = { modelPolicy: 'native-default', extra: { cliModel: null } }; + expect(normalizeObservedModel('vercel/google/gemini-3-pro-preview', input)).toBe('vercel/google/gemini-3-pro-preview'); + expect(normalizeObservedModel(undefined, input)).toBeUndefined(); + }); +}); + describe('OpenCode observed model extraction', () => { it('extracts the primary build model from printed logs', () => { const output = [ diff --git a/packages/agent-eval/src/lib/agents/opencode/agent.ts b/packages/agent-eval/src/lib/agents/opencode/agent.ts index 461156f..419fd4d 100644 --- a/packages/agent-eval/src/lib/agents/opencode/agent.ts +++ b/packages/agent-eval/src/lib/agents/opencode/agent.ts @@ -45,6 +45,38 @@ export interface OpenCodeProviderConfig { }>; } +/** + * Resolve a model override into the id the OpenCode CLI expects. + * + * OpenCode reads `--model` as `/` and the generated + * opencode.json only configures the `vercel` (AI Gateway) provider. A canonical + * gateway id like `anthropic/claude-sonnet-5` therefore selects a *provider* + * named `anthropic` — which is not configured and has no key in the sandbox — + * and the CLI dies at session start ("Unexpected server error") on every task. + * The gateway form OpenCode understands is `vercel/anthropic/claude-sonnet-5`. + * + * So: prefix `vercel/` unless the caller already targets a configured provider + * (`vercel` itself, or a key in agentOptions.extraProviders — the escape hatch + * for unreleased models). Mirrors codex's `openai/` prefixing in + * generateCodexConfig, and the package's own defaultModel shape + * (`vercel/anthropic/claude-sonnet-4`). A bare id with no `/` (never a valid + * gateway id) is prefixed too, trading the opaque provider error for the + * gateway's model-not-found. + * + * Stays host-side (and exported for the unit tests) — the resolution needs + * extraProviders, which only the host knows. + */ +export function resolveOpenCodeModel( + model: string, + extraProviders?: Record +): string { + const providerID = model.split('/')[0]; + if (providerID === 'vercel' || (extraProviders && Object.hasOwn(extraProviders, providerID))) { + return model; + } + return `vercel/${model}`; +} + /** * Generate OpenCode config file content. * Configures the Vercel AI Gateway provider, plus any additional providers. @@ -190,6 +222,22 @@ export function createOpenCodeDefinition(): AgentDefinition { ...(options.webResearch ? { OPENCODE_ENABLE_EXA: '1' } : {}), }; }, + + /** + * Host-computed value the runner passes as `--model`: the model override + * resolved against the configured providers (see resolveOpenCodeModel). + * Computed here because extraProviders is host-side knowledge; the runner + * also uses cliModel-vs-model to report observations back in the caller's + * namespace (see normalizeObservedModel in run.mjs). null on native-default. + */ + runnerExtra(options: AgentRunOptions): Record { + const extraProviders = options.agentOptions?.extraProviders as + | Record + | undefined; + return { + cliModel: options.model ? resolveOpenCodeModel(options.model, extraProviders) : null, + }; + }, }; } diff --git a/packages/agent-eval/src/lib/agents/opencode/run.mjs b/packages/agent-eval/src/lib/agents/opencode/run.mjs index 2f99ded..d55115d 100644 --- a/packages/agent-eval/src/lib/agents/opencode/run.mjs +++ b/packages/agent-eval/src/lib/agents/opencode/run.mjs @@ -25,27 +25,65 @@ import { fileURLToPath } from 'node:url'; /** * Build the OpenCode CLI argument list. * - * Preserved verbatim from the old adapter: * - `run --format json` is always emitted (`--format json` gives the * structured JSON event stream we parse for the transcript). - * - `--model ` is appended when a model override is present. + * - `--model ` is appended when a model override is present. + * cliModel is HOST-computed (resolveOpenCodeModel in agent.ts, threaded via + * runnerExtra like codex's resolved model) so a canonical gateway id like + * `anthropic/claude-sonnet-5` arrives as `vercel/anthropic/claude-sonnet-5` — + * the form OpenCode routes through the configured `vercel` provider. There is + * deliberately NO fallback to input.model: passing it verbatim is the exact + * mis-route this fixes, and both runner entry points (the orchestrator and + * the judge's eval-helper) always ship extra. * - else, when modelPolicy === 'native-default', `--print-logs --log-level INFO` * is appended so older OpenCode versions print the CLI-selected model (the * log-scrape path) without a follow-up export call. * - * @param {{prompt:string, model?:string, modelPolicy?:string}} input + * @param {{prompt:string, modelPolicy?:string, extra?:Record}} input * @returns {string[]} */ export function buildOpenCodeCliArgs(input) { const cliArgs = ['run', input.prompt, '--format', 'json']; - if (input.model) { - cliArgs.push('--model', input.model); + const cliModel = input.extra?.cliModel; + if (cliModel) { + cliArgs.push('--model', String(cliModel)); } else if (input.modelPolicy === 'native-default') { cliArgs.push('--print-logs', '--log-level', 'INFO'); } return cliArgs; } +/** + * Report the observed model in the caller's namespace. + * + * When the host prefixed the requested model with `vercel/` (extra.cliModel !== + * input.model), OpenCode observes it back in its own provider namespace — + * `vercel/anthropic/claude-sonnet-5` for a requested `anthropic/claude-sonnet-5`. + * Un-apply exactly that translation so observed === requested holds for callers + * that speak canonical gateway ids, and a gateway substitution still surfaces as + * a clean gateway id (`anthropic/claude-haiku-4`, not `vercel/anthropic/...`). + * + * Everything else passes through untouched: native-default runs (no override — + * consumers already treat `vercel/google/gemini-3-pro-preview` as canonical), + * callers that already speak OpenCode's namespace (`vercel/...` requested), and + * extraProviders runs (their providerID isn't `vercel`). + * + * @param {string|undefined} observedModel + * @param {{model?:string, extra?:Record}} input + * @returns {string|undefined} + */ +export function normalizeObservedModel(observedModel, input) { + if (!observedModel) { + return observedModel; + } + const cliModel = input.extra?.cliModel; + const hostPrefixed = typeof cliModel === 'string' && cliModel !== input.model; + if (hostPrefixed && observedModel.startsWith('vercel/')) { + return observedModel.slice('vercel/'.length); + } + return observedModel; +} + /** * Extract transcript from OpenCode JSON output. * When run with --format json, OpenCode outputs JSON events to stdout. @@ -228,7 +266,7 @@ export async function runAgent(input) { } } } - const observedModelOrNull = observedModel ?? null; + const observedModelOrNull = normalizeObservedModel(observedModel, input) ?? null; if (res.error || agentExitCode !== 0) { // Mirror the old error string: last 5 lines of output, else a coded fallback.