Route OpenCode explicit model overrides through the AI Gateway

.changeset/opencode-gateway-model-prefix.md

@@ -0,0 +1,5 @@
+---
+"@vercel/agent-eval": patch
+---
+
+Fix OpenCode explicit model overrides never routing through the AI Gateway. `--model anthropic/claude-sonnet-5` was passed to the OpenCode CLI verbatim, which reads the first segment as its *provider* id — a provider that is not configured in the generated opencode.json (only `vercel`, the AI Gateway, is) — so every explicit-model run died at session start with "Unexpected server error". The model override is now resolved host-side (`vercel/` is prefixed unless the caller already targets `vercel/...` or a configured `extraProviders` key) and handed to the runner via `input.extra`, mirroring codex's `openai/` prefixing. Observed models are reported back in the caller's namespace: when the host added the prefix, the runner strips the leading `vercel/` from the observation, so `observedModel === requestedModel` holds for canonical gateway ids and a gateway substitution still surfaces as a clean gateway id. Native-default observations (e.g. `vercel/google/gemini-3-pro-preview`) and already-prefixed or extra-provider requests are unchanged.

README.md

@@ -349,16 +349,19 @@ transcript or logs. Provide `model` to force a specific model.
 
 ### OpenCode model format
 
-OpenCode uses Vercel AI Gateway exclusively. Models must use the `vercel/{provider}/{model}` format:
+OpenCode uses Vercel AI Gateway exclusively. The OpenCode CLI reads models as
+`{providerID}/{modelID}`, where the provider is the CLI's own `vercel` (AI
+Gateway) provider — so both of these forms work:
 
 ```typescript
-model: 'vercel/anthropic/claude-sonnet-4'
-model: 'vercel/openai/gpt-4o'
-model: 'vercel/moonshotai/kimi-k2'
-model: 'vercel/minimax/minimax-m2.1'
+model: 'anthropic/claude-sonnet-4'        // canonical gateway id — vercel/ is added automatically
+model: 'vercel/anthropic/claude-sonnet-4' // OpenCode's native form — passed verbatim
 ```
 
-The `vercel/` prefix is required. Using `anthropic/claude-sonnet-4` (without `vercel/`) will fail with a "provider not found" error.
+When the prefix was added automatically, `observedModel` is reported back in
+the request's namespace (`anthropic/claude-sonnet-4`), so requested-vs-observed
+comparisons hold. Models targeting a provider configured via
+`agentOptions.extraProviders` are passed verbatim.
 
 ### Response-only evals
 

packages/agent-eval/src/lib/agents/opencode.test.ts

@@ -3,11 +3,13 @@ import { describe, expect, it } from 'vitest';
 // tested logic is exactly what the sandbox runs); the pure config generator lives
 // in the host-side definition.
 import {
+  buildOpenCodeCliArgs,
   extractObservedModelFromOpenCodeOutput,
   extractObservedModelFromSessionExport,
   extractSessionIdFromTranscript,
+  normalizeObservedModel,
 } from './opencode/run.mjs';
-import { generateOpenCodeConfig } from './opencode/agent.js';
+import { createOpenCodeDefinition, generateOpenCodeConfig, resolveOpenCodeModel } from './opencode/agent.js';
 
 describe('generateOpenCodeConfig', () => {
   it('grants only the default tool permissions when webResearch is off', () => {
@@ -33,6 +35,96 @@ describe('generateOpenCodeConfig', () => {
   });
 });
 
+describe('resolveOpenCodeModel', () => {
+  it('prefixes canonical gateway ids with the vercel provider', () => {
+    expect(resolveOpenCodeModel('anthropic/claude-sonnet-5')).toBe('vercel/anthropic/claude-sonnet-5');
+    expect(resolveOpenCodeModel('openai/gpt-5.5')).toBe('vercel/openai/gpt-5.5');
+  });
+
+  it('keeps already-prefixed ids verbatim', () => {
+    expect(resolveOpenCodeModel('vercel/anthropic/claude-sonnet-5')).toBe('vercel/anthropic/claude-sonnet-5');
+  });
+
+  it('keeps ids targeting an extra provider verbatim', () => {
+    const extraProviders = { 'anthropic-preview': { npm: '@ai-sdk/anthropic' } };
+    expect(resolveOpenCodeModel('anthropic-preview/claude-x', extraProviders)).toBe('anthropic-preview/claude-x');
+    // A gateway id still gets prefixed when extra providers are configured.
+    expect(resolveOpenCodeModel('anthropic/claude-sonnet-5', extraProviders)).toBe('vercel/anthropic/claude-sonnet-5');
+  });
+
+  it('prefixes bare ids so the gateway rejects them with a model error, not a provider error', () => {
+    expect(resolveOpenCodeModel('claude-sonnet-5')).toBe('vercel/claude-sonnet-5');
+  });
+});
+
+describe('OpenCode definition runnerExtra', () => {
+  const definition = createOpenCodeDefinition();
+
+  const baseOptions = { prompt: 'p', apiKey: 'k', timeout: 1000 };
+
+  it('resolves the model override against extraProviders from agentOptions', () => {
+    const extra = definition.runnerExtra!({
+      ...baseOptions,
+      model: 'anthropic/claude-sonnet-5',
+    });
+    expect(extra).toEqual({ cliModel: 'vercel/anthropic/claude-sonnet-5' });
+
+    const withProvider = definition.runnerExtra!({
+      ...baseOptions,
+      model: 'anthropic-preview/claude-x',
+      agentOptions: { extraProviders: { 'anthropic-preview': {} } },
+    });
+    expect(withProvider).toEqual({ cliModel: 'anthropic-preview/claude-x' });
+  });
+
+  it('is null on native-default runs (no model override)', () => {
+    expect(definition.runnerExtra!(baseOptions)).toEqual({ cliModel: null });
+  });
+});
+
+describe('buildOpenCodeCliArgs', () => {
+  it('passes the host-resolved model from extra.cliModel', () => {
+    const args = buildOpenCodeCliArgs({
+      prompt: 'do the thing',
+      model: 'anthropic/claude-sonnet-5',
+      extra: { cliModel: 'vercel/anthropic/claude-sonnet-5' },
+    });
+    expect(args).toEqual(['run', 'do the thing', '--format', 'json', '--model', 'vercel/anthropic/claude-sonnet-5']);
+  });
+
+  it('never passes input.model verbatim — extra.cliModel is the only model source', () => {
+    // Verbatim pass-through is the mis-route this fixes (opencode would read
+    // `anthropic` as its provider id). Without extra there is no --model at all.
+    const args = buildOpenCodeCliArgs({ prompt: 'p', model: 'anthropic/claude-sonnet-5' });
+    expect(args).toEqual(['run', 'p', '--format', 'json']);
+  });
+
+  it('enables log printing instead of --model on native-default runs', () => {
+    const args = buildOpenCodeCliArgs({ prompt: 'p', modelPolicy: 'native-default', extra: { cliModel: null } });
+    expect(args).toEqual(['run', 'p', '--format', 'json', '--print-logs', '--log-level', 'INFO']);
+  });
+});
+
+describe('normalizeObservedModel', () => {
+  it('un-applies the host vercel/ prefix so observed matches the requested gateway id', () => {
+    const input = { model: 'anthropic/claude-sonnet-5', extra: { cliModel: 'vercel/anthropic/claude-sonnet-5' } };
+    expect(normalizeObservedModel('vercel/anthropic/claude-sonnet-5', input)).toBe('anthropic/claude-sonnet-5');
+    // A gateway substitution still surfaces as a clean gateway id.
+    expect(normalizeObservedModel('vercel/anthropic/claude-haiku-4', input)).toBe('anthropic/claude-haiku-4');
+  });
+
+  it('passes observations through when the caller already spoke the OpenCode namespace', () => {
+    const input = { model: 'vercel/openai/gpt-5.5', extra: { cliModel: 'vercel/openai/gpt-5.5' } };
+    expect(normalizeObservedModel('vercel/openai/gpt-5.5', input)).toBe('vercel/openai/gpt-5.5');
+  });
+
+  it('passes native-default observations through untouched', () => {
+    const input = { modelPolicy: 'native-default', extra: { cliModel: null } };
+    expect(normalizeObservedModel('vercel/google/gemini-3-pro-preview', input)).toBe('vercel/google/gemini-3-pro-preview');
+    expect(normalizeObservedModel(undefined, input)).toBeUndefined();
+  });
+});
+
 describe('OpenCode observed model extraction', () => {
   it('extracts the primary build model from printed logs', () => {
     const output = [

packages/agent-eval/src/lib/agents/opencode/agent.ts

@@ -45,6 +45,38 @@ export interface OpenCodeProviderConfig {
   }>;
 }
 
+/**
+ * Resolve a model override into the id the OpenCode CLI expects.
+ *
+ * OpenCode reads `--model` as `<providerID>/<modelID>` and the generated
+ * opencode.json only configures the `vercel` (AI Gateway) provider. A canonical
+ * gateway id like `anthropic/claude-sonnet-5` therefore selects a *provider*
+ * named `anthropic` — which is not configured and has no key in the sandbox —
+ * and the CLI dies at session start ("Unexpected server error") on every task.
+ * The gateway form OpenCode understands is `vercel/anthropic/claude-sonnet-5`.
+ *
+ * So: prefix `vercel/` unless the caller already targets a configured provider
+ * (`vercel` itself, or a key in agentOptions.extraProviders — the escape hatch
+ * for unreleased models). Mirrors codex's `openai/` prefixing in
+ * generateCodexConfig, and the package's own defaultModel shape
+ * (`vercel/anthropic/claude-sonnet-4`). A bare id with no `/` (never a valid
+ * gateway id) is prefixed too, trading the opaque provider error for the
+ * gateway's model-not-found.
+ *
+ * Stays host-side (and exported for the unit tests) — the resolution needs
+ * extraProviders, which only the host knows.
+ */
+export function resolveOpenCodeModel(
+  model: string,
+  extraProviders?: Record<string, OpenCodeProviderConfig>
+): string {
+  const providerID = model.split('/')[0];
+  if (providerID === 'vercel' || (extraProviders && Object.hasOwn(extraProviders, providerID))) {
+    return model;
+  }
+  return `vercel/${model}`;
+}
+
 /**
  * Generate OpenCode config file content.
  * Configures the Vercel AI Gateway provider, plus any additional providers.
@@ -190,6 +222,22 @@ export function createOpenCodeDefinition(): AgentDefinition {
         ...(options.webResearch ? { OPENCODE_ENABLE_EXA: '1' } : {}),
       };
     },
+
+    /**
+     * Host-computed value the runner passes as `--model`: the model override
+     * resolved against the configured providers (see resolveOpenCodeModel).
+     * Computed here because extraProviders is host-side knowledge; the runner
+     * also uses cliModel-vs-model to report observations back in the caller's
+     * namespace (see normalizeObservedModel in run.mjs). null on native-default.
+     */
+    runnerExtra(options: AgentRunOptions): Record<string, unknown> {
+      const extraProviders = options.agentOptions?.extraProviders as
+        | Record<string, OpenCodeProviderConfig>
+        | undefined;
+      return {
+        cliModel: options.model ? resolveOpenCodeModel(options.model, extraProviders) : null,
+      };
+    },
   };
 }
 

packages/agent-eval/src/lib/agents/opencode/run.mjs

@@ -25,27 +25,65 @@ import { fileURLToPath } from 'node:url';
 /**
  * Build the OpenCode CLI argument list.
  *
- * Preserved verbatim from the old adapter:
  *   - `run <prompt> --format json` is always emitted (`--format json` gives the
  *     structured JSON event stream we parse for the transcript).
- *   - `--model <model>` is appended when a model override is present.
+ *   - `--model <extra.cliModel>` is appended when a model override is present.
+ *     cliModel is HOST-computed (resolveOpenCodeModel in agent.ts, threaded via
+ *     runnerExtra like codex's resolved model) so a canonical gateway id like
+ *     `anthropic/claude-sonnet-5` arrives as `vercel/anthropic/claude-sonnet-5` —
+ *     the form OpenCode routes through the configured `vercel` provider. There is
+ *     deliberately NO fallback to input.model: passing it verbatim is the exact
+ *     mis-route this fixes, and both runner entry points (the orchestrator and
+ *     the judge's eval-helper) always ship extra.
  *   - else, when modelPolicy === 'native-default', `--print-logs --log-level INFO`
  *     is appended so older OpenCode versions print the CLI-selected model (the
  *     log-scrape path) without a follow-up export call.
  *
- * @param {{prompt:string, model?:string, modelPolicy?:string}} input
+ * @param {{prompt:string, modelPolicy?:string, extra?:Record<string,unknown>}} input
  * @returns {string[]}
  */
 export function buildOpenCodeCliArgs(input) {
   const cliArgs = ['run', input.prompt, '--format', 'json'];
-  if (input.model) {
-    cliArgs.push('--model', input.model);
+  const cliModel = input.extra?.cliModel;
+  if (cliModel) {
+    cliArgs.push('--model', String(cliModel));
   } else if (input.modelPolicy === 'native-default') {
     cliArgs.push('--print-logs', '--log-level', 'INFO');
   }
   return cliArgs;
 }
 
+/**
+ * Report the observed model in the caller's namespace.
+ *
+ * When the host prefixed the requested model with `vercel/` (extra.cliModel !==
+ * input.model), OpenCode observes it back in its own provider namespace —
+ * `vercel/anthropic/claude-sonnet-5` for a requested `anthropic/claude-sonnet-5`.
+ * Un-apply exactly that translation so observed === requested holds for callers
+ * that speak canonical gateway ids, and a gateway substitution still surfaces as
+ * a clean gateway id (`anthropic/claude-haiku-4`, not `vercel/anthropic/...`).
+ *
+ * Everything else passes through untouched: native-default runs (no override —
+ * consumers already treat `vercel/google/gemini-3-pro-preview` as canonical),
+ * callers that already speak OpenCode's namespace (`vercel/...` requested), and
+ * extraProviders runs (their providerID isn't `vercel`).
+ *
+ * @param {string|undefined} observedModel
+ * @param {{model?:string, extra?:Record<string,unknown>}} input
+ * @returns {string|undefined}
+ */
+export function normalizeObservedModel(observedModel, input) {
+  if (!observedModel) {
+    return observedModel;
+  }
+  const cliModel = input.extra?.cliModel;
+  const hostPrefixed = typeof cliModel === 'string' && cliModel !== input.model;
+  if (hostPrefixed && observedModel.startsWith('vercel/')) {
+    return observedModel.slice('vercel/'.length);
+  }
+  return observedModel;
+}
+
 /**
  * Extract transcript from OpenCode JSON output.
  * When run with --format json, OpenCode outputs JSON events to stdout.
@@ -228,7 +266,7 @@ export async function runAgent(input) {
       }
     }
   }
-  const observedModelOrNull = observedModel ?? null;
+  const observedModelOrNull = normalizeObservedModel(observedModel, input) ?? null;
 
   if (res.error || agentExitCode !== 0) {
     // Mirror the old error string: last 5 lines of output, else a coded fallback.