diff --git a/.agentv/targets.yaml b/.agentv/targets.yaml index b76bfc74..067a75bf 100644 --- a/.agentv/targets.yaml +++ b/.agentv/targets.yaml @@ -56,12 +56,22 @@ targets: # Claude via Z.ai provider (GLM models). Requires cc-mirror: # npx cc-mirror quick --provider zai --api-key "$Z_AI_API_KEY" + # Alternative: set Z_AI_API_KEY in ~/.cc-mirror/claude-zai/config/settings.json # See https://github.com/numman-ali/cc-mirror - name: claude-zai - provider: claude-cli + provider: cc-mirror executable: claude-zai grader_target: grader + # Generic cc-mirror target. Set CC_MIRROR_VARIANT to the installed variant + # name (the directory under ~/.cc-mirror/, e.g. claude-zai, my-kimi). + # Setup: npx cc-mirror quick --provider --name --api-key "$KEY" + # See https://github.com/numman-ali/cc-mirror + - name: cc-mirror + provider: cc-mirror + variant: ${{ CC_MIRROR_VARIANT }} + grader_target: grader + - name: claude-sdk provider: claude-sdk grader_target: grader diff --git a/apps/web/src/content/docs/docs/targets/coding-agents.mdx b/apps/web/src/content/docs/docs/targets/coding-agents.mdx index 5c18ac08..6ee94a3d 100644 --- a/apps/web/src/content/docs/docs/targets/coding-agents.mdx +++ b/apps/web/src/content/docs/docs/targets/coding-agents.mdx @@ -75,6 +75,40 @@ targets: | `cwd` | No | Working directory (mutually exclusive with workspace_template) | | `grader_target` | Yes | LLM target for evaluation | +## cc-mirror + +[cc-mirror](https://github.com/numman-ali/cc-mirror) creates isolated Claude Code variants that route through alternative providers (Z.ai, Kimi, MiniMax, OpenRouter, etc.). The `cc-mirror` provider alias resolves to `claude-cli` and auto-discovers the binary path from `~/.cc-mirror//variant.json`. + +```yaml +targets: + # Explicit variant with known executable + - name: claude-zai + provider: cc-mirror + executable: claude-zai + grader_target: azure-base + + # Auto-discover binary from variant.json + - name: my-kimi + provider: cc-mirror + grader_target: azure-base +``` + +| Field | Required | Description | +|-------|----------|-------------| +| `executable` | No | CLI binary name or path. When set, used directly (skips variant.json lookup). | +| `variant` | No | Variant name (directory under `~/.cc-mirror/`). Defaults to target `name`. Used to locate `variant.json` when `executable` is not set. | +| `workspace_template` | No | Path to workspace template directory | +| `cwd` | No | Working directory (mutually exclusive with workspace_template) | +| `grader_target` | Yes | LLM target for evaluation | + +Setup a variant first, then reference it by name: + +```bash +npx cc-mirror quick --provider zai --name claude-zai --api-key "$Z_AI_API_KEY" +``` + +Since `cc-mirror` resolves to `claude-cli`, all Claude target fields (model, system_prompt, timeout_seconds, etc.) are also supported. + ## Codex CLI ```yaml diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index ce34eae7..d41d7198 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -1,3 +1,5 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { homedir } from 'node:os'; import path from 'node:path'; import { z } from 'zod'; @@ -1037,6 +1039,22 @@ export function resolveTargetDefinition( ...base, config: resolvePiCliConfig(parsed, env, evalFilePath), }; + case 'cc-mirror': { + const variantName = + resolveOptionalString(parsed.variant, env, `${parsed.name} cc-mirror variant`, { + allowLiteral: true, + optionalEnv: true, + }) ?? parsed.name; + // If executable is explicitly set, use it; otherwise auto-discover from variant.json + if (!parsed.executable) { + parsed.executable = resolveCcMirrorBinaryPath(variantName); + } + return { + kind: 'claude-cli', + ...base, + config: resolveClaudeConfig(parsed, env, evalFilePath), + }; + } case 'claude': case 'claude-code': case 'claude-cli': @@ -2028,6 +2046,33 @@ function resolveClaudeConfig( }; } +/** + * Resolve the binary path for a cc-mirror variant. + * Reads ~/.cc-mirror//variant.json → binaryPath. + */ +function resolveCcMirrorBinaryPath(variant: string): string { + const variantJsonPath = path.join(homedir(), '.cc-mirror', variant, 'variant.json'); + if (!existsSync(variantJsonPath)) { + throw new Error( + `cc-mirror variant "${variant}": ${variantJsonPath} not found. Install the variant or set "executable" explicitly.`, + ); + } + let parsed: { binaryPath?: string }; + try { + parsed = JSON.parse(readFileSync(variantJsonPath, 'utf8')); + } catch (e) { + throw new Error( + `cc-mirror variant "${variant}": failed to parse ${variantJsonPath}: ${(e as Error).message}`, + ); + } + if (typeof parsed.binaryPath !== 'string' || parsed.binaryPath.trim().length === 0) { + throw new Error( + `cc-mirror variant "${variant}": ${variantJsonPath} missing "binaryPath" field`, + ); + } + return parsed.binaryPath; +} + function normalizeClaudeLogFormat(value: unknown): 'summary' | 'json' | undefined { if (value === undefined || value === null) { return undefined; diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index 57c392c9..92b19fe0 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -114,6 +114,7 @@ export const PROVIDER_ALIASES: readonly string[] = [ 'pi', // alias for "pi-coding-agent" 'claude-code', // alias for "claude" (legacy) + 'cc-mirror', // alias for "claude-cli" (auto-discovers binary from ~/.cc-mirror//) 'bedrock', // legacy/future support 'vertex', // legacy/future support ] as const; diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index 0c1b4ddf..9506a902 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -179,6 +179,8 @@ const CLAUDE_SETTINGS = new Set([ 'max_budget_usd', ]); +const CC_MIRROR_SETTINGS = new Set([...CLAUDE_SETTINGS, 'variant']); + function getKnownSettings(provider: string): Set | null { const normalizedProvider = provider.toLowerCase(); switch (normalizedProvider) { @@ -204,6 +206,8 @@ function getKnownSettings(provider: string): Set | null { case 'copilot': case 'copilot-cli': return COPILOT_CLI_SETTINGS; + case 'cc-mirror': + return CC_MIRROR_SETTINGS; case 'claude': case 'claude-code': case 'claude-cli': diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index c5fd8861..7088c30f 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -747,6 +747,43 @@ describe('resolveTargetDefinition', () => { expect(target.config.executable).toBe('claude-zai'); }); + it('cc-mirror with explicit executable resolves to claude-cli kind', () => { + const target = resolveTargetDefinition( + { + name: 'claude-zai', + provider: 'cc-mirror', + executable: '/usr/local/bin/claude-zai', + }, + {}, + ); + + expect(target.kind).toBe('claude-cli'); + if (target.kind !== 'claude-cli') { + throw new Error('expected claude-cli target'); + } + + expect(target.config.executable).toBe('/usr/local/bin/claude-zai'); + }); + + it('cc-mirror with explicit variant and executable', () => { + const target = resolveTargetDefinition( + { + name: 'my-mirror', + provider: 'cc-mirror', + variant: 'claude-zai', + executable: '/opt/bin/zai', + }, + {}, + ); + + expect(target.kind).toBe('claude-cli'); + if (target.kind !== 'claude-cli') { + throw new Error('expected claude-cli target'); + } + + expect(target.config.executable).toBe('/opt/bin/zai'); + }); + it('resolves copilot-cli as its own provider kind', () => { const target = resolveTargetDefinition( {