From 060480b0447916e0dabb7384be8b33805def2d23 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 14 Apr 2026 01:27:02 +0000 Subject: [PATCH 1/6] feat(providers): add cc-mirror provider alias for auto-discovering cc-mirror variants cc-mirror resolves to claude-cli kind and auto-discovers the binary path from ~/.cc-mirror//variant.json. If executable is explicitly set, it's used directly without variant.json lookup. Co-Authored-By: Claude Opus 4.6 --- .agentv/targets.yaml | 3 +- .../core/src/evaluation/providers/targets.ts | 43 +++++++++++++++++++ .../core/src/evaluation/providers/types.ts | 1 + .../validation/targets-validator.ts | 4 ++ .../test/evaluation/providers/targets.test.ts | 37 ++++++++++++++++ 5 files changed, 87 insertions(+), 1 deletion(-) diff --git a/.agentv/targets.yaml b/.agentv/targets.yaml index b76bfc74..165ae4db 100644 --- a/.agentv/targets.yaml +++ b/.agentv/targets.yaml @@ -56,9 +56,10 @@ targets: # Claude via Z.ai provider (GLM models). Requires cc-mirror: # npx cc-mirror quick --provider zai --api-key "$Z_AI_API_KEY" + # Alternative: set Z_AI_API_KEY in ~/.cc-mirror/claude-zai/config/settings.json # See https://github.com/numman-ali/cc-mirror - name: claude-zai - provider: claude-cli + provider: cc-mirror executable: claude-zai grader_target: grader diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index ce34eae7..d17d40f1 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -1,3 +1,5 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { homedir } from 'node:os'; import path from 'node:path'; import { z } from 'zod'; @@ -1037,6 +1039,19 @@ export function resolveTargetDefinition( ...base, config: resolvePiCliConfig(parsed, env, evalFilePath), }; + case 'cc-mirror': { + const variantName = + typeof parsed.variant === 'string' ? parsed.variant : parsed.name; + // If executable is explicitly set, use it; otherwise auto-discover from variant.json + if (!parsed.executable) { + parsed.executable = resolveCcMirrorBinaryPath(variantName); + } + return { + kind: 'claude-cli', + ...base, + config: resolveClaudeConfig(parsed, env, evalFilePath), + }; + } case 'claude': case 'claude-code': case 'claude-cli': @@ -2028,6 +2043,34 @@ function resolveClaudeConfig( }; } +/** + * Resolve the binary path for a cc-mirror variant. + * Reads ~/.cc-mirror//variant.json → binaryPath. + */ +function resolveCcMirrorBinaryPath(variant: string): string { + const variantJsonPath = path.join(homedir(), '.cc-mirror', variant, 'variant.json'); + if (!existsSync(variantJsonPath)) { + throw new Error( + `cc-mirror variant "${variant}": ${variantJsonPath} not found. ` + + `Install the variant or set "executable" explicitly.`, + ); + } + let parsed: { binaryPath?: string }; + try { + parsed = JSON.parse(readFileSync(variantJsonPath, 'utf8')); + } catch (e) { + throw new Error( + `cc-mirror variant "${variant}": failed to parse ${variantJsonPath}: ${(e as Error).message}`, + ); + } + if (typeof parsed.binaryPath !== 'string' || parsed.binaryPath.trim().length === 0) { + throw new Error( + `cc-mirror variant "${variant}": ${variantJsonPath} missing "binaryPath" field`, + ); + } + return parsed.binaryPath; +} + function normalizeClaudeLogFormat(value: unknown): 'summary' | 'json' | undefined { if (value === undefined || value === null) { return undefined; diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index 57c392c9..92b19fe0 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -114,6 +114,7 @@ export const PROVIDER_ALIASES: readonly string[] = [ 'pi', // alias for "pi-coding-agent" 'claude-code', // alias for "claude" (legacy) + 'cc-mirror', // alias for "claude-cli" (auto-discovers binary from ~/.cc-mirror//) 'bedrock', // legacy/future support 'vertex', // legacy/future support ] as const; diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index 0c1b4ddf..9506a902 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -179,6 +179,8 @@ const CLAUDE_SETTINGS = new Set([ 'max_budget_usd', ]); +const CC_MIRROR_SETTINGS = new Set([...CLAUDE_SETTINGS, 'variant']); + function getKnownSettings(provider: string): Set | null { const normalizedProvider = provider.toLowerCase(); switch (normalizedProvider) { @@ -204,6 +206,8 @@ function getKnownSettings(provider: string): Set | null { case 'copilot': case 'copilot-cli': return COPILOT_CLI_SETTINGS; + case 'cc-mirror': + return CC_MIRROR_SETTINGS; case 'claude': case 'claude-code': case 'claude-cli': diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index c5fd8861..7088c30f 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -747,6 +747,43 @@ describe('resolveTargetDefinition', () => { expect(target.config.executable).toBe('claude-zai'); }); + it('cc-mirror with explicit executable resolves to claude-cli kind', () => { + const target = resolveTargetDefinition( + { + name: 'claude-zai', + provider: 'cc-mirror', + executable: '/usr/local/bin/claude-zai', + }, + {}, + ); + + expect(target.kind).toBe('claude-cli'); + if (target.kind !== 'claude-cli') { + throw new Error('expected claude-cli target'); + } + + expect(target.config.executable).toBe('/usr/local/bin/claude-zai'); + }); + + it('cc-mirror with explicit variant and executable', () => { + const target = resolveTargetDefinition( + { + name: 'my-mirror', + provider: 'cc-mirror', + variant: 'claude-zai', + executable: '/opt/bin/zai', + }, + {}, + ); + + expect(target.kind).toBe('claude-cli'); + if (target.kind !== 'claude-cli') { + throw new Error('expected claude-cli target'); + } + + expect(target.config.executable).toBe('/opt/bin/zai'); + }); + it('resolves copilot-cli as its own provider kind', () => { const target = resolveTargetDefinition( { From 3926ba190ed8303d8225cc1f25a6906bc95df601 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 14 Apr 2026 01:28:32 +0000 Subject: [PATCH 2/6] style: fix lint issues in cc-mirror helper Co-Authored-By: Claude Opus 4.6 --- packages/core/src/evaluation/providers/targets.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index d17d40f1..7f19bfa5 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -1040,8 +1040,7 @@ export function resolveTargetDefinition( config: resolvePiCliConfig(parsed, env, evalFilePath), }; case 'cc-mirror': { - const variantName = - typeof parsed.variant === 'string' ? parsed.variant : parsed.name; + const variantName = typeof parsed.variant === 'string' ? parsed.variant : parsed.name; // If executable is explicitly set, use it; otherwise auto-discover from variant.json if (!parsed.executable) { parsed.executable = resolveCcMirrorBinaryPath(variantName); @@ -2051,8 +2050,7 @@ function resolveCcMirrorBinaryPath(variant: string): string { const variantJsonPath = path.join(homedir(), '.cc-mirror', variant, 'variant.json'); if (!existsSync(variantJsonPath)) { throw new Error( - `cc-mirror variant "${variant}": ${variantJsonPath} not found. ` + - `Install the variant or set "executable" explicitly.`, + `cc-mirror variant "${variant}": ${variantJsonPath} not found. Install the variant or set "executable" explicitly.`, ); } let parsed: { binaryPath?: string }; From 438c7e87f5dc237055b1d5dda94ce3479320aee1 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 14 Apr 2026 01:30:20 +0000 Subject: [PATCH 3/6] feat(targets): add generic cc-mirror target with env-var variant selection Adds a cc-mirror target that delegates to CC_MIRROR_VARIANT env var, supporting kimi, zai, minimax, and other cc-mirror providers. Co-Authored-By: Claude Opus 4.6 --- .agentv/targets.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.agentv/targets.yaml b/.agentv/targets.yaml index 165ae4db..410396ec 100644 --- a/.agentv/targets.yaml +++ b/.agentv/targets.yaml @@ -63,6 +63,15 @@ targets: executable: claude-zai grader_target: grader + # Generic cc-mirror target. Set CC_MIRROR_VARIANT to the variant name + # (e.g., kimi, zai, minimax). Requires cc-mirror setup first: + # npx cc-mirror quick --provider --api-key "$API_KEY" + # See https://github.com/numman-ali/cc-mirror + - name: cc-mirror + provider: cc-mirror + variant: ${{ CC_MIRROR_VARIANT }} + grader_target: grader + - name: claude-sdk provider: claude-sdk grader_target: grader From 85c750b118b085eff0be20ad7fbc3f4151c2de85 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 14 Apr 2026 01:33:25 +0000 Subject: [PATCH 4/6] docs(targets): clarify cc-mirror variant vs provider naming Co-Authored-By: Claude Opus 4.6 --- .agentv/targets.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.agentv/targets.yaml b/.agentv/targets.yaml index 410396ec..067a75bf 100644 --- a/.agentv/targets.yaml +++ b/.agentv/targets.yaml @@ -63,9 +63,9 @@ targets: executable: claude-zai grader_target: grader - # Generic cc-mirror target. Set CC_MIRROR_VARIANT to the variant name - # (e.g., kimi, zai, minimax). Requires cc-mirror setup first: - # npx cc-mirror quick --provider --api-key "$API_KEY" + # Generic cc-mirror target. Set CC_MIRROR_VARIANT to the installed variant + # name (the directory under ~/.cc-mirror/, e.g. claude-zai, my-kimi). + # Setup: npx cc-mirror quick --provider --name --api-key "$KEY" # See https://github.com/numman-ali/cc-mirror - name: cc-mirror provider: cc-mirror From 3eddb6f215cd374bc0505270b071e35bf48f21cc Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 14 Apr 2026 01:37:29 +0000 Subject: [PATCH 5/6] docs(targets): add cc-mirror provider documentation Co-Authored-By: Claude Opus 4.6 --- .../docs/docs/targets/coding-agents.mdx | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/apps/web/src/content/docs/docs/targets/coding-agents.mdx b/apps/web/src/content/docs/docs/targets/coding-agents.mdx index 5c18ac08..6ee94a3d 100644 --- a/apps/web/src/content/docs/docs/targets/coding-agents.mdx +++ b/apps/web/src/content/docs/docs/targets/coding-agents.mdx @@ -75,6 +75,40 @@ targets: | `cwd` | No | Working directory (mutually exclusive with workspace_template) | | `grader_target` | Yes | LLM target for evaluation | +## cc-mirror + +[cc-mirror](https://github.com/numman-ali/cc-mirror) creates isolated Claude Code variants that route through alternative providers (Z.ai, Kimi, MiniMax, OpenRouter, etc.). The `cc-mirror` provider alias resolves to `claude-cli` and auto-discovers the binary path from `~/.cc-mirror//variant.json`. + +```yaml +targets: + # Explicit variant with known executable + - name: claude-zai + provider: cc-mirror + executable: claude-zai + grader_target: azure-base + + # Auto-discover binary from variant.json + - name: my-kimi + provider: cc-mirror + grader_target: azure-base +``` + +| Field | Required | Description | +|-------|----------|-------------| +| `executable` | No | CLI binary name or path. When set, used directly (skips variant.json lookup). | +| `variant` | No | Variant name (directory under `~/.cc-mirror/`). Defaults to target `name`. Used to locate `variant.json` when `executable` is not set. | +| `workspace_template` | No | Path to workspace template directory | +| `cwd` | No | Working directory (mutually exclusive with workspace_template) | +| `grader_target` | Yes | LLM target for evaluation | + +Setup a variant first, then reference it by name: + +```bash +npx cc-mirror quick --provider zai --name claude-zai --api-key "$Z_AI_API_KEY" +``` + +Since `cc-mirror` resolves to `claude-cli`, all Claude target fields (model, system_prompt, timeout_seconds, etc.) are also supported. + ## Codex CLI ```yaml From a510e5b8055cbdd3c92556cbd47dbb0abed2908d Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 14 Apr 2026 01:43:28 +0000 Subject: [PATCH 6/6] fix(providers): resolve ${{ }} env vars in cc-mirror variant field The variant field was read as a raw string, so ${{ CC_MIRROR_VARIANT }} was passed literally to resolveCcMirrorBinaryPath instead of being resolved to the env var value. Now uses resolveOptionalString. Co-Authored-By: Claude Opus 4.6 --- packages/core/src/evaluation/providers/targets.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 7f19bfa5..d41d7198 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -1040,7 +1040,11 @@ export function resolveTargetDefinition( config: resolvePiCliConfig(parsed, env, evalFilePath), }; case 'cc-mirror': { - const variantName = typeof parsed.variant === 'string' ? parsed.variant : parsed.name; + const variantName = + resolveOptionalString(parsed.variant, env, `${parsed.name} cc-mirror variant`, { + allowLiteral: true, + optionalEnv: true, + }) ?? parsed.name; // If executable is explicitly set, use it; otherwise auto-discover from variant.json if (!parsed.executable) { parsed.executable = resolveCcMirrorBinaryPath(variantName);