From 035825e2f582f2f4f1145a91ac384ec36550a59b Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sun, 5 Apr 2026 01:02:09 +0000 Subject: [PATCH 1/3] fix(core): remove camelCase target aliases --- .../core/src/evaluation/providers/targets.ts | 304 +++++++++--------- .../core/src/evaluation/providers/types.ts | 36 --- .../validation/targets-validator.ts | 69 +--- .../evaluation/providers/cli-schema.test.ts | 45 +-- .../providers/fallback-targets.test.ts | 21 +- .../test/evaluation/providers/targets.test.ts | 42 ++- .../validation/targets-validator.test.ts | 41 +-- 7 files changed, 238 insertions(+), 320 deletions(-) diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index d820d8ba0..35f92ed20 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -3,6 +3,41 @@ import { z } from 'zod'; import type { EnvLookup, TargetDefinition } from './types.js'; +const CLI_TARGET_CAMEL_CASE_ALIASES = new Map([ + ['filesFormat', 'files_format'], + ['attachmentsFormat', 'attachments_format'], + ['workspaceTemplate', 'workspace_template'], + ['timeoutSeconds', 'timeout_seconds'], + ['cliVerbose', 'cli_verbose'], + ['keepTempFiles', 'keep_temp_files'], + ['keepOutputFiles', 'keep_output_files'], + ['providerBatching', 'provider_batching'], +]); + +const CLI_HEALTHCHECK_CAMEL_CASE_ALIASES = new Map([ + ['timeoutSeconds', 'timeout_seconds'], +]); + +function rejectCamelCaseAliases( + value: unknown, + ctx: z.RefinementCtx, + aliases: ReadonlyMap, +): void { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return; + } + + for (const [camelCaseField, snakeCaseField] of aliases) { + if (Object.prototype.hasOwnProperty.call(value, camelCaseField)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [camelCaseField], + message: `camelCase field '${camelCaseField}' is no longer supported in targets.yaml. Use '${snakeCaseField}' instead.`, + }); + } + } +} + // --------------------------------------------------------------------------- // Zod Schemas for CLI Provider Configuration // --------------------------------------------------------------------------- @@ -22,8 +57,11 @@ import type { EnvLookup, TargetDefinition } from './types.js'; export const CliHealthcheckHttpInputSchema = z.object({ url: z.string().min(1, 'healthcheck URL is required'), timeout_seconds: z.number().positive().optional(), - timeoutSeconds: z.number().positive().optional(), -}); +}) + .passthrough() + .superRefine((value, ctx) => { + rejectCamelCaseAliases(value, ctx, CLI_HEALTHCHECK_CAMEL_CASE_ALIASES); + }); /** * Loose input schema for command healthcheck configuration. @@ -42,8 +80,11 @@ export const CliHealthcheckCommandInputSchema = z.object({ command: z.string().min(1, 'healthcheck command is required'), cwd: z.string().optional(), timeout_seconds: z.number().positive().optional(), - timeoutSeconds: z.number().positive().optional(), -}); +}) + .passthrough() + .superRefine((value, ctx) => { + rejectCamelCaseAliases(value, ctx, CLI_HEALTHCHECK_CAMEL_CASE_ALIASES); + }); /** * Union for healthcheck input configuration. @@ -89,20 +130,16 @@ export const CliTargetInputSchema = z.object({ // Files format - optional files_format: z.string().optional(), - filesFormat: z.string().optional(), attachments_format: z.string().optional(), - attachmentsFormat: z.string().optional(), // Working directory - optional cwd: z.string().optional(), // Workspace template directory - optional (mutually exclusive with cwd) workspace_template: z.string().optional(), - workspaceTemplate: z.string().optional(), // Timeout in seconds - optional timeout_seconds: z.number().positive().optional(), - timeoutSeconds: z.number().positive().optional(), // Healthcheck configuration - optional healthcheck: CliHealthcheckInputSchema.optional(), @@ -110,21 +147,21 @@ export const CliTargetInputSchema = z.object({ // Verbose mode - optional verbose: z.boolean().optional(), cli_verbose: z.boolean().optional(), - cliVerbose: z.boolean().optional(), // Keep temp files - optional keep_temp_files: z.boolean().optional(), - keepTempFiles: z.boolean().optional(), keep_output_files: z.boolean().optional(), - keepOutputFiles: z.boolean().optional(), // Common target fields grader_target: z.string().optional(), judge_target: z.string().optional(), // backward compat workers: z.number().int().min(1).optional(), provider_batching: z.boolean().optional(), - providerBatching: z.boolean().optional(), -}); +}) + .passthrough() + .superRefine((value, ctx) => { + rejectCamelCaseAliases(value, ctx, CLI_TARGET_CAMEL_CASE_ALIASES); + }); /** * Strict normalized schema for HTTP healthcheck configuration. @@ -225,7 +262,7 @@ export function normalizeCliHealthcheck( targetName: string, evalFilePath?: string, ): CliNormalizedHealthcheck { - const timeoutSeconds = input.timeout_seconds ?? input.timeoutSeconds; + const timeoutSeconds = input.timeout_seconds; const timeoutMs = timeoutSeconds !== undefined ? Math.floor(timeoutSeconds * 1000) : undefined; if ('url' in input && input.url) { @@ -288,12 +325,11 @@ export function normalizeCliTargetInput( const command = resolveString(input.command, env, `${targetName} CLI command`, true); // Coalesce files format variants - const filesFormatSource = - input.files_format ?? input.filesFormat ?? input.attachments_format ?? input.attachmentsFormat; + const filesFormatSource = input.files_format ?? input.attachments_format; const filesFormat = resolveOptionalLiteralString(filesFormatSource); // Resolve workspace template (mutually exclusive with cwd) - const workspaceTemplateSource = input.workspace_template ?? input.workspaceTemplate; + const workspaceTemplateSource = input.workspace_template; let workspaceTemplate = resolveOptionalString( workspaceTemplateSource, env, @@ -333,18 +369,15 @@ export function normalizeCliTargetInput( } // Coalesce timeout variants (seconds -> ms) - const timeoutSeconds = input.timeout_seconds ?? input.timeoutSeconds; + const timeoutSeconds = input.timeout_seconds; const timeoutMs = timeoutSeconds !== undefined ? Math.floor(timeoutSeconds * 1000) : undefined; // Coalesce verbose variants - const verbose = resolveOptionalBoolean(input.verbose ?? input.cli_verbose ?? input.cliVerbose); + const verbose = resolveOptionalBoolean(input.verbose ?? input.cli_verbose); // Coalesce keepTempFiles variants const keepTempFiles = resolveOptionalBoolean( - input.keep_temp_files ?? - input.keepTempFiles ?? - input.keep_output_files ?? - input.keepOutputFiles, + input.keep_temp_files ?? input.keep_output_files, ); // Normalize healthcheck if present @@ -641,6 +674,30 @@ function collectDeprecatedCamelCaseWarnings( return warnings; } +function assertNoDeprecatedCamelCaseTargetFields(definition: TargetDefinition): void { + if (Object.prototype.hasOwnProperty.call(definition, 'workspaceTemplate')) { + throw new Error( + `${definition.name}: target-level workspace_template has been removed. Use eval-level workspace.template.`, + ); + } + + const warning = findDeprecatedCamelCaseTargetWarnings( + definition, + `target "${definition.name}"`, + )[0]; + if (!warning) { + return; + } + + const fieldMatch = warning.message.match(/field '([^']+)'/); + const replacementMatch = warning.message.match(/Use '([^']+)' instead/); + const field = fieldMatch?.[1] ?? 'unknown'; + const replacement = replacementMatch?.[1] ?? 'snake_case'; + throw new Error( + `${warning.location}: camelCase field '${field}' is no longer supported in targets.yaml. Use '${replacement}' instead.`, + ); +} + export function findDeprecatedCamelCaseTargetWarnings( target: unknown, location: string, @@ -667,15 +724,6 @@ export function findDeprecatedCamelCaseTargetWarnings( return warnings; } -function emitDeprecatedCamelCaseTargetWarnings(definition: TargetDefinition): void { - for (const warning of findDeprecatedCamelCaseTargetWarnings( - definition, - `target "${definition.name}"`, - )) { - console.warn(`Warning: ${warning.message}`); - } -} - /** * Healthcheck configuration type derived from CliHealthcheckSchema. * Supports both HTTP and command-based healthchecks. @@ -750,11 +798,8 @@ export type ResolvedTarget = export const COMMON_TARGET_SETTINGS = [ 'use_target', 'provider_batching', - 'providerBatching', 'subagent_mode_allowed', - 'subagentModeAllowed', 'fallback_targets', - 'fallbackTargets', ] as const; const USE_TARGET_ENV_PATTERN = /^\$\{\{\s*([A-Z0-9_]+)\s*\}\}$/i; @@ -768,10 +813,8 @@ const BASE_TARGET_SCHEMA = z judge_target: z.string().optional(), // backward compat workers: z.number().int().min(1).optional(), workspace_template: z.string().optional(), - workspaceTemplate: z.string().optional(), subagent_mode_allowed: z.boolean().optional(), fallback_targets: z.array(z.string().min(1)).optional(), - fallbackTargets: z.array(z.string().min(1)).optional(), }) .passthrough(); @@ -801,23 +844,23 @@ function normalizeAzureApiVersion( function resolveRetryConfig(target: z.infer): RetryConfig | undefined { const maxRetries = resolveOptionalNumber( - target.max_retries ?? target.maxRetries, + target.max_retries, `${target.name} max retries`, ); const initialDelayMs = resolveOptionalNumber( - target.retry_initial_delay_ms ?? target.retryInitialDelayMs, + target.retry_initial_delay_ms, `${target.name} retry initial delay`, ); const maxDelayMs = resolveOptionalNumber( - target.retry_max_delay_ms ?? target.retryMaxDelayMs, + target.retry_max_delay_ms, `${target.name} retry max delay`, ); const backoffFactor = resolveOptionalNumber( - target.retry_backoff_factor ?? target.retryBackoffFactor, + target.retry_backoff_factor, `${target.name} retry backoff factor`, ); const retryableStatusCodes = resolveOptionalNumberArray( - target.retry_status_codes ?? target.retryStatusCodes, + target.retry_status_codes, `${target.name} retry status codes`, ); @@ -909,12 +952,11 @@ export function resolveTargetDefinition( evalFilePath?: string, options?: { readonly emitDeprecationWarnings?: boolean }, ): ResolvedTarget { - if (options?.emitDeprecationWarnings !== false) { - emitDeprecatedCamelCaseTargetWarnings(definition); - } + void options; + assertNoDeprecatedCamelCaseTargetFields(definition); const parsed = BASE_TARGET_SCHEMA.parse(definition); - if (parsed.workspace_template !== undefined || parsed.workspaceTemplate !== undefined) { + if (parsed.workspace_template !== undefined) { throw new Error( `${parsed.name}: target-level workspace_template has been removed. Use eval-level workspace.template.`, ); @@ -930,15 +972,11 @@ export function resolveTargetDefinition( `${parsed.name} provider`, true, ).toLowerCase(); - const providerBatching = resolveOptionalBoolean( - parsed.provider_batching ?? parsed.providerBatching, - ); - const subagentModeAllowed = resolveOptionalBoolean( - parsed.subagent_mode_allowed ?? parsed.subagentModeAllowed, - ); + const providerBatching = resolveOptionalBoolean(parsed.provider_batching); + const subagentModeAllowed = resolveOptionalBoolean(parsed.subagent_mode_allowed); // Shared base fields for all resolved targets - const fallbackTargets = parsed.fallback_targets ?? parsed.fallbackTargets; + const fallbackTargets = parsed.fallback_targets; const base = { name: parsed.name, graderTarget: parsed.grader_target ?? parsed.judge_target, @@ -1102,12 +1140,12 @@ function resolveAzureConfig( target: z.infer, env: EnvLookup, ): AzureResolvedConfig { - const endpointSource = target.endpoint ?? target.resource ?? target.resourceName; - const apiKeySource = target.api_key ?? target.apiKey; - const deploymentSource = target.deployment ?? target.deploymentName ?? target.model; + const endpointSource = target.endpoint ?? target.resource; + const apiKeySource = target.api_key; + const deploymentSource = target.deployment ?? target.model; const versionSource = target.version ?? target.api_version; const temperatureSource = target.temperature; - const maxTokensSource = target.max_output_tokens ?? target.maxTokens; + const maxTokensSource = target.max_output_tokens; const resourceName = resolveString(endpointSource, env, `${target.name} endpoint`); const apiKey = resolveString(apiKeySource, env, `${target.name} api key`); @@ -1145,7 +1183,7 @@ function resolveApiFormat( targetName: string, ): ApiFormat | undefined { const raw = resolveOptionalString( - target.api_format ?? target.apiFormat, + target.api_format, env, `${targetName} api format`, { @@ -1164,11 +1202,11 @@ function resolveOpenAIConfig( target: z.infer, env: EnvLookup, ): OpenAIResolvedConfig { - const endpointSource = target.endpoint ?? target.base_url ?? target.baseUrl; - const apiKeySource = target.api_key ?? target.apiKey; + const endpointSource = target.endpoint ?? target.base_url; + const apiKeySource = target.api_key; const modelSource = target.model ?? target.deployment ?? target.variant; const temperatureSource = target.temperature; - const maxTokensSource = target.max_output_tokens ?? target.maxTokens; + const maxTokensSource = target.max_output_tokens; const baseURL = normalizeOpenAIBaseUrl( resolveOptionalString(endpointSource, env, `${target.name} endpoint`, { @@ -1195,10 +1233,10 @@ function resolveOpenRouterConfig( target: z.infer, env: EnvLookup, ): OpenRouterResolvedConfig { - const apiKeySource = target.api_key ?? target.apiKey; + const apiKeySource = target.api_key; const modelSource = target.model ?? target.deployment ?? target.variant; const temperatureSource = target.temperature; - const maxTokensSource = target.max_output_tokens ?? target.maxTokens; + const maxTokensSource = target.max_output_tokens; const retry = resolveRetryConfig(target); return { @@ -1214,11 +1252,11 @@ function resolveAnthropicConfig( target: z.infer, env: EnvLookup, ): AnthropicResolvedConfig { - const apiKeySource = target.api_key ?? target.apiKey; + const apiKeySource = target.api_key; const modelSource = target.model ?? target.deployment ?? target.variant; const temperatureSource = target.temperature; - const maxTokensSource = target.max_output_tokens ?? target.maxTokens; - const thinkingBudgetSource = target.thinking_budget ?? target.thinkingBudget; + const maxTokensSource = target.max_output_tokens; + const thinkingBudgetSource = target.thinking_budget; const apiKey = resolveString(apiKeySource, env, `${target.name} Anthropic api key`); const model = resolveString(modelSource, env, `${target.name} Anthropic model`); @@ -1238,10 +1276,10 @@ function resolveGeminiConfig( target: z.infer, env: EnvLookup, ): GeminiResolvedConfig { - const apiKeySource = target.api_key ?? target.apiKey; + const apiKeySource = target.api_key; const modelSource = target.model ?? target.deployment ?? target.variant; const temperatureSource = target.temperature; - const maxTokensSource = target.max_output_tokens ?? target.maxTokens; + const maxTokensSource = target.max_output_tokens; const apiKey = resolveString(apiKeySource, env, `${target.name} Google API key`); const model = @@ -1269,17 +1307,12 @@ function resolveCodexConfig( const executableSource = target.executable ?? target.command ?? target.binary; const argsSource = target.args ?? target.arguments; const cwdSource = target.cwd; - const workspaceTemplateSource = target.workspace_template ?? target.workspaceTemplate; - const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds; - const logDirSource = - target.log_dir ?? target.logDir ?? target.log_directory ?? target.logDirectory; + const workspaceTemplateSource = target.workspace_template; + const timeoutSource = target.timeout_seconds; + const logDirSource = target.log_dir ?? target.log_directory; const logFormatSource = - target.log_format ?? - target.logFormat ?? - target.log_output_format ?? - target.logOutputFormat ?? - env.AGENTV_CODEX_LOG_FORMAT; - const systemPromptSource = target.system_prompt ?? target.systemPrompt; + target.log_format ?? target.log_output_format ?? env.AGENTV_CODEX_LOG_FORMAT; + const systemPromptSource = target.system_prompt; const model = resolveOptionalString(modelSource, env, `${target.name} codex model`, { allowLiteral: true, @@ -1365,17 +1398,16 @@ function resolveCopilotSdkConfig( env: EnvLookup, evalFilePath?: string, ): CopilotSdkResolvedConfig { - const cliUrlSource = target.cli_url ?? target.cliUrl; - const cliPathSource = target.cli_path ?? target.cliPath; - const githubTokenSource = target.github_token ?? target.githubToken; + const cliUrlSource = target.cli_url; + const cliPathSource = target.cli_path; + const githubTokenSource = target.github_token; const modelSource = target.model; const cwdSource = target.cwd; - const workspaceTemplateSource = target.workspace_template ?? target.workspaceTemplate; - const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds; - const logDirSource = - target.log_dir ?? target.logDir ?? target.log_directory ?? target.logDirectory; - const logFormatSource = target.log_format ?? target.logFormat; - const systemPromptSource = target.system_prompt ?? target.systemPrompt; + const workspaceTemplateSource = target.workspace_template; + const timeoutSource = target.timeout_seconds; + const logDirSource = target.log_dir ?? target.log_directory; + const logFormatSource = target.log_format; + const systemPromptSource = target.system_prompt; const cliUrl = resolveOptionalString(cliUrlSource, env, `${target.name} copilot-sdk cli URL`, { allowLiteral: true, @@ -1471,12 +1503,11 @@ function resolveCopilotCliConfig( const modelSource = target.model; const argsSource = target.args ?? target.arguments; const cwdSource = target.cwd; - const workspaceTemplateSource = target.workspace_template ?? target.workspaceTemplate; - const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds; - const logDirSource = - target.log_dir ?? target.logDir ?? target.log_directory ?? target.logDirectory; - const logFormatSource = target.log_format ?? target.logFormat; - const systemPromptSource = target.system_prompt ?? target.systemPrompt; + const workspaceTemplateSource = target.workspace_template; + const timeoutSource = target.timeout_seconds; + const logDirSource = target.log_dir ?? target.log_directory; + const logFormatSource = target.log_format; + const systemPromptSource = target.system_prompt; const executable = resolveOptionalString(executableSource, env, `${target.name} copilot-cli executable`, { @@ -1564,17 +1595,16 @@ function resolvePiCodingAgentConfig( evalFilePath?: string, ): PiCodingAgentResolvedConfig { const subproviderSource = target.subprovider; - const modelSource = target.model ?? target.pi_model ?? target.piModel; - const apiKeySource = target.api_key ?? target.apiKey; - const toolsSource = target.tools ?? target.pi_tools ?? target.piTools; - const thinkingSource = target.thinking ?? target.pi_thinking ?? target.piThinking; + const modelSource = target.model ?? target.pi_model; + const apiKeySource = target.api_key; + const toolsSource = target.tools ?? target.pi_tools; + const thinkingSource = target.thinking ?? target.pi_thinking; const cwdSource = target.cwd; - const workspaceTemplateSource = target.workspace_template ?? target.workspaceTemplate; - const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds; - const logDirSource = - target.log_dir ?? target.logDir ?? target.log_directory ?? target.logDirectory; - const logFormatSource = target.log_format ?? target.logFormat; - const systemPromptSource = target.system_prompt ?? target.systemPrompt; + const workspaceTemplateSource = target.workspace_template; + const timeoutSource = target.timeout_seconds; + const logDirSource = target.log_dir ?? target.log_directory; + const logFormatSource = target.log_format; + const systemPromptSource = target.system_prompt; const subprovider = resolveOptionalString( subproviderSource, @@ -1596,7 +1626,7 @@ function resolvePiCodingAgentConfig( optionalEnv: true, }); - const baseUrlSource = target.base_url ?? target.baseUrl ?? target.endpoint; + const baseUrlSource = target.base_url ?? target.endpoint; const baseUrl = resolveOptionalString(baseUrlSource, env, `${target.name} pi base url`, { allowLiteral: true, optionalEnv: true, @@ -1677,17 +1707,16 @@ function resolvePiCliConfig( ): PiCliResolvedConfig { const executableSource = target.executable ?? target.command ?? target.binary; const subproviderSource = target.subprovider; - const modelSource = target.model ?? target.pi_model ?? target.piModel; - const apiKeySource = target.api_key ?? target.apiKey; - const toolsSource = target.tools ?? target.pi_tools ?? target.piTools; - const thinkingSource = target.thinking ?? target.pi_thinking ?? target.piThinking; + const modelSource = target.model ?? target.pi_model; + const apiKeySource = target.api_key; + const toolsSource = target.tools ?? target.pi_tools; + const thinkingSource = target.thinking ?? target.pi_thinking; const cwdSource = target.cwd; - const workspaceTemplateSource = target.workspace_template ?? target.workspaceTemplate; - const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds; - const logDirSource = - target.log_dir ?? target.logDir ?? target.log_directory ?? target.logDirectory; - const logFormatSource = target.log_format ?? target.logFormat; - const systemPromptSource = target.system_prompt ?? target.systemPrompt; + const workspaceTemplateSource = target.workspace_template; + const timeoutSource = target.timeout_seconds; + const logDirSource = target.log_dir ?? target.log_directory; + const logFormatSource = target.log_format; + const systemPromptSource = target.system_prompt; const executable = resolveOptionalString(executableSource, env, `${target.name} pi-cli executable`, { @@ -1712,7 +1741,7 @@ function resolvePiCliConfig( optionalEnv: true, }); - const baseUrlSource = target.base_url ?? target.baseUrl ?? target.endpoint; + const baseUrlSource = target.base_url ?? target.endpoint; const baseUrl = resolveOptionalString(baseUrlSource, env, `${target.name} pi-cli base url`, { allowLiteral: true, optionalEnv: true, @@ -1791,17 +1820,12 @@ function resolveClaudeConfig( ): ClaudeResolvedConfig { const modelSource = target.model; const cwdSource = target.cwd; - const workspaceTemplateSource = target.workspace_template ?? target.workspaceTemplate; - const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds; - const logDirSource = - target.log_dir ?? target.logDir ?? target.log_directory ?? target.logDirectory; + const workspaceTemplateSource = target.workspace_template; + const timeoutSource = target.timeout_seconds; + const logDirSource = target.log_dir ?? target.log_directory; const logFormatSource = - target.log_format ?? - target.logFormat ?? - target.log_output_format ?? - target.logOutputFormat ?? - env.AGENTV_CLAUDE_LOG_FORMAT; - const systemPromptSource = target.system_prompt ?? target.systemPrompt; + target.log_format ?? target.log_output_format ?? env.AGENTV_CLAUDE_LOG_FORMAT; + const systemPromptSource = target.system_prompt; const model = resolveOptionalString(modelSource, env, `${target.name} claude model`, { allowLiteral: true, @@ -1850,18 +1874,10 @@ function resolveClaudeConfig( : undefined; const maxTurns = - typeof target.max_turns === 'number' - ? target.max_turns - : typeof target.maxTurns === 'number' - ? target.maxTurns - : undefined; + typeof target.max_turns === 'number' ? target.max_turns : undefined; const maxBudgetUsd = - typeof target.max_budget_usd === 'number' - ? target.max_budget_usd - : typeof target.maxBudgetUsd === 'number' - ? target.maxBudgetUsd - : undefined; + typeof target.max_budget_usd === 'number' ? target.max_budget_usd : undefined; return { model, @@ -1901,9 +1917,7 @@ function resolveVSCodeConfig( insiders: boolean, evalFilePath?: string, ): VSCodeResolvedConfig { - const workspaceTemplateEnvVar = resolveOptionalLiteralString( - target.workspace_template ?? target.workspaceTemplate, - ); + const workspaceTemplateEnvVar = resolveOptionalLiteralString(target.workspace_template); let workspaceTemplate = workspaceTemplateEnvVar ? resolveOptionalString( workspaceTemplateEnvVar, @@ -1923,9 +1937,9 @@ function resolveVSCodeConfig( const executableSource = target.executable; const waitSource = target.wait; - const dryRunSource = target.dry_run ?? target.dryRun; - const subagentRootSource = target.subagent_root ?? target.subagentRoot; - const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds; + const dryRunSource = target.dry_run; + const subagentRootSource = target.subagent_root; + const timeoutSource = target.timeout_seconds; const defaultCommand = insiders ? 'code-insiders' : 'code'; const executable = @@ -2033,7 +2047,7 @@ function resolveDiscoveredProviderConfig( : `bun run .agentv/providers/${providerKind}.ts {PROMPT}`; // Resolve optional fields using the same patterns as CLI providers - const timeoutSeconds = target.timeout_seconds ?? target.timeoutSeconds; + const timeoutSeconds = target.timeout_seconds; const timeoutMs = resolveTimeoutMs(timeoutSeconds, `${target.name} timeout`); let cwd = resolveOptionalString(target.cwd, env, `${target.name} working directory`, { @@ -2117,10 +2131,10 @@ function resolveCopilotLogConfig( target: z.infer, env: EnvLookup, ): CopilotLogResolvedConfig { - const sessionDirSource = target.session_dir ?? target.sessionDir; - const sessionIdSource = target.session_id ?? target.sessionId; + const sessionDirSource = target.session_dir; + const sessionIdSource = target.session_id; const discoverSource = target.discover; - const sessionStateDirSource = target.session_state_dir ?? target.sessionStateDir; + const sessionStateDirSource = target.session_state_dir; const cwdSource = target.cwd; return { diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index 2897b2a4d..970d254dd 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -292,30 +292,22 @@ export interface TargetDefinition { readonly workers?: number | undefined; // Provider batching readonly provider_batching?: boolean | undefined; - readonly providerBatching?: boolean | undefined; readonly subagent_mode_allowed?: boolean | undefined; - readonly subagentModeAllowed?: boolean | undefined; // Azure fields readonly endpoint?: string | unknown | undefined; readonly base_url?: string | unknown | undefined; - readonly baseUrl?: string | unknown | undefined; readonly resource?: string | unknown | undefined; - readonly resourceName?: string | unknown | undefined; readonly api_key?: string | unknown | undefined; - readonly apiKey?: string | unknown | undefined; readonly deployment?: string | unknown | undefined; - readonly deploymentName?: string | unknown | undefined; readonly model?: string | unknown | undefined; readonly version?: string | unknown | undefined; readonly api_version?: string | unknown | undefined; // Anthropic fields readonly variant?: string | unknown | undefined; readonly thinking_budget?: number | unknown | undefined; - readonly thinkingBudget?: number | unknown | undefined; // Common fields readonly temperature?: number | unknown | undefined; readonly max_output_tokens?: number | unknown | undefined; - readonly maxTokens?: number | unknown | undefined; // Codex fields readonly executable?: string | unknown | undefined; readonly command?: string | unknown | undefined; @@ -324,70 +316,42 @@ export interface TargetDefinition { readonly arguments?: unknown | undefined; readonly cwd?: string | unknown | undefined; readonly timeout_seconds?: number | unknown | undefined; - readonly timeoutSeconds?: number | unknown | undefined; readonly log_dir?: string | unknown | undefined; - readonly logDir?: string | unknown | undefined; readonly log_directory?: string | unknown | undefined; - readonly logDirectory?: string | unknown | undefined; readonly log_format?: string | unknown | undefined; - readonly logFormat?: string | unknown | undefined; readonly log_output_format?: string | unknown | undefined; - readonly logOutputFormat?: string | unknown | undefined; // System prompt (codex, copilot, claude, pi-coding-agent) readonly system_prompt?: string | unknown | undefined; - readonly systemPrompt?: string | unknown | undefined; // Claude Agent SDK fields readonly max_turns?: number | unknown | undefined; - readonly maxTurns?: number | unknown | undefined; readonly max_budget_usd?: number | unknown | undefined; - readonly maxBudgetUsd?: number | unknown | undefined; // Mock fields readonly response?: string | unknown | undefined; - readonly delayMs?: number | unknown | undefined; - readonly delayMinMs?: number | unknown | undefined; - readonly delayMaxMs?: number | unknown | undefined; // VSCode fields readonly wait?: boolean | unknown | undefined; readonly dry_run?: boolean | unknown | undefined; - readonly dryRun?: boolean | unknown | undefined; readonly subagent_root?: string | unknown | undefined; - readonly subagentRoot?: string | unknown | undefined; readonly workspace_template?: string | unknown | undefined; - readonly workspaceTemplate?: string | unknown | undefined; // CLI fields readonly files_format?: string | unknown | undefined; - readonly filesFormat?: string | unknown | undefined; readonly attachments_format?: string | unknown | undefined; - readonly attachmentsFormat?: string | unknown | undefined; readonly env?: unknown | undefined; readonly healthcheck?: unknown | undefined; // Copilot-log fields readonly session_dir?: string | unknown | undefined; - readonly sessionDir?: string | unknown | undefined; readonly session_id?: string | unknown | undefined; - readonly sessionId?: string | unknown | undefined; readonly discover?: string | unknown | undefined; readonly session_state_dir?: string | unknown | undefined; - readonly sessionStateDir?: string | unknown | undefined; // Copilot SDK fields readonly cli_url?: string | unknown | undefined; - readonly cliUrl?: string | unknown | undefined; readonly cli_path?: string | unknown | undefined; - readonly cliPath?: string | unknown | undefined; readonly github_token?: string | unknown | undefined; - readonly githubToken?: string | unknown | undefined; // Retry configuration fields readonly max_retries?: number | unknown | undefined; - readonly maxRetries?: number | unknown | undefined; readonly retry_initial_delay_ms?: number | unknown | undefined; - readonly retryInitialDelayMs?: number | unknown | undefined; readonly retry_max_delay_ms?: number | unknown | undefined; - readonly retryMaxDelayMs?: number | unknown | undefined; readonly retry_backoff_factor?: number | unknown | undefined; - readonly retryBackoffFactor?: number | unknown | undefined; readonly retry_status_codes?: unknown | undefined; - readonly retryStatusCodes?: unknown | undefined; // Fallback targets for provider errors readonly fallback_targets?: readonly string[] | unknown | undefined; - readonly fallbackTargets?: readonly string[] | unknown | undefined; } diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index 22e6a715c..f4cfb7626 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -24,15 +24,10 @@ const COMMON_SETTINGS = new Set(COMMON_TARGET_SETTINGS); const RETRY_SETTINGS = new Set([ 'max_retries', - 'maxRetries', 'retry_initial_delay_ms', - 'retryInitialDelayMs', 'retry_max_delay_ms', - 'retryMaxDelayMs', 'retry_backoff_factor', - 'retryBackoffFactor', 'retry_status_codes', - 'retryStatusCodes', ]); const AZURE_SETTINGS = new Set([ @@ -40,18 +35,14 @@ const AZURE_SETTINGS = new Set([ ...RETRY_SETTINGS, 'endpoint', 'resource', - 'resourceName', 'api_key', - 'apiKey', 'deployment', - 'deploymentName', 'model', 'version', 'api_version', 'api_format', 'temperature', 'max_output_tokens', - 'maxTokens', ]); const OPENAI_SETTINGS = new Set([ @@ -59,57 +50,47 @@ const OPENAI_SETTINGS = new Set([ ...RETRY_SETTINGS, 'endpoint', 'base_url', - 'baseUrl', 'api_key', - 'apiKey', 'model', 'deployment', 'variant', 'api_format', 'temperature', 'max_output_tokens', - 'maxTokens', ]); const OPENROUTER_SETTINGS = new Set([ ...COMMON_SETTINGS, ...RETRY_SETTINGS, 'api_key', - 'apiKey', 'model', 'deployment', 'variant', 'temperature', 'max_output_tokens', - 'maxTokens', ]); const ANTHROPIC_SETTINGS = new Set([ ...COMMON_SETTINGS, ...RETRY_SETTINGS, 'api_key', - 'apiKey', 'model', 'deployment', 'variant', 'temperature', 'max_output_tokens', - 'maxTokens', 'thinking_budget', - 'thinkingBudget', ]); const GEMINI_SETTINGS = new Set([ ...COMMON_SETTINGS, ...RETRY_SETTINGS, 'api_key', - 'apiKey', 'model', 'deployment', 'variant', 'temperature', 'max_output_tokens', - 'maxTokens', ]); const CODEX_SETTINGS = new Set([ @@ -122,41 +103,26 @@ const CODEX_SETTINGS = new Set([ 'arguments', 'cwd', 'timeout_seconds', - 'timeoutSeconds', 'log_dir', - 'logDir', 'log_directory', - 'logDirectory', 'log_format', - 'logFormat', 'log_output_format', - 'logOutputFormat', 'system_prompt', - 'systemPrompt', 'workspace_template', - 'workspaceTemplate', ]); const COPILOT_SDK_SETTINGS = new Set([ ...COMMON_SETTINGS, 'cli_url', - 'cliUrl', 'cli_path', - 'cliPath', 'github_token', - 'githubToken', 'model', 'cwd', 'timeout_seconds', - 'timeoutSeconds', 'log_dir', - 'logDir', 'log_format', - 'logFormat', 'system_prompt', - 'systemPrompt', 'workspace_template', - 'workspaceTemplate', ]); const COPILOT_CLI_SETTINGS = new Set([ @@ -169,37 +135,25 @@ const COPILOT_CLI_SETTINGS = new Set([ 'model', 'cwd', 'timeout_seconds', - 'timeoutSeconds', 'log_dir', - 'logDir', 'log_format', - 'logFormat', 'system_prompt', - 'systemPrompt', 'workspace_template', - 'workspaceTemplate', ]); const VSCODE_SETTINGS = new Set([ ...COMMON_SETTINGS, 'executable', 'workspace_template', - 'workspaceTemplate', 'wait', 'dry_run', - 'dryRun', 'subagent_root', - 'subagentRoot', 'timeout_seconds', - 'timeoutSeconds', ]); const MOCK_SETTINGS = new Set([ ...COMMON_SETTINGS, 'response', - 'delayMs', - 'delayMinMs', - 'delayMaxMs', 'trace', // For testing tool-trajectory evaluator ]); @@ -211,23 +165,14 @@ const CLAUDE_SETTINGS = new Set([ 'model', 'cwd', 'timeout_seconds', - 'timeoutSeconds', 'log_dir', - 'logDir', 'log_directory', - 'logDirectory', 'log_format', - 'logFormat', 'log_output_format', - 'logOutputFormat', 'system_prompt', - 'systemPrompt', 'workspace_template', - 'workspaceTemplate', 'max_turns', - 'maxTurns', 'max_budget_usd', - 'maxBudgetUsd', ]); function getKnownSettings(provider: string): Set | null { @@ -395,15 +340,15 @@ export async function validateTargetsFile(filePath: string): Promise { - it('accepts HTTP and command healthchecks with mixed snake_case/camelCase', () => { - // HTTP with snake_case timeout + it('accepts snake_case healthcheck fields', () => { const httpInput = { url: 'http://localhost:8080/health', timeout_seconds: 30, }; expect(CliHealthcheckInputSchema.safeParse(httpInput).success).toBe(true); - // Command with camelCase properties const commandInput = { command: 'curl http://localhost:8080/health', cwd: '/app', - timeoutSeconds: 30, + timeout_seconds: 30, }; expect(CliHealthcheckInputSchema.safeParse(commandInput).success).toBe(true); }); + it('rejects camelCase healthcheck aliases', () => { + expect( + CliHealthcheckInputSchema.safeParse({ + command: 'curl http://localhost:8080/health', + timeoutSeconds: 30, + }).success, + ).toBe(false); + }); + it('rejects missing required fields', () => { // Empty object (no url or command) expect(CliHealthcheckInputSchema.safeParse({}).success).toBe(false); @@ -42,8 +49,8 @@ describe('CliTargetInputSchema', () => { name: 'test-target', provider: 'cli', command: 'agent run {PROMPT}', - timeoutSeconds: 60, - keepTempFiles: true, + timeout_seconds: 60, + keep_temp_files: true, files_format: '--file {path}', }; @@ -103,6 +110,17 @@ describe('CliTargetInputSchema', () => { const input = { provider: 'cli', command: 'agent run {PROMPT}' }; expect(CliTargetInputSchema.safeParse(input).success).toBe(false); }); + + it('rejects camelCase target aliases', () => { + expect( + CliTargetInputSchema.safeParse({ + name: 'test-target', + provider: 'cli', + command: 'agent run {PROMPT}', + timeoutSeconds: 60, + }).success, + ).toBe(false); + }); }); describe('CliHealthcheckSchema (strict)', () => { @@ -190,10 +208,9 @@ describe('normalizeCliHealthcheck', () => { expect(httpResult.timeoutMs).toBe(30000); } - // Command with camelCase timeout const commandInput = { command: 'health-check.sh', - timeoutSeconds: 5, + timeout_seconds: 5, }; const commandResult = normalizeCliHealthcheck(commandInput, {}, 'test-target'); @@ -304,8 +321,7 @@ describe('normalizeCliTargetInput', () => { } }); - it('accepts attachments_format/attachmentsFormat as alias for files_format', () => { - // snake_case alias + it('accepts attachments_format as alias for files_format', () => { const snakeInput = { name: 'test-target', provider: 'cli', @@ -313,14 +329,5 @@ describe('normalizeCliTargetInput', () => { attachments_format: '--attach {path}', }; expect(normalizeCliTargetInput(snakeInput, {}).filesFormat).toBe('--attach {path}'); - - // camelCase alias - const camelInput = { - name: 'test-target', - provider: 'cli', - command: 'agent {PROMPT}', - attachmentsFormat: '--attach {path}', - }; - expect(normalizeCliTargetInput(camelInput, {}).filesFormat).toBe('--attach {path}'); }); }); diff --git a/packages/core/test/evaluation/providers/fallback-targets.test.ts b/packages/core/test/evaluation/providers/fallback-targets.test.ts index cd4b2b24e..0f3b7a51c 100644 --- a/packages/core/test/evaluation/providers/fallback-targets.test.ts +++ b/packages/core/test/evaluation/providers/fallback-targets.test.ts @@ -144,7 +144,7 @@ describe('resolveTargetDefinition - fallback_targets', () => { expect(resolved.fallbackTargets).toEqual(['azure-llm', 'gemini-flash']); }); - it('resolves fallbackTargets from camelCase field', () => { + it('rejects fallbackTargets camelCase field', () => { const definition = { name: 'test-openai', provider: 'openai', @@ -153,22 +153,9 @@ describe('resolveTargetDefinition - fallback_targets', () => { fallbackTargets: ['backup-1'], }; - const resolved = resolveTargetDefinition(definition, env); - expect(resolved.fallbackTargets).toEqual(['backup-1']); - }); - - it('snake_case takes priority over camelCase', () => { - const definition = { - name: 'test-openai', - provider: 'openai', - api_key: '${{ TEST_KEY }}', - model: '${{ TEST_MODEL }}', - fallback_targets: ['snake-wins'], - fallbackTargets: ['camel-loses'], - }; - - const resolved = resolveTargetDefinition(definition, env); - expect(resolved.fallbackTargets).toEqual(['snake-wins']); + expect(() => resolveTargetDefinition(definition, env)).toThrow( + /fallbackTargets.*fallback_targets/i, + ); }); it('omits fallbackTargets when not specified', () => { diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index b7c2cf865..aac8d357a 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -764,30 +764,24 @@ describe('resolveTargetDefinition', () => { ).toThrow(/workspace_template has been removed/i); }); - it('warns when deprecated camelCase target fields are used', () => { - const warnSpy = spyOn(console, 'warn').mockImplementation(() => {}); - - resolveTargetDefinition( - { - name: 'deprecated-camel-case', - provider: 'openai', - baseUrl: '${{ OPENAI_BASE_URL }}', - apiKey: '${{ OPENAI_API_KEY }}', - model: '${{ OPENAI_MODEL }}', - maxTokens: 100, - }, - { - OPENAI_BASE_URL: 'https://api.openai.com/v1', - OPENAI_API_KEY: 'test-key', - OPENAI_MODEL: 'gpt-5-mini', - }, - ); - - expect(warnSpy).toHaveBeenCalledTimes(3); - expect(warnSpy.mock.calls[0]?.[0]).toContain("Deprecated camelCase field 'baseUrl'"); - expect(warnSpy.mock.calls[1]?.[0]).toContain("Deprecated camelCase field 'apiKey'"); - expect(warnSpy.mock.calls[2]?.[0]).toContain("Deprecated camelCase field 'maxTokens'"); - warnSpy.mockRestore(); + it('rejects camelCase target fields', () => { + expect(() => + resolveTargetDefinition( + { + name: 'deprecated-camel-case', + provider: 'openai', + baseUrl: '${{ OPENAI_BASE_URL }}', + apiKey: '${{ OPENAI_API_KEY }}', + model: '${{ OPENAI_MODEL }}', + maxTokens: 100, + }, + { + OPENAI_BASE_URL: 'https://api.openai.com/v1', + OPENAI_API_KEY: 'test-key', + OPENAI_MODEL: 'gpt-5-mini', + }, + ), + ).toThrow(/baseUrl.*base_url/i); }); it('resolves agentv target with model and default temperature', () => { diff --git a/packages/core/test/evaluation/validation/targets-validator.test.ts b/packages/core/test/evaluation/validation/targets-validator.test.ts index 8733544b2..dde6434e0 100644 --- a/packages/core/test/evaluation/validation/targets-validator.test.ts +++ b/packages/core/test/evaluation/validation/targets-validator.test.ts @@ -88,7 +88,7 @@ describe('validateTargetsFile', () => { ).toBe(false); }); - it('warns on deprecated camelCase target aliases', async () => { + it('rejects camelCase target aliases', async () => { const filePath = path.join(tempDir, 'camel-case-aliases.yaml'); await writeFile( filePath, @@ -108,35 +108,38 @@ describe('validateTargetsFile', () => { ); const result = await validateTargetsFile(filePath); - const warnings = result.errors.filter((error) => error.severity === 'warning'); - expect(result.valid).toBe(true); + expect(result.valid).toBe(false); expect( - warnings.some( - (warning) => - warning.location === 'targets[0].timeoutSeconds' && - warning.message.includes("Use 'timeout_seconds' instead"), + result.errors.some( + (error) => + error.severity === 'error' && + error.location === 'targets[0].timeoutSeconds' && + error.message.includes("Use 'timeout_seconds' instead"), ), ).toBe(true); expect( - warnings.some( - (warning) => - warning.location === 'targets[0].logDir' && - warning.message.includes("Use 'log_dir' instead"), + result.errors.some( + (error) => + error.severity === 'error' && + error.location === 'targets[0].logDir' && + error.message.includes("Use 'log_dir' instead"), ), ).toBe(true); expect( - warnings.some( - (warning) => - warning.location === 'targets[0].systemPrompt' && - warning.message.includes("Use 'system_prompt' instead"), + result.errors.some( + (error) => + error.severity === 'error' && + error.location === 'targets[0].systemPrompt' && + error.message.includes("Use 'system_prompt' instead"), ), ).toBe(true); expect( - warnings.some( - (warning) => - warning.location === 'targets[1].healthcheck.timeoutSeconds' && - warning.message.includes("Use 'timeout_seconds' instead"), + result.errors.some( + (error) => + error.severity === 'error' && + error.location === 'targets[1].healthcheck.timeoutSeconds' && + error.message.includes("Use 'timeout_seconds' instead"), ), ).toBe(true); }); From a33c4127f9c51491e556ba118359e18bf6b783c8 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sun, 5 Apr 2026 02:08:37 +0000 Subject: [PATCH 2/3] refactor(core): drop cli alias mapping --- .../core/src/evaluation/providers/targets.ts | 174 ++++++------------ .../evaluation/providers/cli-schema.test.ts | 19 -- 2 files changed, 61 insertions(+), 132 deletions(-) diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 35f92ed20..69e774b3c 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -3,41 +3,6 @@ import { z } from 'zod'; import type { EnvLookup, TargetDefinition } from './types.js'; -const CLI_TARGET_CAMEL_CASE_ALIASES = new Map([ - ['filesFormat', 'files_format'], - ['attachmentsFormat', 'attachments_format'], - ['workspaceTemplate', 'workspace_template'], - ['timeoutSeconds', 'timeout_seconds'], - ['cliVerbose', 'cli_verbose'], - ['keepTempFiles', 'keep_temp_files'], - ['keepOutputFiles', 'keep_output_files'], - ['providerBatching', 'provider_batching'], -]); - -const CLI_HEALTHCHECK_CAMEL_CASE_ALIASES = new Map([ - ['timeoutSeconds', 'timeout_seconds'], -]); - -function rejectCamelCaseAliases( - value: unknown, - ctx: z.RefinementCtx, - aliases: ReadonlyMap, -): void { - if (typeof value !== 'object' || value === null || Array.isArray(value)) { - return; - } - - for (const [camelCaseField, snakeCaseField] of aliases) { - if (Object.prototype.hasOwnProperty.call(value, camelCaseField)) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: [camelCaseField], - message: `camelCase field '${camelCaseField}' is no longer supported in targets.yaml. Use '${snakeCaseField}' instead.`, - }); - } - } -} - // --------------------------------------------------------------------------- // Zod Schemas for CLI Provider Configuration // --------------------------------------------------------------------------- @@ -54,14 +19,12 @@ function rejectCamelCaseAliases( * timeout_seconds: 30 * ``` */ -export const CliHealthcheckHttpInputSchema = z.object({ - url: z.string().min(1, 'healthcheck URL is required'), - timeout_seconds: z.number().positive().optional(), -}) - .passthrough() - .superRefine((value, ctx) => { - rejectCamelCaseAliases(value, ctx, CLI_HEALTHCHECK_CAMEL_CASE_ALIASES); - }); +export const CliHealthcheckHttpInputSchema = z + .object({ + url: z.string().min(1, 'healthcheck URL is required'), + timeout_seconds: z.number().positive().optional(), + }) + .passthrough(); /** * Loose input schema for command healthcheck configuration. @@ -76,15 +39,13 @@ export const CliHealthcheckHttpInputSchema = z.object({ * timeout_seconds: 10 * ``` */ -export const CliHealthcheckCommandInputSchema = z.object({ - command: z.string().min(1, 'healthcheck command is required'), - cwd: z.string().optional(), - timeout_seconds: z.number().positive().optional(), -}) - .passthrough() - .superRefine((value, ctx) => { - rejectCamelCaseAliases(value, ctx, CLI_HEALTHCHECK_CAMEL_CASE_ALIASES); - }); +export const CliHealthcheckCommandInputSchema = z + .object({ + command: z.string().min(1, 'healthcheck command is required'), + cwd: z.string().optional(), + timeout_seconds: z.number().positive().optional(), + }) + .passthrough(); /** * Union for healthcheck input configuration. @@ -119,49 +80,47 @@ export const CliHealthcheckInputSchema = z.union([ * url: http://localhost:8080/health * ``` */ -export const CliTargetInputSchema = z.object({ - name: z.string().min(1, 'target name is required'), - provider: z - .string() - .refine((p) => p.toLowerCase() === 'cli', { message: "provider must be 'cli'" }), - - // Command - required - command: z.string(), - - // Files format - optional - files_format: z.string().optional(), - attachments_format: z.string().optional(), - - // Working directory - optional - cwd: z.string().optional(), - - // Workspace template directory - optional (mutually exclusive with cwd) - workspace_template: z.string().optional(), - - // Timeout in seconds - optional - timeout_seconds: z.number().positive().optional(), - - // Healthcheck configuration - optional - healthcheck: CliHealthcheckInputSchema.optional(), - - // Verbose mode - optional - verbose: z.boolean().optional(), - cli_verbose: z.boolean().optional(), - - // Keep temp files - optional - keep_temp_files: z.boolean().optional(), - keep_output_files: z.boolean().optional(), - - // Common target fields - grader_target: z.string().optional(), - judge_target: z.string().optional(), // backward compat - workers: z.number().int().min(1).optional(), - provider_batching: z.boolean().optional(), -}) - .passthrough() - .superRefine((value, ctx) => { - rejectCamelCaseAliases(value, ctx, CLI_TARGET_CAMEL_CASE_ALIASES); - }); +export const CliTargetInputSchema = z + .object({ + name: z.string().min(1, 'target name is required'), + provider: z + .string() + .refine((p) => p.toLowerCase() === 'cli', { message: "provider must be 'cli'" }), + + // Command - required + command: z.string(), + + // Files format - optional + files_format: z.string().optional(), + attachments_format: z.string().optional(), + + // Working directory - optional + cwd: z.string().optional(), + + // Workspace template directory - optional (mutually exclusive with cwd) + workspace_template: z.string().optional(), + + // Timeout in seconds - optional + timeout_seconds: z.number().positive().optional(), + + // Healthcheck configuration - optional + healthcheck: CliHealthcheckInputSchema.optional(), + + // Verbose mode - optional + verbose: z.boolean().optional(), + cli_verbose: z.boolean().optional(), + + // Keep temp files - optional + keep_temp_files: z.boolean().optional(), + keep_output_files: z.boolean().optional(), + + // Common target fields + grader_target: z.string().optional(), + judge_target: z.string().optional(), // backward compat + workers: z.number().int().min(1).optional(), + provider_batching: z.boolean().optional(), + }) + .passthrough(); /** * Strict normalized schema for HTTP healthcheck configuration. @@ -376,9 +335,7 @@ export function normalizeCliTargetInput( const verbose = resolveOptionalBoolean(input.verbose ?? input.cli_verbose); // Coalesce keepTempFiles variants - const keepTempFiles = resolveOptionalBoolean( - input.keep_temp_files ?? input.keep_output_files, - ); + const keepTempFiles = resolveOptionalBoolean(input.keep_temp_files ?? input.keep_output_files); // Normalize healthcheck if present const healthcheck = input.healthcheck @@ -843,10 +800,7 @@ function normalizeAzureApiVersion( } function resolveRetryConfig(target: z.infer): RetryConfig | undefined { - const maxRetries = resolveOptionalNumber( - target.max_retries, - `${target.name} max retries`, - ); + const maxRetries = resolveOptionalNumber(target.max_retries, `${target.name} max retries`); const initialDelayMs = resolveOptionalNumber( target.retry_initial_delay_ms, `${target.name} retry initial delay`, @@ -1182,15 +1136,10 @@ function resolveApiFormat( env: EnvLookup, targetName: string, ): ApiFormat | undefined { - const raw = resolveOptionalString( - target.api_format, - env, - `${targetName} api format`, - { - allowLiteral: true, - optionalEnv: true, - }, - ); + const raw = resolveOptionalString(target.api_format, env, `${targetName} api format`, { + allowLiteral: true, + optionalEnv: true, + }); if (raw === undefined) return undefined; if (raw === 'chat' || raw === 'responses') return raw; throw new Error( @@ -1873,8 +1822,7 @@ function resolveClaudeConfig( ? systemPromptSource.trim() : undefined; - const maxTurns = - typeof target.max_turns === 'number' ? target.max_turns : undefined; + const maxTurns = typeof target.max_turns === 'number' ? target.max_turns : undefined; const maxBudgetUsd = typeof target.max_budget_usd === 'number' ? target.max_budget_usd : undefined; diff --git a/packages/core/test/evaluation/providers/cli-schema.test.ts b/packages/core/test/evaluation/providers/cli-schema.test.ts index 4a350651e..acff93e1e 100644 --- a/packages/core/test/evaluation/providers/cli-schema.test.ts +++ b/packages/core/test/evaluation/providers/cli-schema.test.ts @@ -25,15 +25,6 @@ describe('CliHealthcheckInputSchema', () => { expect(CliHealthcheckInputSchema.safeParse(commandInput).success).toBe(true); }); - it('rejects camelCase healthcheck aliases', () => { - expect( - CliHealthcheckInputSchema.safeParse({ - command: 'curl http://localhost:8080/health', - timeoutSeconds: 30, - }).success, - ).toBe(false); - }); - it('rejects missing required fields', () => { // Empty object (no url or command) expect(CliHealthcheckInputSchema.safeParse({}).success).toBe(false); @@ -111,16 +102,6 @@ describe('CliTargetInputSchema', () => { expect(CliTargetInputSchema.safeParse(input).success).toBe(false); }); - it('rejects camelCase target aliases', () => { - expect( - CliTargetInputSchema.safeParse({ - name: 'test-target', - provider: 'cli', - command: 'agent run {PROMPT}', - timeoutSeconds: 60, - }).success, - ).toBe(false); - }); }); describe('CliHealthcheckSchema (strict)', () => { From 2a0376b43490652c8a3628264c7343cb68f1510c Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Sun, 5 Apr 2026 02:33:10 +0000 Subject: [PATCH 3/3] docs(core): update camelCase removal wording --- .../core/src/evaluation/providers/targets.ts | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 69e774b3c..f855e54fd 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -9,8 +9,7 @@ import type { EnvLookup, TargetDefinition } from './types.js'; /** * Loose input schema for HTTP healthcheck configuration. - * Accepts both snake_case (YAML convention) and camelCase (JavaScript convention) - * property names for flexibility in configuration files. + * Accepts raw YAML input before normalization and validation. * * @example * ```yaml @@ -28,8 +27,7 @@ export const CliHealthcheckHttpInputSchema = z /** * Loose input schema for command healthcheck configuration. - * Accepts both snake_case (YAML convention) and camelCase (JavaScript convention) - * property names for flexibility in configuration files. + * Accepts raw YAML input before normalization and validation. * * @example * ```yaml @@ -62,8 +60,7 @@ export const CliHealthcheckInputSchema = z.union([ /** * Loose input schema for CLI target configuration. - * Accepts both snake_case (YAML convention) and camelCase (JavaScript convention) - * property names for maximum flexibility in configuration files. + * Accepts raw YAML input before normalization and validation. * * This schema validates the raw YAML input structure before normalization * and environment variable resolution. Unknown properties are allowed @@ -164,7 +161,7 @@ export const CliHealthcheckSchema = z.union([ /** * Strict normalized schema for CLI target configuration. * This is the final validated shape after environment variable resolution - * and snake_case to camelCase normalization. + * and internal field normalization. * * Uses .strict() to reject unknown properties, ensuring configuration * errors are caught early rather than silently ignored. @@ -206,8 +203,8 @@ export type CliNormalizedConfig = z.infer; export type CliResolvedConfig = Readonly; /** - * Normalizes a healthcheck input from loose (snake_case + camelCase) to - * strict normalized form (camelCase only). Resolves environment variables. + * Normalizes a healthcheck input from raw YAML input to the strict internal + * form used by the CLI provider. Resolves environment variables. * * @param input - The loose healthcheck input from YAML * @param env - Environment variable lookup @@ -262,12 +259,12 @@ export function normalizeCliHealthcheck( } /** - * Normalizes a CLI target input from loose (snake_case + camelCase) to - * strict normalized form (camelCase only). Resolves environment variables. + * Normalizes a CLI target input from raw YAML input to the strict internal + * form used by the CLI provider. Resolves environment variables. * - * This function coalesces snake_case/camelCase variants and resolves - * environment variable references using ${{ VAR_NAME }} syntax. - * snake_case takes precedence over camelCase when both are present (matching YAML convention). + * This function resolves environment variable references using + * ${{ VAR_NAME }} syntax and converts external YAML field names to the + * internal runtime shape. * * @param input - The loose CLI target input from YAML * @param env - Environment variable lookup @@ -623,7 +620,7 @@ function collectDeprecatedCamelCaseWarnings( if (Object.prototype.hasOwnProperty.call(value, camelCaseField)) { warnings.push({ location: `${location}.${camelCaseField}`, - message: `Deprecated camelCase field '${camelCaseField}' in targets.yaml. Use '${snakeCaseField}' instead.`, + message: `camelCase field '${camelCaseField}' is no longer supported in targets.yaml. Use '${snakeCaseField}' instead.`, }); } }