Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ cd ../agentv.worktrees/<type>-<short-desc>
- Subagents for: research, file exploration, running tests, code review.
- For complex problems, throw more subagents at it — parallelize where possible.
- Name subagents descriptively.
- Before declaring a repo change complete or opening/finalizing a PR, spawn a subagent for a final code review pass unless the user explicitly says not to.

### Autonomous Bug Fixes
- When you spot a bug, just fix it. Don't ask for hand-holding.
Expand Down Expand Up @@ -369,18 +370,29 @@ When working on a GitHub issue, **ALWAYS** follow this workflow:

4. **Implement the changes** and commit following the commit convention

5. **Push the branch and create a Pull Request**:
5. **Push regularly and open a draft Pull Request early**:
```bash
git push -u origin <branch-name>
gh pr create --title "<type>(scope): description" --body "Closes #<issue-number>"
gh pr create --draft --title "<type>(scope): description" --body "Closes #<issue-number>"
```
Push incremental commits to the draft PR as you work so progress is visible and recoverable.

6. **Before merging**, ensure:
6. **Before marking the PR ready for review or merging a low-risk change**, ensure:
- **E2E verification completed** (see "Completing Work — E2E Checklist")
- For CLI or other user-facing changes, run at least one manual end-to-end check of the real user flow, not just unit/integration tests.
- A final subagent code review pass has been run and any findings addressed or called out.
- CI pipeline passes (all checks green)
- Code has been reviewed if required
- No merge conflicts with `main`

7. **Only after verification is complete**:
- Mark the draft PR ready for review, or
- Merge directly if the change is low risk and the repo policy allows it

8. **After merge, clean up local state**:
- Delete the local feature branch
- Remove the local worktree created for the issue
- Confirm the primary checkout is back on an up-to-date `main`

The `in-progress` label stays on the issue until the PR is merged and the issue is closed. Do not remove it manually.

**IMPORTANT:** Never push directly to `main`. Always use branches and PRs.
Expand Down
8 changes: 6 additions & 2 deletions apps/cli/src/commands/eval/targets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,9 @@ export async function selectTarget(options: TargetSelectionOptions): Promise<Tar
}

try {
const resolvedTarget = resolveTargetDefinition(targetDefinition, env, testFilePath);
const resolvedTarget = resolveTargetDefinition(targetDefinition, env, testFilePath, {
emitDeprecationWarnings: false,
});
return {
definitions,
resolvedTarget,
Expand Down Expand Up @@ -292,7 +294,9 @@ export async function selectMultipleTargets(
});
} else {
try {
const resolvedTarget = resolveTargetDefinition(targetDefinition, env, testFilePath);
const resolvedTarget = resolveTargetDefinition(targetDefinition, env, testFilePath, {
emitDeprecationWarnings: false,
});
results.push({
definitions,
resolvedTarget,
Expand Down
107 changes: 107 additions & 0 deletions packages/core/src/evaluation/providers/targets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -573,6 +573,108 @@ export interface AgentVResolvedConfig {
readonly temperature: number;
}

export interface TargetDeprecationWarning {
readonly location: string;
readonly message: string;
}

const DEPRECATED_TARGET_CAMEL_CASE_FIELDS = new Map<string, string>([
['providerBatching', 'provider_batching'],
['subagentModeAllowed', 'subagent_mode_allowed'],
['fallbackTargets', 'fallback_targets'],
['resourceName', 'endpoint'],
['baseUrl', 'base_url'],
['apiKey', 'api_key'],
['deploymentName', 'model'],
['thinkingBudget', 'thinking_budget'],
['maxTokens', 'max_output_tokens'],
['apiFormat', 'api_format'],
['timeoutSeconds', 'timeout_seconds'],
['logDir', 'log_dir'],
['logDirectory', 'log_directory'],
['logFormat', 'log_format'],
['logOutputFormat', 'log_output_format'],
['systemPrompt', 'system_prompt'],
['maxTurns', 'max_turns'],
['maxBudgetUsd', 'max_budget_usd'],
['dryRun', 'dry_run'],
['subagentRoot', 'subagent_root'],
['filesFormat', 'files_format'],
['attachmentsFormat', 'attachments_format'],
['cliUrl', 'cli_url'],
['cliPath', 'cli_path'],
['githubToken', 'github_token'],
['sessionDir', 'session_dir'],
['sessionId', 'session_id'],
['sessionStateDir', 'session_state_dir'],
['maxRetries', 'max_retries'],
['retryInitialDelayMs', 'retry_initial_delay_ms'],
['retryMaxDelayMs', 'retry_max_delay_ms'],
['retryBackoffFactor', 'retry_backoff_factor'],
['retryStatusCodes', 'retry_status_codes'],
]);

const DEPRECATED_HEALTHCHECK_CAMEL_CASE_FIELDS = new Map<string, string>([
['timeoutSeconds', 'timeout_seconds'],
]);

function collectDeprecatedCamelCaseWarnings(
value: unknown,
location: string,
aliases: ReadonlyMap<string, string>,
): TargetDeprecationWarning[] {
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
return [];
}

const warnings: TargetDeprecationWarning[] = [];
for (const [camelCaseField, snakeCaseField] of aliases) {
if (Object.prototype.hasOwnProperty.call(value, camelCaseField)) {
warnings.push({
location: `${location}.${camelCaseField}`,
message: `Deprecated camelCase field '${camelCaseField}' in targets.yaml. Use '${snakeCaseField}' instead.`,
});
}
}

return warnings;
}

export function findDeprecatedCamelCaseTargetWarnings(
target: unknown,
location: string,
): readonly TargetDeprecationWarning[] {
const warnings = collectDeprecatedCamelCaseWarnings(
target,
location,
DEPRECATED_TARGET_CAMEL_CASE_FIELDS,
);

if (typeof target !== 'object' || target === null || Array.isArray(target)) {
return warnings;
}

const healthcheck = (target as { healthcheck?: unknown }).healthcheck;
warnings.push(
...collectDeprecatedCamelCaseWarnings(
healthcheck,
`${location}.healthcheck`,
DEPRECATED_HEALTHCHECK_CAMEL_CASE_FIELDS,
),
);

return warnings;
}

function emitDeprecatedCamelCaseTargetWarnings(definition: TargetDefinition): void {
for (const warning of findDeprecatedCamelCaseTargetWarnings(
definition,
`target "${definition.name}"`,
)) {
console.warn(`Warning: ${warning.message}`);
}
}

/**
* Healthcheck configuration type derived from CliHealthcheckSchema.
* Supports both HTTP and command-based healthchecks.
Expand Down Expand Up @@ -797,7 +899,12 @@ export function resolveTargetDefinition(
definition: TargetDefinition,
env: EnvLookup = process.env,
evalFilePath?: string,
options?: { readonly emitDeprecationWarnings?: boolean },
): ResolvedTarget {
if (options?.emitDeprecationWarnings !== false) {
emitDeprecatedCamelCaseTargetWarnings(definition);
}

const parsed = BASE_TARGET_SCHEMA.parse(definition);
if (parsed.workspace_template !== undefined || parsed.workspaceTemplate !== undefined) {
throw new Error(
Expand Down
2 changes: 2 additions & 0 deletions packages/core/src/evaluation/providers/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,8 @@ export interface TargetDefinition {
// Provider batching
readonly provider_batching?: boolean | undefined;
readonly providerBatching?: boolean | undefined;
readonly subagent_mode_allowed?: boolean | undefined;
readonly subagentModeAllowed?: boolean | undefined;
// Azure fields
readonly endpoint?: string | unknown | undefined;
readonly base_url?: string | unknown | undefined;
Expand Down
15 changes: 14 additions & 1 deletion packages/core/src/evaluation/validation/targets-validator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@ import { readFile } from 'node:fs/promises';
import path from 'node:path';
import { parse } from 'yaml';

import { CLI_PLACEHOLDERS, COMMON_TARGET_SETTINGS } from '../providers/targets.js';
import {
CLI_PLACEHOLDERS,
COMMON_TARGET_SETTINGS,
findDeprecatedCamelCaseTargetWarnings,
} from '../providers/targets.js';
import { KNOWN_PROVIDERS, PROVIDER_ALIASES } from '../providers/types.js';
import type { ValidationError, ValidationResult } from './types.js';

Expand Down Expand Up @@ -522,6 +526,15 @@ export async function validateTargetsFile(filePath: string): Promise<ValidationR
continue;
}

for (const warning of findDeprecatedCamelCaseTargetWarnings(target, location)) {
errors.push({
severity: 'warning',
filePath: absolutePath,
location: warning.location,
message: warning.message,
});
}

// Required field: name
const name = target.name;
if (typeof name !== 'string' || name.trim().length === 0) {
Expand Down
28 changes: 27 additions & 1 deletion packages/core/test/evaluation/providers/targets.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { beforeEach, describe, expect, it, mock } from 'bun:test';
import { beforeEach, describe, expect, it, mock, spyOn } from 'bun:test';
const generateTextMock = mock(async () => ({
text: 'ok',
reasoningText: undefined,
Expand Down Expand Up @@ -677,6 +677,32 @@ describe('resolveTargetDefinition', () => {
).toThrow(/workspace_template has been removed/i);
});

it('warns when deprecated camelCase target fields are used', () => {
const warnSpy = spyOn(console, 'warn').mockImplementation(() => {});

resolveTargetDefinition(
{
name: 'deprecated-camel-case',
provider: 'openai',
baseUrl: '${{ OPENAI_BASE_URL }}',
apiKey: '${{ OPENAI_API_KEY }}',
model: '${{ OPENAI_MODEL }}',
maxTokens: 100,
},
{
OPENAI_BASE_URL: 'https://api.openai.com/v1',
OPENAI_API_KEY: 'test-key',
OPENAI_MODEL: 'gpt-5-mini',
},
);

expect(warnSpy).toHaveBeenCalledTimes(3);
expect(warnSpy.mock.calls[0]?.[0]).toContain("Deprecated camelCase field 'baseUrl'");
expect(warnSpy.mock.calls[1]?.[0]).toContain("Deprecated camelCase field 'apiKey'");
expect(warnSpy.mock.calls[2]?.[0]).toContain("Deprecated camelCase field 'maxTokens'");
warnSpy.mockRestore();
});

it('resolves agentv target with model and default temperature', () => {
const target = resolveTargetDefinition(
{
Expand Down
53 changes: 53 additions & 0 deletions packages/core/test/evaluation/validation/targets-validator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,4 +87,57 @@ describe('validateTargetsFile', () => {
),
).toBe(false);
});

it('warns on deprecated camelCase target aliases', async () => {
const filePath = path.join(tempDir, 'camel-case-aliases.yaml');
await writeFile(
filePath,
`targets:
- name: codex-target
provider: codex
timeoutSeconds: 30
logDir: ./logs
systemPrompt: Be precise.
- name: cli-target
provider: cli
command: echo {PROMPT}
healthcheck:
command: echo ok
timeoutSeconds: 5
`,
);

const result = await validateTargetsFile(filePath);
const warnings = result.errors.filter((error) => error.severity === 'warning');

expect(result.valid).toBe(true);
expect(
warnings.some(
(warning) =>
warning.location === 'targets[0].timeoutSeconds' &&
warning.message.includes("Use 'timeout_seconds' instead"),
),
).toBe(true);
expect(
warnings.some(
(warning) =>
warning.location === 'targets[0].logDir' &&
warning.message.includes("Use 'log_dir' instead"),
),
).toBe(true);
expect(
warnings.some(
(warning) =>
warning.location === 'targets[0].systemPrompt' &&
warning.message.includes("Use 'system_prompt' instead"),
),
).toBe(true);
expect(
warnings.some(
(warning) =>
warning.location === 'targets[1].healthcheck.timeoutSeconds' &&
warning.message.includes("Use 'timeout_seconds' instead"),
),
).toBe(true);
});
});
Loading