Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 13 additions & 4 deletions packages/core/src/evaluation/orchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -731,10 +731,22 @@ export async function runEvaluation(
if (cliWorkspacePath && workspaceMode && workspaceMode !== 'static') {
throw new Error('--workspace-path requires --workspace-mode static when both are provided');
}
const configuredMode = cliWorkspacePath
let configuredMode = cliWorkspacePath
? 'static'
: (workspaceMode ?? suiteWorkspace?.mode ?? (yamlWorkspacePath ? 'static' : 'pooled'));
const configuredStaticPath = cliWorkspacePath ?? yamlWorkspacePath;

// When mode=static with no path and no repos, the static mode is a no-op (nothing to
// skip cloning). Fall back to temp mode so agentv creates a temp directory automatically.
if (configuredMode === 'static' && !configuredStaticPath) {
if (!suiteWorkspace?.repos?.length) {
setupLog('workspace.mode=static with no path and no repos — falling back to temp mode');
configuredMode = 'temp';
} else {
throw new Error('workspace.mode=static requires workspace.path or --workspace-path');
}
}

const useStaticWorkspace = configuredMode === 'static';

// static workspace is incompatible with per_test isolation
Expand All @@ -743,9 +755,6 @@ export async function runEvaluation(
'static workspace mode is incompatible with isolation: per_test. Use isolation: shared (default).',
);
}
if (configuredMode === 'static' && !configuredStaticPath) {
throw new Error('workspace.mode=static requires workspace.path or --workspace-path');
}
if (configuredMode !== 'static' && configuredStaticPath) {
throw new Error('workspace.path requires workspace.mode=static');
}
Expand Down
30 changes: 28 additions & 2 deletions packages/core/test/evaluation/orchestrator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3126,19 +3126,45 @@ fs.writeFileSync(path.join(payload.workspace_path, 'hook.txt'), payload.test_id
expect(results[0].error).toBeUndefined();
});

it('errors when workspaceMode is static without workspace path', async () => {
it('falls back to temp mode when workspaceMode is static with no path and no repos', async () => {
const provider = new SequenceProvider('mock', {
responses: [{ output: [{ role: 'assistant', content: [{ type: 'text', text: 'answer' }] }] }],
});

const results = await runEvaluation({
testFilePath: 'in-memory.yaml',
repoRoot: 'in-memory',
target: baseTarget,
providerFactory: () => provider,
evaluators: evaluatorRegistry,
evalCases: [baseTestCase],
workspaceMode: 'static',
});

expect(results).toHaveLength(1);
expect(results[0].error).toBeUndefined();
});

it('errors when workspaceMode is static without workspace path but with repos', async () => {
const provider = new SequenceProvider('mock', {
responses: [{ output: [{ role: 'assistant', content: [{ type: 'text', text: 'answer' }] }] }],
});

const evalCase = {
...baseTestCase,
workspace: {
repos: [{ source: { type: 'git' as const, url: 'https://example.com/repo.git' } }],
},
};

await expect(
runEvaluation({
testFilePath: 'in-memory.yaml',
repoRoot: 'in-memory',
target: baseTarget,
providerFactory: () => provider,
evaluators: evaluatorRegistry,
evalCases: [baseTestCase],
evalCases: [evalCase],
workspaceMode: 'static',
}),
).rejects.toThrow('workspace.mode=static requires workspace.path or --workspace-path');
Expand Down
Loading