Add ability to choose which eval --smoke runs

README.md

@@ -56,16 +56,16 @@ The argument is the experiment filename without `.ts`. This resolves to `experim
 | Flag | Description |
 |------|-------------|
 | `--dry` | Preview what would run without executing. No API calls, no cost. |
-| `--smoke` | Quick setup verification. Picks the first eval alphabetically, runs once per model. |
+| `--smoke [name]` | Quick setup verification: run one eval, one run per model. Optionally specify eval folder name (default: first alphabetically). |
 | `--force` | Ignore cached fingerprints and re-run everything. Only applies when running all experiments. |
 
 Flags work with both modes:
 
 ```bash
 npx @vercel/agent-eval --dry          # preview all experiments
 npx @vercel/agent-eval cc --dry       # preview a single experiment
-npx @vercel/agent-eval --smoke        # smoke test all experiments
-npx @vercel/agent-eval cc --smoke     # smoke test one experiment
+npx @vercel/agent-eval --smoke        # smoke test all experiments (first eval alphabetically)
+npx @vercel/agent-eval cc --smoke=my-eval   # smoke test using a specific eval folder
 ```
 
 ### Other commands

packages/agent-eval/src/cli.test.ts

@@ -134,6 +134,53 @@ describe('CLI', () => {
       expect(result.exitCode).toBe(0);
     });
 
+    it('--smoke=name runs the specified eval folder', () => {
+      const projectDir = join(TEST_DIR, 'smoke-name-project');
+      const experimentsDir = join(projectDir, 'experiments');
+      mkdirSync(experimentsDir, { recursive: true });
+
+      const configContent = `export default { agent: 'claude-code' };`;
+      writeFileSync(join(experimentsDir, 'cc.ts'), configContent);
+
+      const evalsDir = join(projectDir, 'evals');
+      mkdirSync(evalsDir);
+
+      for (const evalName of ['beta-eval', 'alpha-eval']) {
+        const fixture = join(evalsDir, evalName);
+        mkdirSync(fixture);
+        writeFileSync(join(fixture, 'PROMPT.md'), 'Test task');
+        writeFileSync(join(fixture, 'EVAL.ts'), 'test code');
+        writeFileSync(join(fixture, 'package.json'), JSON.stringify({ type: 'module' }));
+      }
+
+      const result = runCli(['cc', '--smoke=beta-eval', '--dry'], projectDir);
+      expect(result.stdout).toContain('SMOKE TEST');
+      expect(result.stdout).toContain('beta-eval');
+      expect(result.stdout).not.toContain('alpha-eval');
+      expect(result.exitCode).toBe(0);
+    });
+
+    it('--smoke=invalid-name exits with error', () => {
+      const projectDir = join(TEST_DIR, 'smoke-invalid-project');
+      const experimentsDir = join(projectDir, 'experiments');
+      mkdirSync(experimentsDir, { recursive: true });
+
+      writeFileSync(join(experimentsDir, 'cc.ts'), `export default { agent: 'claude-code' };`);
+
+      const evalsDir = join(projectDir, 'evals');
+      mkdirSync(evalsDir);
+      const fixture = join(evalsDir, 'only-eval');
+      mkdirSync(fixture);
+      writeFileSync(join(fixture, 'PROMPT.md'), 'Test');
+      writeFileSync(join(fixture, 'EVAL.ts'), 'test');
+      writeFileSync(join(fixture, 'package.json'), JSON.stringify({ type: 'module' }));
+
+      const result = runCli(['cc', '--smoke=no-such-eval', '--dry'], projectDir);
+      expect(result.stderr).toContain('not found');
+      expect(result.stderr).toContain('no-such-eval');
+      expect(result.exitCode).toBe(1);
+    });
+
     it('shows error when no valid fixtures found', () => {
       // Create project structure matching convention
       const projectDir = join(TEST_DIR, 'empty-project');

packages/agent-eval/src/cli.ts

@@ -59,7 +59,10 @@ function resolveConfigPath(input: string): string {
 /**
  * Run experiment command handler
  */
-async function runExperimentCommand(configInput: string, options: { dry?: boolean; smoke?: boolean }) {
+async function runExperimentCommand(
+  configInput: string,
+  options: { dry?: boolean; smoke?: boolean | string }
+) {
   try {
     const configPath = resolveConfigPath(configInput);
     const absoluteConfigPath = resolve(process.cwd(), configPath);
@@ -72,8 +75,6 @@ async function runExperimentCommand(configInput: string, options: { dry?: boolea
     console.log(chalk.blue(`Loading config from ${configPath}...`));
     const config = await loadConfig(absoluteConfigPath);
 
-    // Discover evals - infer from config file location
-    // Config at project/experiments/foo.ts -> evals at project/evals/
     const projectDir = dirname(dirname(absoluteConfigPath));
     const evalsDir = resolve(projectDir, 'evals');
     if (!existsSync(evalsDir)) {
@@ -106,8 +107,18 @@ async function runExperimentCommand(configInput: string, options: { dry?: boolea
       process.exit(1);
     }
 
-    // Smoke mode: pick first eval alphabetically, override runs to 1
-    const smokeEvalNames = options.smoke ? [evalNames.sort()[0]] : evalNames;
+    // Smoke mode: use --smoke=name if provided, else first eval alphabetically; override runs to 1
+    const smokeEvalNames = options.smoke
+      ? typeof options.smoke === 'string'
+        ? (() => {
+            if (!availableNames.includes(options.smoke)) {
+              console.error(chalk.red(`Eval "${options.smoke}" not found. Available: ${availableNames.join(', ')}`));
+              process.exit(1);
+            }
+            return [options.smoke];
+          })()
+        : [evalNames.sort()[0]]
+      : evalNames;
     const smokeRuns = options.smoke ? 1 : config.runs;
 
     if (options.smoke) {
@@ -181,7 +192,7 @@ async function runExperimentCommand(configInput: string, options: { dry?: boolea
         apiKey,
         resultsDir,
         experimentName,
-        smoke: options.smoke,
+        smoke: !!options.smoke,
         onProgress: createConsoleProgressHandler({
           experimentName,
           model,
@@ -278,7 +289,10 @@ program
  * and classification. Used by both `run-all` subcommand and the default
  * (no-args) invocation.
  */
-async function runAllCommand(experimentArgs: string[], options: { dry?: boolean; force?: boolean; smoke?: boolean; ackFailures?: boolean }) {
+async function runAllCommand(
+  experimentArgs: string[],
+  options: { dry?: boolean; force?: boolean; smoke?: boolean | string; ackFailures?: boolean }
+) {
     try {
       const projectDir = process.cwd();
       const experimentsDir = resolve(projectDir, 'experiments');
@@ -290,7 +304,7 @@ async function runAllCommand(experimentArgs: string[], options: { dry?: boolean;
         process.exit(1);
       }
       if (!existsSync(evalsDir)) {
-        console.error(chalk.red('evals/ directory not found'));
+        console.error(chalk.red(`Evals directory not found: ${evalsDir}`));
         process.exit(1);
       }
 
@@ -332,6 +346,12 @@ async function runAllCommand(experimentArgs: string[], options: { dry?: boolean;
         process.exit(1);
       }
 
+      const availableNames = fixtures.map((f) => f.name);
+      if (options.smoke && typeof options.smoke === 'string' && !availableNames.includes(options.smoke)) {
+        console.error(chalk.red(`Eval "${options.smoke}" not found. Available: ${availableNames.join(', ')}`));
+        process.exit(1);
+      }
+
       // --- Dry run: collect info and print a single summary table ---
       if (options.dry) {
         interface DryRunInfo { name: string; toRun: string[]; cached: number; total: number }
@@ -350,7 +370,6 @@ async function runAllCommand(experimentArgs: string[], options: { dry?: boolean;
           }
 
           const models = Array.isArray(config.model) ? config.model : [config.model];
-          const availableNames = fixtures.map((f) => f.name);
           let evalNames: string[];
           try {
             evalNames = resolveEvalNames(config.evals, availableNames);
@@ -359,7 +378,7 @@ async function runAllCommand(experimentArgs: string[], options: { dry?: boolean;
           }
 
           if (options.smoke) {
-            evalNames = [evalNames.sort()[0]];
+            evalNames = typeof options.smoke === 'string' ? [options.smoke] : [evalNames.sort()[0]];
           }
 
           for (const model of models) {
@@ -458,7 +477,6 @@ async function runAllCommand(experimentArgs: string[], options: { dry?: boolean;
         }
 
         const models = Array.isArray(config.model) ? config.model : [config.model];
-        const availableNames = fixtures.map((f) => f.name);
         let evalNames: string[];
         try {
           evalNames = resolveEvalNames(config.evals, availableNames);
@@ -467,7 +485,7 @@ async function runAllCommand(experimentArgs: string[], options: { dry?: boolean;
         }
 
         if (options.smoke) {
-          evalNames = [evalNames.sort()[0]];
+          evalNames = typeof options.smoke === 'string' ? [options.smoke] : [evalNames.sort()[0]];
         }
 
         const agent = getAgent(config.agent);
@@ -531,7 +549,7 @@ async function runAllCommand(experimentArgs: string[], options: { dry?: boolean;
               resultsDir,
               experimentName,
               fingerprints,
-              smoke: options.smoke,
+              smoke: !!options.smoke,
               onProgress,
               rateLimiter,
             });
@@ -656,7 +674,7 @@ program
   .argument('[experiments...]', 'Experiment names or glob patterns (default: all)')
   .option('--dry', 'Preview what would run without executing')
   .option('--force', 'Ignore fingerprints, re-run everything')
-  .option('--smoke', 'Run 1 eval per experiment for sanity checking')
+  .option('--smoke [name]', 'Run 1 eval per experiment for sanity checking; optionally specify eval folder name')
   .option('--ack-failures', 'Keep non-model failures (infra/timeout) as final results instead of deleting them')
   .action(runAllCommand);
 
@@ -670,10 +688,10 @@ program
 program
   .argument('[config]', 'Experiment name (e.g., "cc") or path. Omit to run all experiments.')
   .option('--dry', 'Preview what would run without executing')
-  .option('--smoke', 'Run a single eval to verify setup (API keys, model IDs, sandbox)')
+  .option('--smoke [name]', 'Run a single eval to verify setup; optionally specify eval folder name (default: first alphabetically)')
   .option('--force', 'Ignore fingerprints, re-run everything (only applies when running all)')
   .option('--ack-failures', 'Keep non-model failures (infra/timeout) as final results instead of deleting them')
-  .action(async (configInput: string | undefined, options: { dry?: boolean; smoke?: boolean; force?: boolean; ackFailures?: boolean }) => {
+  .action(async (configInput: string | undefined, options: { dry?: boolean; smoke?: boolean | string; force?: boolean; ackFailures?: boolean }) => {
     if (!configInput) {
       await runAllCommand([], options);
       return;