diff --git a/apps/web/src/content/docs/docs/guides/workspace-pool.mdx b/apps/web/src/content/docs/docs/guides/workspace-pool.mdx index 58e024e2b..ea0b33e30 100644 --- a/apps/web/src/content/docs/docs/guides/workspace-pool.mdx +++ b/apps/web/src/content/docs/docs/guides/workspace-pool.mdx @@ -169,6 +169,8 @@ Instead of duplicating workspace configuration across eval files, you can refere workspace: ./path/to/workspace.yaml ``` +The external file should contain the workspace config object directly, not a nested `workspace:` key. + The path is resolved relative to the eval file's directory. Relative paths **inside** the workspace file (template, repo source paths) resolve from the workspace file's own directory. This pattern is especially valuable with pooling: a single `workspace.yaml` guarantees all eval files that reference it produce the same fingerprint and share the same pool. diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 377c719c3..0a9332c3d 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -749,7 +749,23 @@ async function resolveWorkspaceConfig( } // Resolve paths relative to the workspace file's directory const workspaceFileDir = path.dirname(workspaceFilePath); - return parseWorkspaceConfig(parsed, workspaceFileDir); + const resolvedWorkspace = parseWorkspaceConfig(parsed, workspaceFileDir); + if (resolvedWorkspace) { + return resolvedWorkspace; + } + + const parsedObject = parsed as Record; + if ('workspace' in parsedObject && isJsonObject(parsedObject.workspace)) { + throw new Error( + [ + `Invalid workspace file format: ${workspaceFilePath}`, + 'External workspace files must contain the workspace config object directly.', + 'Remove the top-level "workspace:" wrapper.', + ].join(' '), + ); + } + + return undefined; } return parseWorkspaceConfig(raw, evalFileDir); } diff --git a/packages/core/test/evaluation/workspace-config-parsing.test.ts b/packages/core/test/evaluation/workspace-config-parsing.test.ts index 3bc6703e2..5498e0077 100644 --- a/packages/core/test/evaluation/workspace-config-parsing.test.ts +++ b/packages/core/test/evaluation/workspace-config-parsing.test.ts @@ -578,6 +578,39 @@ tests: ); }); + it('should throw a clear error when external workspace file wraps config under workspace', async () => { + const wsDir = path.join(testDir, 'wrapped-workspace'); + await mkdir(wsDir, { recursive: true }); + + const workspaceFile = path.join(wsDir, 'workspace.yaml'); + await writeFile( + workspaceFile, + ` +workspace: + hooks: + after_each: + reset: fast +`, + ); + + const evalFile = path.join(testDir, 'wrapped-workspace-eval.yaml'); + await writeFile( + evalFile, + ` +workspace: ./wrapped-workspace/workspace.yaml + +tests: + - id: wrapped-workspace + input: "Do something" + criteria: "Should work" +`, + ); + + await expect(loadTests(evalFile, testDir)).rejects.toThrow( + /External workspace files must contain the workspace config object directly.*Remove the top-level "workspace:" wrapper/, + ); + }); + it('should allow per-case workspace override with external suite workspace', async () => { const wsDir = path.join(testDir, 'override-shared'); await mkdir(wsDir, { recursive: true });