From a2a6712fb2c6d7bc21f9a56ccbcf38257b55c70e Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 13 Apr 2026 21:36:42 +0000 Subject: [PATCH 1/4] fix(validate): catch workspace setup_error causes at validate-time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds validateWorkspacePaths() to detect two classes of errors that surface as setup_error at runtime but are statically checkable: 1. workspace.hooks.*.command — script arguments that look like relative file paths (./foo, ../bar/setup.mjs) are resolved using the same cwd precedence as the runtime (hook.cwd ?? workspaceFileDir ?? evalDir) and checked for existence. 2. workspace.template — the template path is resolved and checked. Both checks apply to inline workspace configs and to paths inside external workspace files (workspace: "path.yaml"). The external file existence case is already caught by eval-validator; the new validator focuses on the internal hook script paths and template that were previously only caught at runtime. Closes #1078 Co-Authored-By: Claude Sonnet 4.6 --- .../src/commands/validate/validate-files.ts | 15 +- .../core/src/evaluation/validation/index.ts | 1 + .../validation/workspace-path-validator.ts | 161 +++++++++++++++ .../workspace-path-validator.test.ts | 186 ++++++++++++++++++ 4 files changed, 358 insertions(+), 5 deletions(-) create mode 100644 packages/core/src/evaluation/validation/workspace-path-validator.ts create mode 100644 packages/core/test/evaluation/validation/workspace-path-validator.test.ts diff --git a/apps/cli/src/commands/validate/validate-files.ts b/apps/cli/src/commands/validate/validate-files.ts index b00c27b47..f3840c76b 100644 --- a/apps/cli/src/commands/validate/validate-files.ts +++ b/apps/cli/src/commands/validate/validate-files.ts @@ -9,6 +9,7 @@ import { validateEvalFile, validateFileReferences, validateTargetsFile, + validateWorkspacePaths, } from '@agentv/core/evaluation/validation'; import fg from 'fast-glob'; @@ -47,14 +48,18 @@ async function validateSingleFile(filePath: string): Promise { if (fileType === 'eval') { result = await validateEvalFile(absolutePath); - // Also validate file references for eval files + // Also validate file references and workspace paths for eval files if (result.valid || result.errors.filter((e) => e.severity === 'error').length === 0) { - const fileRefErrors = await validateFileReferences(absolutePath); - if (fileRefErrors.length > 0) { + const [fileRefErrors, workspaceErrors] = await Promise.all([ + validateFileReferences(absolutePath), + validateWorkspacePaths(absolutePath), + ]); + const extraErrors = [...fileRefErrors, ...workspaceErrors]; + if (extraErrors.length > 0) { result = { ...result, - errors: [...result.errors, ...fileRefErrors], - valid: result.valid && fileRefErrors.filter((e) => e.severity === 'error').length === 0, + errors: [...result.errors, ...extraErrors], + valid: result.valid && extraErrors.filter((e) => e.severity === 'error').length === 0, }; } } diff --git a/packages/core/src/evaluation/validation/index.ts b/packages/core/src/evaluation/validation/index.ts index 9347974f3..82286775b 100644 --- a/packages/core/src/evaluation/validation/index.ts +++ b/packages/core/src/evaluation/validation/index.ts @@ -7,6 +7,7 @@ export { validateEvalFile } from './eval-validator.js'; export { validateTargetsFile } from './targets-validator.js'; export { validateConfigFile } from './config-validator.js'; export { validateFileReferences } from './file-reference-validator.js'; +export { validateWorkspacePaths } from './workspace-path-validator.js'; export type { FileType, ValidationSeverity, diff --git a/packages/core/src/evaluation/validation/workspace-path-validator.ts b/packages/core/src/evaluation/validation/workspace-path-validator.ts new file mode 100644 index 000000000..5a15d4ab9 --- /dev/null +++ b/packages/core/src/evaluation/validation/workspace-path-validator.ts @@ -0,0 +1,161 @@ +import { access } from 'node:fs/promises'; +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { parse } from 'yaml'; + +import type { ValidationError } from './types.js'; + +type JsonValue = string | number | boolean | null | JsonObject | JsonArray; +type JsonObject = { readonly [key: string]: JsonValue }; +type JsonArray = readonly JsonValue[]; + +function isObject(value: unknown): value is JsonObject { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** + * Validate that workspace file references and hook script paths in eval files exist. + * + * Catches two classes of errors that surface as `setup_error` at runtime but are + * detectable statically: + * + * 1. `workspace: "path/to/file.yaml"` — the external workspace file must exist. + * 2. `workspace.hooks.*.command` — script arguments that look like relative file + * paths (start with `./`/`../` or carry a script extension) must resolve to + * existing files using the same cwd precedence the runtime uses: + * `hook.cwd ?? workspaceFileDir ?? evalDir` + * 3. `workspace.template` — the template path must exist. + */ +export async function validateWorkspacePaths( + evalFilePath: string, +): Promise { + const errors: ValidationError[] = []; + const absolutePath = path.resolve(evalFilePath); + const evalDir = path.dirname(absolutePath); + + let parsed: unknown; + try { + const content = await readFile(absolutePath, 'utf8'); + parsed = parse(content); + } catch { + // Parse errors are already caught by eval-validator + return errors; + } + + if (!isObject(parsed)) return errors; + + const workspaceRaw = parsed.workspace; + if (workspaceRaw === undefined || workspaceRaw === null) return errors; + + if (typeof workspaceRaw === 'string') { + // External workspace file reference + const workspaceFilePath = path.resolve(evalDir, workspaceRaw); + if (!(await fileExists(workspaceFilePath))) { + errors.push({ + severity: 'error', + filePath: absolutePath, + location: 'workspace', + message: `Workspace file not found: ${workspaceRaw} (resolved to ${workspaceFilePath})`, + }); + return errors; + } + + // File exists — also check paths inside the external workspace file + try { + const wsContent = await readFile(workspaceFilePath, 'utf8'); + const wsParsed = parse(wsContent); + if (isObject(wsParsed)) { + const wsDir = path.dirname(workspaceFilePath); + await validateWorkspaceObject(wsParsed, wsDir, absolutePath, 'workspace', errors); + } + } catch { + // YAML parse errors in the referenced file are not this validator's concern + } + } else if (isObject(workspaceRaw)) { + await validateWorkspaceObject(workspaceRaw, evalDir, absolutePath, 'workspace', errors); + } + + return errors; +} + +async function validateWorkspaceObject( + obj: JsonObject, + baseDir: string, + evalFilePath: string, + location: string, + errors: ValidationError[], +): Promise { + // Check template path + const template = obj.template; + if (typeof template === 'string') { + const templatePath = path.isAbsolute(template) ? template : path.resolve(baseDir, template); + if (!(await fileExists(templatePath))) { + errors.push({ + severity: 'error', + filePath: evalFilePath, + location: `${location}.template`, + message: `Template path not found: ${template} (resolved to ${templatePath})`, + }); + } + } + + // Check hook script paths + const hooks = obj.hooks; + if (!isObject(hooks)) return; + + for (const hookName of ['before_all', 'before_each', 'after_each', 'after_all'] as const) { + const hook = hooks[hookName]; + if (!isObject(hook)) continue; + + // Resolve hook cwd the same way the runtime does: + // config.cwd (resolved against baseDir) ?? baseDir + const hookCwdRaw = typeof hook.cwd === 'string' ? hook.cwd : undefined; + const hookCwd = hookCwdRaw + ? path.isAbsolute(hookCwdRaw) + ? hookCwdRaw + : path.resolve(baseDir, hookCwdRaw) + : baseDir; + + // Support both `command` (canonical) and `script` (deprecated alias) + const command = hook.command ?? hook.script; + if (!Array.isArray(command)) continue; + + for (const arg of command) { + if (typeof arg !== 'string') continue; + if (!looksLikeFilePath(arg)) continue; + + const resolved = path.isAbsolute(arg) ? arg : path.resolve(hookCwd, arg); + if (!(await fileExists(resolved))) { + errors.push({ + severity: 'error', + filePath: evalFilePath, + location: `${location}.hooks.${hookName}.command`, + message: `Hook script not found: ${arg} (resolved to ${resolved})`, + }); + } + } + } +} + +/** + * Heuristic: does this command argument look like a file path rather than a + * system binary name? + * + * Detects: + * - Explicit relative paths: `./foo`, `../bar/baz` + * - Script-extension arguments: `setup.mjs`, `scripts/init.sh` + */ +function looksLikeFilePath(arg: string): boolean { + if (arg.startsWith('./') || arg.startsWith('../')) return true; + const scriptExtensions = ['.mjs', '.cjs', '.js', '.ts', '.sh', '.py', '.rb', '.pl']; + return scriptExtensions.some((ext) => arg.endsWith(ext)); +} + +async function fileExists(filePath: string): Promise { + try { + await access(filePath); + return true; + } catch { + return false; + } +} diff --git a/packages/core/test/evaluation/validation/workspace-path-validator.test.ts b/packages/core/test/evaluation/validation/workspace-path-validator.test.ts new file mode 100644 index 000000000..6457514d0 --- /dev/null +++ b/packages/core/test/evaluation/validation/workspace-path-validator.test.ts @@ -0,0 +1,186 @@ +import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { validateWorkspacePaths } from '../../../src/evaluation/validation/workspace-path-validator.js'; + +const minimalEvalPrefix = `tests: + - id: t1 + criteria: Goal + input: hello +`; + +describe('validateWorkspacePaths', () => { + let tempDir: string; + + beforeAll(async () => { + tempDir = path.join(os.tmpdir(), `agentv-test-workspace-${Date.now()}`); + await mkdir(tempDir, { recursive: true }); + }); + + afterAll(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('returns no errors when workspace field is absent', async () => { + const filePath = path.join(tempDir, 'no-workspace.yaml'); + await writeFile(filePath, minimalEvalPrefix); + const errors = await validateWorkspacePaths(filePath); + expect(errors).toHaveLength(0); + }); + + it('returns no errors when workspace file reference exists', async () => { + const wsFilePath = path.join(tempDir, 'workspace.yaml'); + await writeFile(wsFilePath, 'template: ~\n'); + + const evalFilePath = path.join(tempDir, 'eval-ws-ref-ok.yaml'); + await writeFile(evalFilePath, `${minimalEvalPrefix}workspace: workspace.yaml\n`); + + const errors = await validateWorkspacePaths(evalFilePath); + expect(errors).toHaveLength(0); + }); + + it('errors when workspace file reference does not exist', async () => { + const evalFilePath = path.join(tempDir, 'eval-ws-ref-missing.yaml'); + await writeFile(evalFilePath, `${minimalEvalPrefix}workspace: missing-workspace.yaml\n`); + + const errors = await validateWorkspacePaths(evalFilePath); + expect(errors).toHaveLength(1); + expect(errors[0]?.severity).toBe('error'); + expect(errors[0]?.location).toBe('workspace'); + expect(errors[0]?.message).toContain('Workspace file not found'); + expect(errors[0]?.message).toContain('missing-workspace.yaml'); + }); + + it('errors when workspace.template does not exist (inline workspace)', async () => { + const evalFilePath = path.join(tempDir, 'eval-inline-template.yaml'); + await writeFile( + evalFilePath, + `${minimalEvalPrefix}workspace:\n template: nonexistent-template\n`, + ); + + const errors = await validateWorkspacePaths(evalFilePath); + expect(errors).toHaveLength(1); + expect(errors[0]?.severity).toBe('error'); + expect(errors[0]?.location).toBe('workspace.template'); + expect(errors[0]?.message).toContain('Template path not found'); + }); + + it('returns no errors when workspace.template exists', async () => { + const templateDir = path.join(tempDir, 'my-template'); + await mkdir(templateDir, { recursive: true }); + + const evalFilePath = path.join(tempDir, 'eval-template-ok.yaml'); + await writeFile( + evalFilePath, + `${minimalEvalPrefix}workspace:\n template: my-template\n`, + ); + + const errors = await validateWorkspacePaths(evalFilePath); + expect(errors).toHaveLength(0); + }); + + it('errors when hook before_all command has a missing relative script', async () => { + const evalFilePath = path.join(tempDir, 'eval-hook-missing.yaml'); + await writeFile( + evalFilePath, + `${minimalEvalPrefix}workspace: + hooks: + before_all: + command: + - node + - ../../scripts/setup.mjs +`, + ); + + const errors = await validateWorkspacePaths(evalFilePath); + expect(errors).toHaveLength(1); + expect(errors[0]?.severity).toBe('error'); + expect(errors[0]?.location).toBe('workspace.hooks.before_all.command'); + expect(errors[0]?.message).toContain('setup.mjs'); + }); + + it('returns no errors when hook script exists at resolved path', async () => { + const scriptsDir = path.join(tempDir, 'scripts'); + await mkdir(scriptsDir, { recursive: true }); + const setupScript = path.join(scriptsDir, 'setup.mjs'); + await writeFile(setupScript, 'console.log("setup");'); + + const evalFilePath = path.join(tempDir, 'eval-hook-ok.yaml'); + await writeFile( + evalFilePath, + `${minimalEvalPrefix}workspace: + hooks: + before_all: + command: + - node + - ./scripts/setup.mjs +`, + ); + + const errors = await validateWorkspacePaths(evalFilePath); + expect(errors).toHaveLength(0); + }); + + it('does not flag system binaries (no extension, no relative prefix)', async () => { + const evalFilePath = path.join(tempDir, 'eval-system-binary.yaml'); + await writeFile( + evalFilePath, + `${minimalEvalPrefix}workspace: + hooks: + before_all: + command: + - bash + - -c + - echo hello +`, + ); + + const errors = await validateWorkspacePaths(evalFilePath); + expect(errors).toHaveLength(0); + }); + + it('checks hooks inside external workspace file', async () => { + const wsFilePath = path.join(tempDir, 'ws-with-hooks.yaml'); + await writeFile( + wsFilePath, + `hooks: + before_all: + command: + - node + - ./missing-setup.mjs +`, + ); + + const evalFilePath = path.join(tempDir, 'eval-ws-hooks.yaml'); + await writeFile(evalFilePath, `${minimalEvalPrefix}workspace: ws-with-hooks.yaml\n`); + + const errors = await validateWorkspacePaths(evalFilePath); + expect(errors).toHaveLength(1); + expect(errors[0]?.message).toContain('missing-setup.mjs'); + }); + + it('respects hook cwd for script path resolution', async () => { + const subDir = path.join(tempDir, 'sub'); + await mkdir(subDir, { recursive: true }); + const scriptPath = path.join(subDir, 'run.sh'); + await writeFile(scriptPath, '#!/bin/bash\necho run'); + + const evalFilePath = path.join(tempDir, 'eval-hook-cwd.yaml'); + await writeFile( + evalFilePath, + `${minimalEvalPrefix}workspace: + hooks: + before_all: + cwd: ./sub + command: + - bash + - ./run.sh +`, + ); + + const errors = await validateWorkspacePaths(evalFilePath); + expect(errors).toHaveLength(0); + }); +}); From 7ae997f202892e192a8a6784dbbc4952914f7522 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 13 Apr 2026 21:38:03 +0000 Subject: [PATCH 2/4] style: fix biome formatting in workspace-path-validator test Co-Authored-By: Claude Sonnet 4.6 --- .../evaluation/validation/workspace-path-validator.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/core/test/evaluation/validation/workspace-path-validator.test.ts b/packages/core/test/evaluation/validation/workspace-path-validator.test.ts index 6457514d0..3decf0863 100644 --- a/packages/core/test/evaluation/validation/workspace-path-validator.test.ts +++ b/packages/core/test/evaluation/validation/workspace-path-validator.test.ts @@ -72,10 +72,7 @@ describe('validateWorkspacePaths', () => { await mkdir(templateDir, { recursive: true }); const evalFilePath = path.join(tempDir, 'eval-template-ok.yaml'); - await writeFile( - evalFilePath, - `${minimalEvalPrefix}workspace:\n template: my-template\n`, - ); + await writeFile(evalFilePath, `${minimalEvalPrefix}workspace:\n template: my-template\n`); const errors = await validateWorkspacePaths(evalFilePath); expect(errors).toHaveLength(0); From af256aa324a9e001e933a508f81201cd33931170 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 13 Apr 2026 21:43:23 +0000 Subject: [PATCH 3/4] fix: address code review findings - Combine duplicate node:fs/promises imports into one - Fix JSDoc: "two classes" now correctly lists two, clarify external file note - Remove redundant workspace string existence check (eval-validator owns it) - Add command arg index to error location: command[1] instead of command - Add .bash and .zsh to script extension heuristic - Add tests for: after_each/after_all hooks, script alias, external workspace template Co-Authored-By: Claude Sonnet 4.6 --- .../validation/workspace-path-validator.ts | 38 +++++------ .../workspace-path-validator.test.ts | 67 ++++++++++++++++--- 2 files changed, 75 insertions(+), 30 deletions(-) diff --git a/packages/core/src/evaluation/validation/workspace-path-validator.ts b/packages/core/src/evaluation/validation/workspace-path-validator.ts index 5a15d4ab9..cc99c452c 100644 --- a/packages/core/src/evaluation/validation/workspace-path-validator.ts +++ b/packages/core/src/evaluation/validation/workspace-path-validator.ts @@ -1,5 +1,4 @@ -import { access } from 'node:fs/promises'; -import { readFile } from 'node:fs/promises'; +import { access, readFile } from 'node:fs/promises'; import path from 'node:path'; import { parse } from 'yaml'; @@ -14,17 +13,20 @@ function isObject(value: unknown): value is JsonObject { } /** - * Validate that workspace file references and hook script paths in eval files exist. + * Validate that workspace template and hook script paths in eval files exist. * * Catches two classes of errors that surface as `setup_error` at runtime but are * detectable statically: * - * 1. `workspace: "path/to/file.yaml"` — the external workspace file must exist. + * 1. `workspace.template` — the template path must exist. * 2. `workspace.hooks.*.command` — script arguments that look like relative file * paths (start with `./`/`../` or carry a script extension) must resolve to * existing files using the same cwd precedence the runtime uses: * `hook.cwd ?? workspaceFileDir ?? evalDir` - * 3. `workspace.template` — the template path must exist. + * + * When `workspace` is a string path to an external file, that file is also read + * and its template/hook paths are checked relative to the workspace file's directory. + * (The workspace file's existence itself is already checked by eval-validator.) */ export async function validateWorkspacePaths( evalFilePath: string, @@ -48,19 +50,9 @@ export async function validateWorkspacePaths( if (workspaceRaw === undefined || workspaceRaw === null) return errors; if (typeof workspaceRaw === 'string') { - // External workspace file reference + // External workspace file — existence is already checked by eval-validator. + // Read and check template/hook paths inside the external file. const workspaceFilePath = path.resolve(evalDir, workspaceRaw); - if (!(await fileExists(workspaceFilePath))) { - errors.push({ - severity: 'error', - filePath: absolutePath, - location: 'workspace', - message: `Workspace file not found: ${workspaceRaw} (resolved to ${workspaceFilePath})`, - }); - return errors; - } - - // File exists — also check paths inside the external workspace file try { const wsContent = await readFile(workspaceFilePath, 'utf8'); const wsParsed = parse(wsContent); @@ -69,7 +61,7 @@ export async function validateWorkspacePaths( await validateWorkspaceObject(wsParsed, wsDir, absolutePath, 'workspace', errors); } } catch { - // YAML parse errors in the referenced file are not this validator's concern + // File missing or invalid YAML — eval-validator already reports this } } else if (isObject(workspaceRaw)) { await validateWorkspaceObject(workspaceRaw, evalDir, absolutePath, 'workspace', errors); @@ -107,8 +99,9 @@ async function validateWorkspaceObject( const hook = hooks[hookName]; if (!isObject(hook)) continue; - // Resolve hook cwd the same way the runtime does: + // Resolve hook cwd the same way yaml-parser does at parse time: // config.cwd (resolved against baseDir) ?? baseDir + // baseDir = workspaceFileDir for external workspace files, evalDir for inline. const hookCwdRaw = typeof hook.cwd === 'string' ? hook.cwd : undefined; const hookCwd = hookCwdRaw ? path.isAbsolute(hookCwdRaw) @@ -120,7 +113,8 @@ async function validateWorkspaceObject( const command = hook.command ?? hook.script; if (!Array.isArray(command)) continue; - for (const arg of command) { + for (let i = 0; i < command.length; i++) { + const arg = command[i]; if (typeof arg !== 'string') continue; if (!looksLikeFilePath(arg)) continue; @@ -129,7 +123,7 @@ async function validateWorkspaceObject( errors.push({ severity: 'error', filePath: evalFilePath, - location: `${location}.hooks.${hookName}.command`, + location: `${location}.hooks.${hookName}.command[${i}]`, message: `Hook script not found: ${arg} (resolved to ${resolved})`, }); } @@ -147,7 +141,7 @@ async function validateWorkspaceObject( */ function looksLikeFilePath(arg: string): boolean { if (arg.startsWith('./') || arg.startsWith('../')) return true; - const scriptExtensions = ['.mjs', '.cjs', '.js', '.ts', '.sh', '.py', '.rb', '.pl']; + const scriptExtensions = ['.mjs', '.cjs', '.js', '.ts', '.sh', '.bash', '.zsh', '.py', '.rb', '.pl']; return scriptExtensions.some((ext) => arg.endsWith(ext)); } diff --git a/packages/core/test/evaluation/validation/workspace-path-validator.test.ts b/packages/core/test/evaluation/validation/workspace-path-validator.test.ts index 3decf0863..7735b528e 100644 --- a/packages/core/test/evaluation/validation/workspace-path-validator.test.ts +++ b/packages/core/test/evaluation/validation/workspace-path-validator.test.ts @@ -30,7 +30,7 @@ describe('validateWorkspacePaths', () => { expect(errors).toHaveLength(0); }); - it('returns no errors when workspace file reference exists', async () => { + it('returns no errors when workspace file reference exists (with no internal paths)', async () => { const wsFilePath = path.join(tempDir, 'workspace.yaml'); await writeFile(wsFilePath, 'template: ~\n'); @@ -41,16 +41,13 @@ describe('validateWorkspacePaths', () => { expect(errors).toHaveLength(0); }); - it('errors when workspace file reference does not exist', async () => { + it('returns no errors when workspace file is missing (eval-validator owns that check)', async () => { const evalFilePath = path.join(tempDir, 'eval-ws-ref-missing.yaml'); await writeFile(evalFilePath, `${minimalEvalPrefix}workspace: missing-workspace.yaml\n`); + // eval-validator already catches the missing file error; this validator silently skips const errors = await validateWorkspacePaths(evalFilePath); - expect(errors).toHaveLength(1); - expect(errors[0]?.severity).toBe('error'); - expect(errors[0]?.location).toBe('workspace'); - expect(errors[0]?.message).toContain('Workspace file not found'); - expect(errors[0]?.message).toContain('missing-workspace.yaml'); + expect(errors).toHaveLength(0); }); it('errors when workspace.template does not exist (inline workspace)', async () => { @@ -94,7 +91,7 @@ describe('validateWorkspacePaths', () => { const errors = await validateWorkspacePaths(evalFilePath); expect(errors).toHaveLength(1); expect(errors[0]?.severity).toBe('error'); - expect(errors[0]?.location).toBe('workspace.hooks.before_all.command'); + expect(errors[0]?.location).toBe('workspace.hooks.before_all.command[1]'); expect(errors[0]?.message).toContain('setup.mjs'); }); @@ -158,6 +155,19 @@ describe('validateWorkspacePaths', () => { expect(errors[0]?.message).toContain('missing-setup.mjs'); }); + it('checks template inside external workspace file', async () => { + const wsFilePath = path.join(tempDir, 'ws-with-template.yaml'); + await writeFile(wsFilePath, 'template: ./missing-template\n'); + + const evalFilePath = path.join(tempDir, 'eval-ws-template.yaml'); + await writeFile(evalFilePath, `${minimalEvalPrefix}workspace: ws-with-template.yaml\n`); + + const errors = await validateWorkspacePaths(evalFilePath); + expect(errors).toHaveLength(1); + expect(errors[0]?.location).toBe('workspace.template'); + expect(errors[0]?.message).toContain('missing-template'); + }); + it('respects hook cwd for script path resolution', async () => { const subDir = path.join(tempDir, 'sub'); await mkdir(subDir, { recursive: true }); @@ -180,4 +190,45 @@ describe('validateWorkspacePaths', () => { const errors = await validateWorkspacePaths(evalFilePath); expect(errors).toHaveLength(0); }); + + it('checks after_each and after_all hooks too', async () => { + const evalFilePath = path.join(tempDir, 'eval-other-hooks.yaml'); + await writeFile( + evalFilePath, + `${minimalEvalPrefix}workspace: + hooks: + after_each: + command: + - node + - ./missing-after-each.mjs + after_all: + command: + - node + - ./missing-after-all.mjs +`, + ); + + const errors = await validateWorkspacePaths(evalFilePath); + expect(errors).toHaveLength(2); + expect(errors.some((e) => e.message.includes('missing-after-each.mjs'))).toBe(true); + expect(errors.some((e) => e.message.includes('missing-after-all.mjs'))).toBe(true); + }); + + it('supports deprecated script alias', async () => { + const evalFilePath = path.join(tempDir, 'eval-script-alias.yaml'); + await writeFile( + evalFilePath, + `${minimalEvalPrefix}workspace: + hooks: + before_all: + script: + - node + - ./missing-via-alias.mjs +`, + ); + + const errors = await validateWorkspacePaths(evalFilePath); + expect(errors).toHaveLength(1); + expect(errors[0]?.message).toContain('missing-via-alias.mjs'); + }); }); From d239a1f01398a1675e051158d9227b30c81cd62f Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 13 Apr 2026 21:44:32 +0000 Subject: [PATCH 4/4] style: fix biome line-length for scriptExtensions array Co-Authored-By: Claude Sonnet 4.6 --- .../validation/workspace-path-validator.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/core/src/evaluation/validation/workspace-path-validator.ts b/packages/core/src/evaluation/validation/workspace-path-validator.ts index cc99c452c..a58795682 100644 --- a/packages/core/src/evaluation/validation/workspace-path-validator.ts +++ b/packages/core/src/evaluation/validation/workspace-path-validator.ts @@ -141,7 +141,18 @@ async function validateWorkspaceObject( */ function looksLikeFilePath(arg: string): boolean { if (arg.startsWith('./') || arg.startsWith('../')) return true; - const scriptExtensions = ['.mjs', '.cjs', '.js', '.ts', '.sh', '.bash', '.zsh', '.py', '.rb', '.pl']; + const scriptExtensions = [ + '.mjs', + '.cjs', + '.js', + '.ts', + '.sh', + '.bash', + '.zsh', + '.py', + '.rb', + '.pl', + ]; return scriptExtensions.some((ext) => arg.endsWith(ext)); }