diff --git a/apps/cli/src/commands/workspace/deps.ts b/apps/cli/src/commands/workspace/deps.ts new file mode 100644 index 000000000..21a67c24f --- /dev/null +++ b/apps/cli/src/commands/workspace/deps.ts @@ -0,0 +1,50 @@ +import path from 'node:path'; +import { command, flag, restPositionals, string } from 'cmd-ts'; + +import { scanRepoDeps } from '@agentv/core'; + +import { resolveEvalPaths } from '../eval/shared.js'; + +export const depsCommand = command({ + name: 'deps', + description: 'Scan eval files and list git repo dependencies needed by workspaces', + args: { + evalPaths: restPositionals({ + type: string, + displayName: 'eval-paths', + description: 'Path(s) or glob(s) to evaluation .yaml file(s)', + }), + usedBy: flag({ + long: 'used-by', + description: 'Include list of eval files that reference each repo', + }), + }, + handler: async ({ evalPaths, usedBy }) => { + if (evalPaths.length === 0) { + console.error('Usage: agentv workspace deps '); + process.exit(1); + } + + const cwd = process.cwd(); + const resolvedPaths = await resolveEvalPaths(evalPaths, cwd); + const result = await scanRepoDeps(resolvedPaths); + + // Print errors to stderr + for (const err of result.errors) { + console.error(`warning: ${path.relative(cwd, err.file)}: ${err.message}`); + } + + // Output JSON manifest to stdout (snake_case per wire format convention) + const output = { + repos: result.repos.map((r) => ({ + url: r.url, + ...(r.ref !== undefined && { ref: r.ref }), + ...(r.clone !== undefined && { clone: r.clone }), + ...(r.checkout !== undefined && { checkout: r.checkout }), + ...(usedBy && { used_by: r.usedBy.map((p) => path.relative(cwd, p)) }), + })), + }; + + console.log(JSON.stringify(output, null, 2)); + }, +}); diff --git a/apps/cli/src/commands/workspace/index.ts b/apps/cli/src/commands/workspace/index.ts index 036d83ea8..50dd3c00f 100644 --- a/apps/cli/src/commands/workspace/index.ts +++ b/apps/cli/src/commands/workspace/index.ts @@ -1,6 +1,7 @@ import { subcommands } from 'cmd-ts'; import { cleanCommand } from './clean.js'; +import { depsCommand } from './deps.js'; import { listCommand } from './list.js'; export const workspaceCommand = subcommands({ @@ -9,5 +10,6 @@ export const workspaceCommand = subcommands({ cmds: { list: listCommand, clean: cleanCommand, + deps: depsCommand, }, }); diff --git a/apps/web/src/content/docs/docs/guides/workspace-pool.mdx b/apps/web/src/content/docs/docs/guides/workspace-pool.mdx index 8b06f2b08..4f1018b66 100644 --- a/apps/web/src/content/docs/docs/guides/workspace-pool.mdx +++ b/apps/web/src/content/docs/docs/guides/workspace-pool.mdx @@ -153,6 +153,10 @@ agentv workspace clean # Remove only pools for a specific repo agentv workspace clean --repo github.com/org/my-repo + +# Scan eval files and output a JSON manifest of required git repos +# Useful in CI to determine what to clone before running evals +agentv workspace deps evals/**/*.eval.yaml ``` ## External workspace config diff --git a/apps/web/src/content/docs/docs/targets/configuration.mdx b/apps/web/src/content/docs/docs/targets/configuration.mdx index 113c03122..24dbda058 100644 --- a/apps/web/src/content/docs/docs/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/targets/configuration.mdx @@ -222,6 +222,7 @@ Similar to GitHub Actions checkout conventions, `source` answers "where does thi Pool management commands: - `agentv workspace list` — list all pool entries with size and repo info - `agentv workspace clean` — remove all pool entries +- `agentv workspace deps ` — scan eval files and output a JSON manifest of required git repos (for CI pre-cloning) **Common patterns:** diff --git a/packages/core/src/evaluation/workspace/deps-scanner.ts b/packages/core/src/evaluation/workspace/deps-scanner.ts new file mode 100644 index 000000000..f00991243 --- /dev/null +++ b/packages/core/src/evaluation/workspace/deps-scanner.ts @@ -0,0 +1,176 @@ +/** + * Lightweight scanner that extracts git repo dependencies from eval YAML files + * without performing full test/grader parsing. + * + * Used by `agentv workspace deps` to determine which repos CI needs to clone + * before running evals. + * + * How it works: + * 1. Reads each eval YAML file and parses it + * 2. Extracts `workspace.repos` at suite-level and per-test level + * 3. Resolves external workspace file references (string → file path) + * 4. Deduplicates git repos by (url, ref) + * 5. Returns a flat list of unique repo dependencies + * + * To extend: add support for new workspace source types by adding a branch + * in `extractReposFromWorkspaceRaw()`. + */ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { parse } from 'yaml'; + +import { interpolateEnv } from '../interpolation.js'; +import type { RepoCheckout, RepoClone, RepoSource } from '../types.js'; +import { parseRepoCheckout, parseRepoClone, parseRepoSource } from './repo-config-parser.js'; + +/** A single git repo dependency discovered from eval files. */ +export interface RepoDep { + /** Git clone URL */ + readonly url: string; + /** Checkout ref (branch, tag, SHA). undefined means HEAD. */ + readonly ref: string | undefined; + /** Clone options (depth, filter, sparse) — first-wins on dedup collision */ + readonly clone: RepoClone | undefined; + /** Checkout options (resolve, ancestor) — first-wins on dedup collision */ + readonly checkout: Omit | undefined; + /** Eval files that reference this repo */ + readonly usedBy: string[]; +} + +/** Full output of the deps scanner. */ +export interface DepsScanResult { + readonly repos: readonly RepoDep[]; + /** Files that failed to parse (non-fatal) */ + readonly errors: readonly { file: string; message: string }[]; +} + +/** Normalize a git URL for dedup: strip trailing .git and lowercase the host. */ +function normalizeGitUrl(url: string): string { + let normalized = url.replace(/\.git$/, ''); + // Lowercase the host portion of https:// URLs + try { + const parsed = new URL(normalized); + parsed.hostname = parsed.hostname.toLowerCase(); + normalized = parsed.toString().replace(/\/$/, ''); + } catch { + // Not a valid URL (e.g., SSH shorthand) — use as-is + } + return normalized; +} + +/** + * Scan eval YAML files and collect unique git repo dependencies. + * Non-YAML files and parse errors are collected in `errors` but don't stop scanning. + * + * Dedup strategy: repos are keyed by (normalized URL, ref). On collision, + * clone/checkout options from the first occurrence win — this is intentional + * since the manifest is advisory (CI can override clone options). + */ +export async function scanRepoDeps(evalFilePaths: readonly string[]): Promise { + const seen = new Map(); + const errors: { file: string; message: string }[] = []; + + for (const filePath of evalFilePaths) { + try { + const repos = await extractReposFromEvalFile(filePath); + for (const repo of repos) { + if (repo.source.type !== 'git') continue; + const ref = repo.checkout?.ref; + const key = `${normalizeGitUrl(repo.source.url)}\0${ref ?? ''}`; + const existing = seen.get(key); + if (existing) { + existing.usedBy.push(filePath); + } else { + const { ref: _ref, ...checkoutRest } = repo.checkout ?? {}; + const hasCheckout = Object.keys(checkoutRest).length > 0; + seen.set(key, { + url: repo.source.url, + ref, + clone: repo.clone, + checkout: hasCheckout ? checkoutRest : undefined, + usedBy: [filePath], + }); + } + } + } catch (err) { + errors.push({ + file: filePath, + message: err instanceof Error ? err.message : String(err), + }); + } + } + + return { repos: [...seen.values()], errors }; +} + +// --------------------------------------------------------------------------- +// Internal helpers — lightweight YAML extraction, no full test parsing +// --------------------------------------------------------------------------- + +interface RawRepo { + source: RepoSource; + checkout?: RepoCheckout; + clone?: RepoClone; +} + +async function extractReposFromEvalFile(filePath: string): Promise { + const content = await readFile(filePath, 'utf8'); + const parsed = interpolateEnv(parse(content), process.env); + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return []; + const obj = parsed as Record; + const evalFileDir = path.dirname(path.resolve(filePath)); + + const repos: RawRepo[] = []; + + // Suite-level workspace + const suiteRepos = await extractReposFromWorkspaceRaw(obj.workspace, evalFileDir); + repos.push(...suiteRepos); + + // Per-test workspace + const tests = Array.isArray(obj.tests) ? obj.tests : []; + for (const test of tests) { + if (test && typeof test === 'object' && !Array.isArray(test)) { + const testObj = test as Record; + const testRepos = await extractReposFromWorkspaceRaw(testObj.workspace, evalFileDir); + repos.push(...testRepos); + } + } + + return repos; +} + +/** + * Extract repos from a raw workspace value. + * Handles both inline objects and string references to external workspace files. + */ +async function extractReposFromWorkspaceRaw(raw: unknown, evalFileDir: string): Promise { + if (typeof raw === 'string') { + // External workspace file reference + const workspaceFilePath = path.resolve(evalFileDir, raw); + const content = await readFile(workspaceFilePath, 'utf8'); + const parsed = interpolateEnv(parse(content), process.env); + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return []; + return extractReposFromObject(parsed as Record); + } + if (raw && typeof raw === 'object' && !Array.isArray(raw)) { + return extractReposFromObject(raw as Record); + } + return []; +} + +function extractReposFromObject(obj: Record): RawRepo[] { + const rawRepos = Array.isArray(obj.repos) ? obj.repos : []; + const result: RawRepo[] = []; + for (const r of rawRepos) { + if (!r || typeof r !== 'object' || Array.isArray(r)) continue; + const repo = r as Record; + const source = parseRepoSource(repo.source); + if (!source) continue; + result.push({ + source, + checkout: parseRepoCheckout(repo.checkout), + clone: parseRepoClone(repo.clone), + }); + } + return result; +} diff --git a/packages/core/src/evaluation/workspace/index.ts b/packages/core/src/evaluation/workspace/index.ts index 60ad80b6b..cb7fa2799 100644 --- a/packages/core/src/evaluation/workspace/index.ts +++ b/packages/core/src/evaluation/workspace/index.ts @@ -21,3 +21,4 @@ export { type AcquireWorkspaceOptions, type PoolSlot, } from './pool-manager.js'; +export { scanRepoDeps, type RepoDep, type DepsScanResult } from './deps-scanner.js'; diff --git a/packages/core/src/evaluation/workspace/repo-config-parser.ts b/packages/core/src/evaluation/workspace/repo-config-parser.ts new file mode 100644 index 000000000..ef2dca082 --- /dev/null +++ b/packages/core/src/evaluation/workspace/repo-config-parser.ts @@ -0,0 +1,66 @@ +/** + * Shared parsers for repo configuration objects (source, checkout, clone). + * + * Used by both the full YAML parser (yaml-parser.ts) and the lightweight + * deps scanner (deps-scanner.ts) to avoid duplicating parsing logic. + */ +import type { RepoCheckout, RepoClone, RepoConfig, RepoSource } from '../types.js'; +import { isJsonObject } from '../types.js'; + +export function parseRepoSource(raw: unknown): RepoSource | undefined { + if (!isJsonObject(raw)) return undefined; + const obj = raw as Record; + if (obj.type === 'git' && typeof obj.url === 'string') { + return { type: 'git', url: obj.url }; + } + if (obj.type === 'local' && typeof obj.path === 'string') { + return { type: 'local', path: obj.path }; + } + return undefined; +} + +export function parseRepoCheckout(raw: unknown): RepoCheckout | undefined { + if (!isJsonObject(raw)) return undefined; + const obj = raw as Record; + const ref = typeof obj.ref === 'string' ? obj.ref : undefined; + const resolve = obj.resolve === 'remote' || obj.resolve === 'local' ? obj.resolve : undefined; + const ancestor = typeof obj.ancestor === 'number' ? obj.ancestor : undefined; + if (!ref && !resolve && ancestor === undefined) return undefined; + return { + ...(ref !== undefined && { ref }), + ...(resolve !== undefined && { resolve }), + ...(ancestor !== undefined && { ancestor }), + }; +} + +export function parseRepoClone(raw: unknown): RepoClone | undefined { + if (!isJsonObject(raw)) return undefined; + const obj = raw as Record; + const depth = typeof obj.depth === 'number' ? obj.depth : undefined; + const filter = typeof obj.filter === 'string' ? obj.filter : undefined; + const sparse = Array.isArray(obj.sparse) + ? obj.sparse.filter((s): s is string => typeof s === 'string') + : undefined; + if (depth === undefined && !filter && !sparse) return undefined; + return { + ...(depth !== undefined && { depth }), + ...(filter !== undefined && { filter }), + ...(sparse !== undefined && { sparse }), + }; +} + +export function parseRepoConfig(raw: unknown): RepoConfig | undefined { + if (!isJsonObject(raw)) return undefined; + const obj = raw as Record; + const repoPath = typeof obj.path === 'string' ? obj.path : undefined; + const source = parseRepoSource(obj.source); + if (!repoPath || !source) return undefined; + const checkout = parseRepoCheckout(obj.checkout); + const clone = parseRepoClone(obj.clone); + return { + path: repoPath, + source, + ...(checkout !== undefined && { checkout }), + ...(clone !== undefined && { clone }), + }; +} diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 4e3a9f5af..dc552c2b5 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -38,10 +38,7 @@ import type { EvalTest, JsonObject, JsonValue, - RepoCheckout, - RepoClone, RepoConfig, - RepoSource, TestMessage, TrialsConfig, WorkspaceConfig, @@ -50,6 +47,7 @@ import type { WorkspaceScriptConfig, } from './types.js'; import { isJsonObject, isTestMessage } from './types.js'; +import { parseRepoConfig } from './workspace/repo-config-parser.js'; // Re-export public APIs from modules export { buildPromptInputs, type PromptInputs } from './formatting/prompt-builder.js'; @@ -590,64 +588,6 @@ function parseWorkspaceScriptConfig( return cwd ? { ...config, cwd } : config; } -function parseRepoSource(raw: unknown): RepoSource | undefined { - if (!isJsonObject(raw)) return undefined; - const obj = raw as Record; - if (obj.type === 'git' && typeof obj.url === 'string') { - return { type: 'git', url: obj.url }; - } - if (obj.type === 'local' && typeof obj.path === 'string') { - return { type: 'local', path: obj.path }; - } - return undefined; -} - -function parseRepoCheckout(raw: unknown): RepoCheckout | undefined { - if (!isJsonObject(raw)) return undefined; - const obj = raw as Record; - const ref = typeof obj.ref === 'string' ? obj.ref : undefined; - const resolve = obj.resolve === 'remote' || obj.resolve === 'local' ? obj.resolve : undefined; - const ancestor = typeof obj.ancestor === 'number' ? obj.ancestor : undefined; - if (!ref && !resolve && ancestor === undefined) return undefined; - return { - ...(ref !== undefined && { ref }), - ...(resolve !== undefined && { resolve }), - ...(ancestor !== undefined && { ancestor }), - }; -} - -function parseRepoClone(raw: unknown): RepoClone | undefined { - if (!isJsonObject(raw)) return undefined; - const obj = raw as Record; - const depth = typeof obj.depth === 'number' ? obj.depth : undefined; - const filter = typeof obj.filter === 'string' ? obj.filter : undefined; - const sparse = Array.isArray(obj.sparse) - ? obj.sparse.filter((s): s is string => typeof s === 'string') - : undefined; - if (depth === undefined && !filter && !sparse) return undefined; - return { - ...(depth !== undefined && { depth }), - ...(filter !== undefined && { filter }), - ...(sparse !== undefined && { sparse }), - }; -} - -function parseRepoConfig(raw: unknown): RepoConfig | undefined { - if (!isJsonObject(raw)) return undefined; - const obj = raw as Record; - const repoPath = typeof obj.path === 'string' ? obj.path : undefined; - const source = parseRepoSource(obj.source); - if (!repoPath || !source) return undefined; - const checkout = parseRepoCheckout(obj.checkout); - const clone = parseRepoClone(obj.clone); - return { - path: repoPath, - source, - ...(checkout !== undefined && { checkout }), - ...(clone !== undefined && { clone }), - }; -} - function parseWorkspaceHookConfig( raw: unknown, evalFileDir: string, diff --git a/packages/core/test/evaluation/workspace/deps-scanner.test.ts b/packages/core/test/evaluation/workspace/deps-scanner.test.ts new file mode 100644 index 000000000..0b082a50e --- /dev/null +++ b/packages/core/test/evaluation/workspace/deps-scanner.test.ts @@ -0,0 +1,459 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +import { scanRepoDeps } from '../../../src/evaluation/workspace/deps-scanner.js'; + +describe('scanRepoDeps', () => { + let tempDir: string; + + beforeAll(async () => { + tempDir = await mkdtemp(path.join(tmpdir(), 'deps-scanner-')); + }); + + afterAll(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + async function writeYaml(name: string, content: string): Promise { + const filePath = path.join(tempDir, name); + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, content, 'utf8'); + return filePath; + } + + it('extracts git repos from suite-level workspace', async () => { + const file = await writeYaml( + 'suite-level.eval.yaml', + ` +workspace: + repos: + - path: ./repo-a + source: + type: git + url: https://github.com/org/repo-a.git + checkout: + ref: main + clone: + depth: 1 +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.errors).toHaveLength(0); + expect(result.repos).toHaveLength(1); + expect(result.repos[0]).toMatchObject({ + url: 'https://github.com/org/repo-a.git', + ref: 'main', + clone: { depth: 1 }, + }); + expect(result.repos[0].usedBy).toEqual([file]); + }); + + it('extracts git repos from per-test workspace', async () => { + const file = await writeYaml( + 'per-test.eval.yaml', + ` +tests: + - id: test-1 + input: hello + criteria: world + workspace: + repos: + - path: ./repo-b + source: + type: git + url: https://github.com/org/repo-b.git + checkout: + ref: v2.0 +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.errors).toHaveLength(0); + expect(result.repos).toHaveLength(1); + expect(result.repos[0]).toMatchObject({ + url: 'https://github.com/org/repo-b.git', + ref: 'v2.0', + }); + }); + + it('deduplicates repos by (url, ref)', async () => { + const file1 = await writeYaml( + 'dedup-1.eval.yaml', + ` +workspace: + repos: + - path: ./repo + source: + type: git + url: https://github.com/org/shared.git + checkout: + ref: main +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + const file2 = await writeYaml( + 'dedup-2.eval.yaml', + ` +workspace: + repos: + - path: ./repo + source: + type: git + url: https://github.com/org/shared.git + checkout: + ref: main +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file1, file2]); + expect(result.repos).toHaveLength(1); + expect(result.repos[0].usedBy).toEqual([file1, file2]); + }); + + it('treats different refs as different deps', async () => { + const file = await writeYaml( + 'diff-refs.eval.yaml', + ` +workspace: + repos: + - path: ./repo-main + source: + type: git + url: https://github.com/org/repo.git + checkout: + ref: main + - path: ./repo-dev + source: + type: git + url: https://github.com/org/repo.git + checkout: + ref: develop +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(2); + const urls = result.repos.map((r) => `${r.url}@${r.ref}`); + expect(urls).toContain('https://github.com/org/repo.git@main'); + expect(urls).toContain('https://github.com/org/repo.git@develop'); + }); + + it('skips local source repos', async () => { + const file = await writeYaml( + 'local-source.eval.yaml', + ` +workspace: + repos: + - path: ./local-repo + source: + type: local + path: /tmp/some-repo + - path: ./git-repo + source: + type: git + url: https://github.com/org/repo.git +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(1); + expect(result.repos[0].url).toBe('https://github.com/org/repo.git'); + }); + + it('resolves external workspace file references', async () => { + await writeYaml( + 'shared/workspace.yaml', + ` +repos: + - path: ./external-repo + source: + type: git + url: https://github.com/org/external.git + checkout: + ref: v1.0 + clone: + depth: 2 + filter: blob:none +`, + ); + + const evalFile = await writeYaml( + 'external-ref.eval.yaml', + ` +workspace: ./shared/workspace.yaml +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([evalFile]); + expect(result.errors).toHaveLength(0); + expect(result.repos).toHaveLength(1); + expect(result.repos[0]).toMatchObject({ + url: 'https://github.com/org/external.git', + ref: 'v1.0', + clone: { depth: 2, filter: 'blob:none' }, + }); + }); + + it('returns empty repos for eval with no workspace', async () => { + const file = await writeYaml( + 'no-workspace.eval.yaml', + ` +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(0); + expect(result.errors).toHaveLength(0); + }); + + it('collects parse errors without stopping', async () => { + const goodFile = await writeYaml( + 'good.eval.yaml', + ` +workspace: + repos: + - path: ./repo + source: + type: git + url: https://github.com/org/good.git +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + const badFile = path.join(tempDir, 'nonexistent.eval.yaml'); + + const result = await scanRepoDeps([goodFile, badFile]); + expect(result.repos).toHaveLength(1); + expect(result.errors).toHaveLength(1); + expect(result.errors[0].file).toBe(badFile); + }); + + it('handles repos with no ref (defaults to undefined)', async () => { + const file = await writeYaml( + 'no-ref.eval.yaml', + ` +workspace: + repos: + - path: ./repo + source: + type: git + url: https://github.com/org/repo.git +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(1); + expect(result.repos[0].ref).toBeUndefined(); + }); + + it('includes clone sparse config', async () => { + const file = await writeYaml( + 'sparse.eval.yaml', + ` +workspace: + repos: + - path: ./repo + source: + type: git + url: https://github.com/org/repo.git + clone: + sparse: + - src/** + - tests/** +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos[0].clone).toEqual({ sparse: ['src/**', 'tests/**'] }); + }); + + it('includes checkout resolve and ancestor', async () => { + const file = await writeYaml( + 'checkout-opts.eval.yaml', + ` +workspace: + repos: + - path: ./repo + source: + type: git + url: https://github.com/org/repo.git + checkout: + ref: main + resolve: remote + ancestor: 3 +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos[0]).toMatchObject({ + ref: 'main', + checkout: { resolve: 'remote', ancestor: 3 }, + }); + }); + + it('collects repos from both suite-level and per-test workspaces', async () => { + const file = await writeYaml( + 'both-levels.eval.yaml', + ` +workspace: + repos: + - path: ./suite-repo + source: + type: git + url: https://github.com/org/suite.git + checkout: + ref: main +tests: + - id: test-1 + input: hello + criteria: world + workspace: + repos: + - path: ./test-repo + source: + type: git + url: https://github.com/org/test-only.git + checkout: + ref: v1 +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(2); + const urls = result.repos.map((r) => r.url); + expect(urls).toContain('https://github.com/org/suite.git'); + expect(urls).toContain('https://github.com/org/test-only.git'); + }); + + it('interpolates env vars in repo URLs', async () => { + const originalEnv = process.env.TEST_REPO_URL; + process.env.TEST_REPO_URL = 'https://github.com/org/from-env.git'; + try { + const file = await writeYaml( + 'env-var.eval.yaml', + ` +workspace: + repos: + - path: ./repo + source: + type: git + url: \${{ TEST_REPO_URL }} +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(1); + expect(result.repos[0].url).toBe('https://github.com/org/from-env.git'); + } finally { + if (originalEnv === undefined) { + process.env.TEST_REPO_URL = undefined; + } else { + process.env.TEST_REPO_URL = originalEnv; + } + } + }); + + it('collects malformed YAML as error without crashing', async () => { + const file = await writeYaml('bad-yaml.eval.yaml', ':\n bad: yaml: [unclosed'); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(0); + expect(result.errors).toHaveLength(1); + expect(result.errors[0].file).toBe(file); + }); + + it('collects error for broken external workspace file reference', async () => { + const file = await writeYaml( + 'broken-ref.eval.yaml', + ` +workspace: ./nonexistent-workspace.yaml +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(0); + expect(result.errors).toHaveLength(1); + expect(result.errors[0].file).toBe(file); + }); + + it('deduplicates URLs with and without trailing .git', async () => { + const file = await writeYaml( + 'url-normalize.eval.yaml', + ` +workspace: + repos: + - path: ./repo-a + source: + type: git + url: https://github.com/org/repo.git + checkout: + ref: main + - path: ./repo-b + source: + type: git + url: https://github.com/org/repo + checkout: + ref: main +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(1); + }); +}); diff --git a/plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md b/plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md index 4ee55f41a..754346dba 100644 --- a/plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md +++ b/plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md @@ -341,6 +341,7 @@ workspace: - `hooks.enabled`: boolean (default `true`); set `false` to skip all lifecycle hooks - Pool reset defaults to `fast` (`git clean -fd`); use `--workspace-clean full` for strict reset (`git clean -fdx`) - Pool entries are managed separately via `agentv workspace list` and `agentv workspace clean` +- `agentv workspace deps ` scans eval files and outputs a JSON manifest of required git repos (useful for CI pre-cloning) See https://agentv.dev/targets/configuration/#repository-lifecycle