From d63db8286b183a0e9a7aac9aa7a62bce451e19a4 Mon Sep 17 00:00:00 2001 From: Christopher Date: Tue, 7 Apr 2026 04:57:34 +0000 Subject: [PATCH 1/6] feat(workspace): add `agentv workspace deps` to scan eval files for repo dependencies Adds a lightweight scanner that extracts git repo dependencies from eval YAML workspace configs without full test/grader parsing. CI pipelines can use this to determine which repos to clone before running evals. Closes #959 Co-Authored-By: Claude Opus 4.6 (1M context) --- apps/cli/src/commands/workspace/deps.ts | 44 +++ apps/cli/src/commands/workspace/index.ts | 2 + .../src/evaluation/workspace/deps-scanner.ts | 205 ++++++++++ .../core/src/evaluation/workspace/index.ts | 1 + .../evaluation/workspace/deps-scanner.test.ts | 371 ++++++++++++++++++ 5 files changed, 623 insertions(+) create mode 100644 apps/cli/src/commands/workspace/deps.ts create mode 100644 packages/core/src/evaluation/workspace/deps-scanner.ts create mode 100644 packages/core/test/evaluation/workspace/deps-scanner.test.ts diff --git a/apps/cli/src/commands/workspace/deps.ts b/apps/cli/src/commands/workspace/deps.ts new file mode 100644 index 000000000..44027e485 --- /dev/null +++ b/apps/cli/src/commands/workspace/deps.ts @@ -0,0 +1,44 @@ +import { command, restPositionals, string } from 'cmd-ts'; + +import { scanRepoDeps } from '@agentv/core'; + +import { resolveEvalPaths } from '../eval/shared.js'; + +export const depsCommand = command({ + name: 'deps', + description: 'Scan eval files and list git repo dependencies needed by workspaces', + args: { + evalPaths: restPositionals({ + type: string, + displayName: 'eval-paths', + description: 'Path(s) or glob(s) to evaluation .yaml file(s)', + }), + }, + handler: async ({ evalPaths }) => { + if (evalPaths.length === 0) { + console.error('Usage: agentv workspace deps '); + process.exit(1); + } + + const resolvedPaths = await resolveEvalPaths(evalPaths, process.cwd()); + const result = await scanRepoDeps(resolvedPaths); + + // Print errors to stderr + for (const err of result.errors) { + console.error(`warning: ${err.file}: ${err.message}`); + } + + // Output JSON manifest to stdout + const output = { + repos: result.repos.map((r) => ({ + url: r.url, + ...(r.ref !== undefined && { ref: r.ref }), + ...(r.clone !== undefined && { clone: r.clone }), + ...(r.checkout !== undefined && { checkout: r.checkout }), + used_by: r.usedBy, + })), + }; + + console.log(JSON.stringify(output, null, 2)); + }, +}); diff --git a/apps/cli/src/commands/workspace/index.ts b/apps/cli/src/commands/workspace/index.ts index 036d83ea8..50dd3c00f 100644 --- a/apps/cli/src/commands/workspace/index.ts +++ b/apps/cli/src/commands/workspace/index.ts @@ -1,6 +1,7 @@ import { subcommands } from 'cmd-ts'; import { cleanCommand } from './clean.js'; +import { depsCommand } from './deps.js'; import { listCommand } from './list.js'; export const workspaceCommand = subcommands({ @@ -9,5 +10,6 @@ export const workspaceCommand = subcommands({ cmds: { list: listCommand, clean: cleanCommand, + deps: depsCommand, }, }); diff --git a/packages/core/src/evaluation/workspace/deps-scanner.ts b/packages/core/src/evaluation/workspace/deps-scanner.ts new file mode 100644 index 000000000..68bacffdc --- /dev/null +++ b/packages/core/src/evaluation/workspace/deps-scanner.ts @@ -0,0 +1,205 @@ +/** + * Lightweight scanner that extracts git repo dependencies from eval YAML files + * without performing full test/grader parsing. + * + * Used by `agentv workspace deps` to determine which repos CI needs to clone + * before running evals. + * + * How it works: + * 1. Reads each eval YAML file and parses it + * 2. Extracts `workspace.repos` at suite-level and per-test level + * 3. Resolves external workspace file references (string → file path) + * 4. Deduplicates git repos by (url, ref) + * 5. Returns a flat list of unique repo dependencies + * + * To extend: add support for new workspace source types by adding a branch + * in `extractReposFromWorkspaceRaw()`. + */ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { parse } from 'yaml'; + +import type { RepoCheckout, RepoClone } from '../types.js'; +import { interpolateEnv } from '../interpolation.js'; + +/** A single git repo dependency discovered from eval files. */ +export interface RepoDep { + /** Git clone URL */ + readonly url: string; + /** Checkout ref (branch, tag, SHA). undefined means HEAD. */ + readonly ref: string | undefined; + /** Clone options (depth, filter, sparse) — merged from first occurrence */ + readonly clone: RepoClone | undefined; + /** Checkout options (resolve, ancestor) — from first occurrence */ + readonly checkout: Omit | undefined; + /** Eval files that reference this repo */ + readonly usedBy: string[]; +} + +/** Full output of the deps scanner. */ +export interface DepsScanResult { + readonly repos: readonly RepoDep[]; + /** Files that failed to parse (non-fatal) */ + readonly errors: readonly { file: string; message: string }[]; +} + +/** + * Scan eval YAML files and collect unique git repo dependencies. + * Non-YAML files and parse errors are collected in `errors` but don't stop scanning. + */ +export async function scanRepoDeps(evalFilePaths: readonly string[]): Promise { + const seen = new Map(); + const errors: { file: string; message: string }[] = []; + + for (const filePath of evalFilePaths) { + try { + const repos = await extractReposFromEvalFile(filePath); + for (const repo of repos) { + if (repo.source.type !== 'git') continue; + const ref = repo.checkout?.ref; + const key = `${repo.source.url}\0${ref ?? ''}`; + const existing = seen.get(key); + if (existing) { + existing.usedBy.push(filePath); + } else { + const { ref: _ref, ...checkoutRest } = repo.checkout ?? {}; + const hasCheckout = Object.keys(checkoutRest).length > 0; + seen.set(key, { + url: repo.source.url, + ref, + clone: repo.clone, + checkout: hasCheckout ? checkoutRest : undefined, + usedBy: [filePath], + }); + } + } + } catch (err) { + errors.push({ + file: filePath, + message: err instanceof Error ? err.message : String(err), + }); + } + } + + return { repos: [...seen.values()], errors }; +} + +// --------------------------------------------------------------------------- +// Internal helpers — lightweight YAML extraction, no full test parsing +// --------------------------------------------------------------------------- + +interface RawRepo { + source: { type: 'git'; url: string } | { type: 'local'; path: string }; + checkout?: RepoCheckout; + clone?: RepoClone; +} + +async function extractReposFromEvalFile(filePath: string): Promise { + const content = await readFile(filePath, 'utf8'); + const parsed = interpolateEnv(parse(content), process.env); + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return []; + const obj = parsed as Record; + const evalFileDir = path.dirname(path.resolve(filePath)); + + const repos: RawRepo[] = []; + + // Suite-level workspace + const suiteRepos = await extractReposFromWorkspaceRaw(obj.workspace, evalFileDir); + repos.push(...suiteRepos); + + // Per-test workspace + const tests = Array.isArray(obj.tests) ? obj.tests : []; + for (const test of tests) { + if (test && typeof test === 'object' && !Array.isArray(test)) { + const testObj = test as Record; + const testRepos = await extractReposFromWorkspaceRaw(testObj.workspace, evalFileDir); + repos.push(...testRepos); + } + } + + return repos; +} + +/** + * Extract repos from a raw workspace value. + * Handles both inline objects and string references to external workspace files. + */ +async function extractReposFromWorkspaceRaw( + raw: unknown, + evalFileDir: string, +): Promise { + if (typeof raw === 'string') { + // External workspace file reference + const workspaceFilePath = path.resolve(evalFileDir, raw); + const content = await readFile(workspaceFilePath, 'utf8'); + const parsed = interpolateEnv(parse(content), process.env); + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return []; + const wsDir = path.dirname(workspaceFilePath); + return extractReposFromObject(parsed as Record, wsDir); + } + if (raw && typeof raw === 'object' && !Array.isArray(raw)) { + return extractReposFromObject(raw as Record, evalFileDir); + } + return []; +} + +function extractReposFromObject(obj: Record, _baseDir: string): RawRepo[] { + const rawRepos = Array.isArray(obj.repos) ? obj.repos : []; + const result: RawRepo[] = []; + for (const r of rawRepos) { + if (!r || typeof r !== 'object' || Array.isArray(r)) continue; + const repo = r as Record; + const source = parseSourceRaw(repo.source); + if (!source) continue; + result.push({ + source, + checkout: parseCheckoutRaw(repo.checkout), + clone: parseCloneRaw(repo.clone), + }); + } + return result; +} + +function parseSourceRaw( + raw: unknown, +): { type: 'git'; url: string } | { type: 'local'; path: string } | undefined { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined; + const obj = raw as Record; + if (obj.type === 'git' && typeof obj.url === 'string') { + return { type: 'git', url: obj.url }; + } + if (obj.type === 'local' && typeof obj.path === 'string') { + return { type: 'local', path: obj.path }; + } + return undefined; +} + +function parseCheckoutRaw(raw: unknown): RepoCheckout | undefined { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined; + const obj = raw as Record; + const ref = typeof obj.ref === 'string' ? obj.ref : undefined; + const resolve = obj.resolve === 'remote' || obj.resolve === 'local' ? obj.resolve : undefined; + const ancestor = typeof obj.ancestor === 'number' ? obj.ancestor : undefined; + if (!ref && !resolve && ancestor === undefined) return undefined; + return { + ...(ref !== undefined && { ref }), + ...(resolve !== undefined && { resolve }), + ...(ancestor !== undefined && { ancestor }), + }; +} + +function parseCloneRaw(raw: unknown): RepoClone | undefined { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined; + const obj = raw as Record; + const depth = typeof obj.depth === 'number' ? obj.depth : undefined; + const filter = typeof obj.filter === 'string' ? obj.filter : undefined; + const sparse = Array.isArray(obj.sparse) + ? obj.sparse.filter((s): s is string => typeof s === 'string') + : undefined; + if (depth === undefined && !filter && !sparse) return undefined; + return { + ...(depth !== undefined && { depth }), + ...(filter !== undefined && { filter }), + ...(sparse !== undefined && { sparse }), + }; +} diff --git a/packages/core/src/evaluation/workspace/index.ts b/packages/core/src/evaluation/workspace/index.ts index 60ad80b6b..cb7fa2799 100644 --- a/packages/core/src/evaluation/workspace/index.ts +++ b/packages/core/src/evaluation/workspace/index.ts @@ -21,3 +21,4 @@ export { type AcquireWorkspaceOptions, type PoolSlot, } from './pool-manager.js'; +export { scanRepoDeps, type RepoDep, type DepsScanResult } from './deps-scanner.js'; diff --git a/packages/core/test/evaluation/workspace/deps-scanner.test.ts b/packages/core/test/evaluation/workspace/deps-scanner.test.ts new file mode 100644 index 000000000..87a8938d8 --- /dev/null +++ b/packages/core/test/evaluation/workspace/deps-scanner.test.ts @@ -0,0 +1,371 @@ +import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +import { scanRepoDeps } from '../../../src/evaluation/workspace/deps-scanner.js'; + +describe('scanRepoDeps', () => { + let tempDir: string; + + beforeAll(async () => { + tempDir = await mkdtemp(path.join(tmpdir(), 'deps-scanner-')); + }); + + afterAll(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + async function writeYaml(name: string, content: string): Promise { + const filePath = path.join(tempDir, name); + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, content, 'utf8'); + return filePath; + } + + it('extracts git repos from suite-level workspace', async () => { + const file = await writeYaml( + 'suite-level.eval.yaml', + ` +workspace: + repos: + - path: ./repo-a + source: + type: git + url: https://github.com/org/repo-a.git + checkout: + ref: main + clone: + depth: 1 +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.errors).toHaveLength(0); + expect(result.repos).toHaveLength(1); + expect(result.repos[0]).toMatchObject({ + url: 'https://github.com/org/repo-a.git', + ref: 'main', + clone: { depth: 1 }, + }); + expect(result.repos[0].usedBy).toEqual([file]); + }); + + it('extracts git repos from per-test workspace', async () => { + const file = await writeYaml( + 'per-test.eval.yaml', + ` +tests: + - id: test-1 + input: hello + criteria: world + workspace: + repos: + - path: ./repo-b + source: + type: git + url: https://github.com/org/repo-b.git + checkout: + ref: v2.0 +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.errors).toHaveLength(0); + expect(result.repos).toHaveLength(1); + expect(result.repos[0]).toMatchObject({ + url: 'https://github.com/org/repo-b.git', + ref: 'v2.0', + }); + }); + + it('deduplicates repos by (url, ref)', async () => { + const file1 = await writeYaml( + 'dedup-1.eval.yaml', + ` +workspace: + repos: + - path: ./repo + source: + type: git + url: https://github.com/org/shared.git + checkout: + ref: main +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + const file2 = await writeYaml( + 'dedup-2.eval.yaml', + ` +workspace: + repos: + - path: ./repo + source: + type: git + url: https://github.com/org/shared.git + checkout: + ref: main +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file1, file2]); + expect(result.repos).toHaveLength(1); + expect(result.repos[0].usedBy).toEqual([file1, file2]); + }); + + it('treats different refs as different deps', async () => { + const file = await writeYaml( + 'diff-refs.eval.yaml', + ` +workspace: + repos: + - path: ./repo-main + source: + type: git + url: https://github.com/org/repo.git + checkout: + ref: main + - path: ./repo-dev + source: + type: git + url: https://github.com/org/repo.git + checkout: + ref: develop +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(2); + const urls = result.repos.map((r) => `${r.url}@${r.ref}`); + expect(urls).toContain('https://github.com/org/repo.git@main'); + expect(urls).toContain('https://github.com/org/repo.git@develop'); + }); + + it('skips local source repos', async () => { + const file = await writeYaml( + 'local-source.eval.yaml', + ` +workspace: + repos: + - path: ./local-repo + source: + type: local + path: /tmp/some-repo + - path: ./git-repo + source: + type: git + url: https://github.com/org/repo.git +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(1); + expect(result.repos[0].url).toBe('https://github.com/org/repo.git'); + }); + + it('resolves external workspace file references', async () => { + const wsFile = await writeYaml( + 'shared/workspace.yaml', + ` +repos: + - path: ./external-repo + source: + type: git + url: https://github.com/org/external.git + checkout: + ref: v1.0 + clone: + depth: 2 + filter: blob:none +`, + ); + + const evalFile = await writeYaml( + 'external-ref.eval.yaml', + ` +workspace: ./shared/workspace.yaml +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([evalFile]); + expect(result.errors).toHaveLength(0); + expect(result.repos).toHaveLength(1); + expect(result.repos[0]).toMatchObject({ + url: 'https://github.com/org/external.git', + ref: 'v1.0', + clone: { depth: 2, filter: 'blob:none' }, + }); + }); + + it('returns empty repos for eval with no workspace', async () => { + const file = await writeYaml( + 'no-workspace.eval.yaml', + ` +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(0); + expect(result.errors).toHaveLength(0); + }); + + it('collects parse errors without stopping', async () => { + const goodFile = await writeYaml( + 'good.eval.yaml', + ` +workspace: + repos: + - path: ./repo + source: + type: git + url: https://github.com/org/good.git +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + const badFile = path.join(tempDir, 'nonexistent.eval.yaml'); + + const result = await scanRepoDeps([goodFile, badFile]); + expect(result.repos).toHaveLength(1); + expect(result.errors).toHaveLength(1); + expect(result.errors[0].file).toBe(badFile); + }); + + it('handles repos with no ref (defaults to undefined)', async () => { + const file = await writeYaml( + 'no-ref.eval.yaml', + ` +workspace: + repos: + - path: ./repo + source: + type: git + url: https://github.com/org/repo.git +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(1); + expect(result.repos[0].ref).toBeUndefined(); + }); + + it('includes clone sparse config', async () => { + const file = await writeYaml( + 'sparse.eval.yaml', + ` +workspace: + repos: + - path: ./repo + source: + type: git + url: https://github.com/org/repo.git + clone: + sparse: + - src/** + - tests/** +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos[0].clone).toEqual({ sparse: ['src/**', 'tests/**'] }); + }); + + it('includes checkout resolve and ancestor', async () => { + const file = await writeYaml( + 'checkout-opts.eval.yaml', + ` +workspace: + repos: + - path: ./repo + source: + type: git + url: https://github.com/org/repo.git + checkout: + ref: main + resolve: remote + ancestor: 3 +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos[0]).toMatchObject({ + ref: 'main', + checkout: { resolve: 'remote', ancestor: 3 }, + }); + }); + + it('collects repos from both suite-level and per-test workspaces', async () => { + const file = await writeYaml( + 'both-levels.eval.yaml', + ` +workspace: + repos: + - path: ./suite-repo + source: + type: git + url: https://github.com/org/suite.git + checkout: + ref: main +tests: + - id: test-1 + input: hello + criteria: world + workspace: + repos: + - path: ./test-repo + source: + type: git + url: https://github.com/org/test-only.git + checkout: + ref: v1 +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(2); + const urls = result.repos.map((r) => r.url); + expect(urls).toContain('https://github.com/org/suite.git'); + expect(urls).toContain('https://github.com/org/test-only.git'); + }); +}); From b1429391febe6e0ab6f81c9d10ca1290cfc1e868 Mon Sep 17 00:00:00 2001 From: Christopher Date: Tue, 7 Apr 2026 04:58:58 +0000 Subject: [PATCH 2/6] style: fix biome lint (import order, formatting) Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/core/src/evaluation/workspace/deps-scanner.ts | 7 ++----- .../core/test/evaluation/workspace/deps-scanner.test.ts | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/core/src/evaluation/workspace/deps-scanner.ts b/packages/core/src/evaluation/workspace/deps-scanner.ts index 68bacffdc..b5c7bb42a 100644 --- a/packages/core/src/evaluation/workspace/deps-scanner.ts +++ b/packages/core/src/evaluation/workspace/deps-scanner.ts @@ -19,8 +19,8 @@ import { readFile } from 'node:fs/promises'; import path from 'node:path'; import { parse } from 'yaml'; -import type { RepoCheckout, RepoClone } from '../types.js'; import { interpolateEnv } from '../interpolation.js'; +import type { RepoCheckout, RepoClone } from '../types.js'; /** A single git repo dependency discovered from eval files. */ export interface RepoDep { @@ -124,10 +124,7 @@ async function extractReposFromEvalFile(filePath: string): Promise { * Extract repos from a raw workspace value. * Handles both inline objects and string references to external workspace files. */ -async function extractReposFromWorkspaceRaw( - raw: unknown, - evalFileDir: string, -): Promise { +async function extractReposFromWorkspaceRaw(raw: unknown, evalFileDir: string): Promise { if (typeof raw === 'string') { // External workspace file reference const workspaceFilePath = path.resolve(evalFileDir, raw); diff --git a/packages/core/test/evaluation/workspace/deps-scanner.test.ts b/packages/core/test/evaluation/workspace/deps-scanner.test.ts index 87a8938d8..1dab2732d 100644 --- a/packages/core/test/evaluation/workspace/deps-scanner.test.ts +++ b/packages/core/test/evaluation/workspace/deps-scanner.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; From c0a2ca0a7a607cc3d490f598dbae607cf8720974 Mon Sep 17 00:00:00 2001 From: Christopher Date: Tue, 7 Apr 2026 05:13:52 +0000 Subject: [PATCH 3/6] docs: add workspace deps command to docs and skill reference Co-Authored-By: Claude Opus 4.6 (1M context) --- apps/web/src/content/docs/docs/guides/workspace-pool.mdx | 4 ++++ apps/web/src/content/docs/docs/targets/configuration.mdx | 1 + plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md | 1 + 3 files changed, 6 insertions(+) diff --git a/apps/web/src/content/docs/docs/guides/workspace-pool.mdx b/apps/web/src/content/docs/docs/guides/workspace-pool.mdx index 8b06f2b08..4f1018b66 100644 --- a/apps/web/src/content/docs/docs/guides/workspace-pool.mdx +++ b/apps/web/src/content/docs/docs/guides/workspace-pool.mdx @@ -153,6 +153,10 @@ agentv workspace clean # Remove only pools for a specific repo agentv workspace clean --repo github.com/org/my-repo + +# Scan eval files and output a JSON manifest of required git repos +# Useful in CI to determine what to clone before running evals +agentv workspace deps evals/**/*.eval.yaml ``` ## External workspace config diff --git a/apps/web/src/content/docs/docs/targets/configuration.mdx b/apps/web/src/content/docs/docs/targets/configuration.mdx index 113c03122..24dbda058 100644 --- a/apps/web/src/content/docs/docs/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/targets/configuration.mdx @@ -222,6 +222,7 @@ Similar to GitHub Actions checkout conventions, `source` answers "where does thi Pool management commands: - `agentv workspace list` — list all pool entries with size and repo info - `agentv workspace clean` — remove all pool entries +- `agentv workspace deps ` — scan eval files and output a JSON manifest of required git repos (for CI pre-cloning) **Common patterns:** diff --git a/plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md b/plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md index 4ee55f41a..754346dba 100644 --- a/plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md +++ b/plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md @@ -341,6 +341,7 @@ workspace: - `hooks.enabled`: boolean (default `true`); set `false` to skip all lifecycle hooks - Pool reset defaults to `fast` (`git clean -fd`); use `--workspace-clean full` for strict reset (`git clean -fdx`) - Pool entries are managed separately via `agentv workspace list` and `agentv workspace clean` +- `agentv workspace deps ` scans eval files and outputs a JSON manifest of required git repos (useful for CI pre-cloning) See https://agentv.dev/targets/configuration/#repository-lifecycle From 1b1c7f19f22cfeae239da3c6e0788563e01d7da7 Mon Sep 17 00:00:00 2001 From: Christopher Date: Tue, 7 Apr 2026 05:27:38 +0000 Subject: [PATCH 4/6] refactor: address code review findings - Extract shared repo config parsers into repo-config-parser.ts (DRY) - Remove duplicated parseRepoSource/Checkout/Clone from deps-scanner - Update yaml-parser.ts to import from shared module - Make used_by paths relative to cwd for CI portability - Normalize git URLs for dedup (strip trailing .git, lowercase host) - Remove unused _baseDir parameter - Use RepoSource type instead of inline union - Add tests: env var interpolation, malformed YAML, broken workspace file ref, URL normalization dedup Co-Authored-By: Claude Opus 4.6 (1M context) --- apps/cli/src/commands/workspace/deps.ts | 10 ++- .../src/evaluation/workspace/deps-scanner.ts | 86 +++++++----------- .../workspace/repo-config-parser.ts | 66 ++++++++++++++ packages/core/src/evaluation/yaml-parser.ts | 62 +------------ .../evaluation/workspace/deps-scanner.test.ts | 90 ++++++++++++++++++- 5 files changed, 192 insertions(+), 122 deletions(-) create mode 100644 packages/core/src/evaluation/workspace/repo-config-parser.ts diff --git a/apps/cli/src/commands/workspace/deps.ts b/apps/cli/src/commands/workspace/deps.ts index 44027e485..ee1b30291 100644 --- a/apps/cli/src/commands/workspace/deps.ts +++ b/apps/cli/src/commands/workspace/deps.ts @@ -1,3 +1,4 @@ +import path from 'node:path'; import { command, restPositionals, string } from 'cmd-ts'; import { scanRepoDeps } from '@agentv/core'; @@ -20,22 +21,23 @@ export const depsCommand = command({ process.exit(1); } - const resolvedPaths = await resolveEvalPaths(evalPaths, process.cwd()); + const cwd = process.cwd(); + const resolvedPaths = await resolveEvalPaths(evalPaths, cwd); const result = await scanRepoDeps(resolvedPaths); // Print errors to stderr for (const err of result.errors) { - console.error(`warning: ${err.file}: ${err.message}`); + console.error(`warning: ${path.relative(cwd, err.file)}: ${err.message}`); } - // Output JSON manifest to stdout + // Output JSON manifest to stdout (snake_case per wire format convention) const output = { repos: result.repos.map((r) => ({ url: r.url, ...(r.ref !== undefined && { ref: r.ref }), ...(r.clone !== undefined && { clone: r.clone }), ...(r.checkout !== undefined && { checkout: r.checkout }), - used_by: r.usedBy, + used_by: r.usedBy.map((p) => path.relative(cwd, p)), })), }; diff --git a/packages/core/src/evaluation/workspace/deps-scanner.ts b/packages/core/src/evaluation/workspace/deps-scanner.ts index b5c7bb42a..f00991243 100644 --- a/packages/core/src/evaluation/workspace/deps-scanner.ts +++ b/packages/core/src/evaluation/workspace/deps-scanner.ts @@ -20,7 +20,8 @@ import path from 'node:path'; import { parse } from 'yaml'; import { interpolateEnv } from '../interpolation.js'; -import type { RepoCheckout, RepoClone } from '../types.js'; +import type { RepoCheckout, RepoClone, RepoSource } from '../types.js'; +import { parseRepoCheckout, parseRepoClone, parseRepoSource } from './repo-config-parser.js'; /** A single git repo dependency discovered from eval files. */ export interface RepoDep { @@ -28,9 +29,9 @@ export interface RepoDep { readonly url: string; /** Checkout ref (branch, tag, SHA). undefined means HEAD. */ readonly ref: string | undefined; - /** Clone options (depth, filter, sparse) — merged from first occurrence */ + /** Clone options (depth, filter, sparse) — first-wins on dedup collision */ readonly clone: RepoClone | undefined; - /** Checkout options (resolve, ancestor) — from first occurrence */ + /** Checkout options (resolve, ancestor) — first-wins on dedup collision */ readonly checkout: Omit | undefined; /** Eval files that reference this repo */ readonly usedBy: string[]; @@ -43,9 +44,27 @@ export interface DepsScanResult { readonly errors: readonly { file: string; message: string }[]; } +/** Normalize a git URL for dedup: strip trailing .git and lowercase the host. */ +function normalizeGitUrl(url: string): string { + let normalized = url.replace(/\.git$/, ''); + // Lowercase the host portion of https:// URLs + try { + const parsed = new URL(normalized); + parsed.hostname = parsed.hostname.toLowerCase(); + normalized = parsed.toString().replace(/\/$/, ''); + } catch { + // Not a valid URL (e.g., SSH shorthand) — use as-is + } + return normalized; +} + /** * Scan eval YAML files and collect unique git repo dependencies. * Non-YAML files and parse errors are collected in `errors` but don't stop scanning. + * + * Dedup strategy: repos are keyed by (normalized URL, ref). On collision, + * clone/checkout options from the first occurrence win — this is intentional + * since the manifest is advisory (CI can override clone options). */ export async function scanRepoDeps(evalFilePaths: readonly string[]): Promise { const seen = new Map(); @@ -57,7 +76,7 @@ export async function scanRepoDeps(evalFilePaths: readonly string[]): Promise, wsDir); + return extractReposFromObject(parsed as Record); } if (raw && typeof raw === 'object' && !Array.isArray(raw)) { - return extractReposFromObject(raw as Record, evalFileDir); + return extractReposFromObject(raw as Record); } return []; } -function extractReposFromObject(obj: Record, _baseDir: string): RawRepo[] { +function extractReposFromObject(obj: Record): RawRepo[] { const rawRepos = Array.isArray(obj.repos) ? obj.repos : []; const result: RawRepo[] = []; for (const r of rawRepos) { if (!r || typeof r !== 'object' || Array.isArray(r)) continue; const repo = r as Record; - const source = parseSourceRaw(repo.source); + const source = parseRepoSource(repo.source); if (!source) continue; result.push({ source, - checkout: parseCheckoutRaw(repo.checkout), - clone: parseCloneRaw(repo.clone), + checkout: parseRepoCheckout(repo.checkout), + clone: parseRepoClone(repo.clone), }); } return result; } - -function parseSourceRaw( - raw: unknown, -): { type: 'git'; url: string } | { type: 'local'; path: string } | undefined { - if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined; - const obj = raw as Record; - if (obj.type === 'git' && typeof obj.url === 'string') { - return { type: 'git', url: obj.url }; - } - if (obj.type === 'local' && typeof obj.path === 'string') { - return { type: 'local', path: obj.path }; - } - return undefined; -} - -function parseCheckoutRaw(raw: unknown): RepoCheckout | undefined { - if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined; - const obj = raw as Record; - const ref = typeof obj.ref === 'string' ? obj.ref : undefined; - const resolve = obj.resolve === 'remote' || obj.resolve === 'local' ? obj.resolve : undefined; - const ancestor = typeof obj.ancestor === 'number' ? obj.ancestor : undefined; - if (!ref && !resolve && ancestor === undefined) return undefined; - return { - ...(ref !== undefined && { ref }), - ...(resolve !== undefined && { resolve }), - ...(ancestor !== undefined && { ancestor }), - }; -} - -function parseCloneRaw(raw: unknown): RepoClone | undefined { - if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined; - const obj = raw as Record; - const depth = typeof obj.depth === 'number' ? obj.depth : undefined; - const filter = typeof obj.filter === 'string' ? obj.filter : undefined; - const sparse = Array.isArray(obj.sparse) - ? obj.sparse.filter((s): s is string => typeof s === 'string') - : undefined; - if (depth === undefined && !filter && !sparse) return undefined; - return { - ...(depth !== undefined && { depth }), - ...(filter !== undefined && { filter }), - ...(sparse !== undefined && { sparse }), - }; -} diff --git a/packages/core/src/evaluation/workspace/repo-config-parser.ts b/packages/core/src/evaluation/workspace/repo-config-parser.ts new file mode 100644 index 000000000..ef2dca082 --- /dev/null +++ b/packages/core/src/evaluation/workspace/repo-config-parser.ts @@ -0,0 +1,66 @@ +/** + * Shared parsers for repo configuration objects (source, checkout, clone). + * + * Used by both the full YAML parser (yaml-parser.ts) and the lightweight + * deps scanner (deps-scanner.ts) to avoid duplicating parsing logic. + */ +import type { RepoCheckout, RepoClone, RepoConfig, RepoSource } from '../types.js'; +import { isJsonObject } from '../types.js'; + +export function parseRepoSource(raw: unknown): RepoSource | undefined { + if (!isJsonObject(raw)) return undefined; + const obj = raw as Record; + if (obj.type === 'git' && typeof obj.url === 'string') { + return { type: 'git', url: obj.url }; + } + if (obj.type === 'local' && typeof obj.path === 'string') { + return { type: 'local', path: obj.path }; + } + return undefined; +} + +export function parseRepoCheckout(raw: unknown): RepoCheckout | undefined { + if (!isJsonObject(raw)) return undefined; + const obj = raw as Record; + const ref = typeof obj.ref === 'string' ? obj.ref : undefined; + const resolve = obj.resolve === 'remote' || obj.resolve === 'local' ? obj.resolve : undefined; + const ancestor = typeof obj.ancestor === 'number' ? obj.ancestor : undefined; + if (!ref && !resolve && ancestor === undefined) return undefined; + return { + ...(ref !== undefined && { ref }), + ...(resolve !== undefined && { resolve }), + ...(ancestor !== undefined && { ancestor }), + }; +} + +export function parseRepoClone(raw: unknown): RepoClone | undefined { + if (!isJsonObject(raw)) return undefined; + const obj = raw as Record; + const depth = typeof obj.depth === 'number' ? obj.depth : undefined; + const filter = typeof obj.filter === 'string' ? obj.filter : undefined; + const sparse = Array.isArray(obj.sparse) + ? obj.sparse.filter((s): s is string => typeof s === 'string') + : undefined; + if (depth === undefined && !filter && !sparse) return undefined; + return { + ...(depth !== undefined && { depth }), + ...(filter !== undefined && { filter }), + ...(sparse !== undefined && { sparse }), + }; +} + +export function parseRepoConfig(raw: unknown): RepoConfig | undefined { + if (!isJsonObject(raw)) return undefined; + const obj = raw as Record; + const repoPath = typeof obj.path === 'string' ? obj.path : undefined; + const source = parseRepoSource(obj.source); + if (!repoPath || !source) return undefined; + const checkout = parseRepoCheckout(obj.checkout); + const clone = parseRepoClone(obj.clone); + return { + path: repoPath, + source, + ...(checkout !== undefined && { checkout }), + ...(clone !== undefined && { clone }), + }; +} diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 4e3a9f5af..dc552c2b5 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -38,10 +38,7 @@ import type { EvalTest, JsonObject, JsonValue, - RepoCheckout, - RepoClone, RepoConfig, - RepoSource, TestMessage, TrialsConfig, WorkspaceConfig, @@ -50,6 +47,7 @@ import type { WorkspaceScriptConfig, } from './types.js'; import { isJsonObject, isTestMessage } from './types.js'; +import { parseRepoConfig } from './workspace/repo-config-parser.js'; // Re-export public APIs from modules export { buildPromptInputs, type PromptInputs } from './formatting/prompt-builder.js'; @@ -590,64 +588,6 @@ function parseWorkspaceScriptConfig( return cwd ? { ...config, cwd } : config; } -function parseRepoSource(raw: unknown): RepoSource | undefined { - if (!isJsonObject(raw)) return undefined; - const obj = raw as Record; - if (obj.type === 'git' && typeof obj.url === 'string') { - return { type: 'git', url: obj.url }; - } - if (obj.type === 'local' && typeof obj.path === 'string') { - return { type: 'local', path: obj.path }; - } - return undefined; -} - -function parseRepoCheckout(raw: unknown): RepoCheckout | undefined { - if (!isJsonObject(raw)) return undefined; - const obj = raw as Record; - const ref = typeof obj.ref === 'string' ? obj.ref : undefined; - const resolve = obj.resolve === 'remote' || obj.resolve === 'local' ? obj.resolve : undefined; - const ancestor = typeof obj.ancestor === 'number' ? obj.ancestor : undefined; - if (!ref && !resolve && ancestor === undefined) return undefined; - return { - ...(ref !== undefined && { ref }), - ...(resolve !== undefined && { resolve }), - ...(ancestor !== undefined && { ancestor }), - }; -} - -function parseRepoClone(raw: unknown): RepoClone | undefined { - if (!isJsonObject(raw)) return undefined; - const obj = raw as Record; - const depth = typeof obj.depth === 'number' ? obj.depth : undefined; - const filter = typeof obj.filter === 'string' ? obj.filter : undefined; - const sparse = Array.isArray(obj.sparse) - ? obj.sparse.filter((s): s is string => typeof s === 'string') - : undefined; - if (depth === undefined && !filter && !sparse) return undefined; - return { - ...(depth !== undefined && { depth }), - ...(filter !== undefined && { filter }), - ...(sparse !== undefined && { sparse }), - }; -} - -function parseRepoConfig(raw: unknown): RepoConfig | undefined { - if (!isJsonObject(raw)) return undefined; - const obj = raw as Record; - const repoPath = typeof obj.path === 'string' ? obj.path : undefined; - const source = parseRepoSource(obj.source); - if (!repoPath || !source) return undefined; - const checkout = parseRepoCheckout(obj.checkout); - const clone = parseRepoClone(obj.clone); - return { - path: repoPath, - source, - ...(checkout !== undefined && { checkout }), - ...(clone !== undefined && { clone }), - }; -} - function parseWorkspaceHookConfig( raw: unknown, evalFileDir: string, diff --git a/packages/core/test/evaluation/workspace/deps-scanner.test.ts b/packages/core/test/evaluation/workspace/deps-scanner.test.ts index 1dab2732d..47154057b 100644 --- a/packages/core/test/evaluation/workspace/deps-scanner.test.ts +++ b/packages/core/test/evaluation/workspace/deps-scanner.test.ts @@ -183,7 +183,7 @@ tests: }); it('resolves external workspace file references', async () => { - const wsFile = await writeYaml( + await writeYaml( 'shared/workspace.yaml', ` repos: @@ -368,4 +368,92 @@ tests: expect(urls).toContain('https://github.com/org/suite.git'); expect(urls).toContain('https://github.com/org/test-only.git'); }); + + it('interpolates env vars in repo URLs', async () => { + const originalEnv = process.env.TEST_REPO_URL; + process.env.TEST_REPO_URL = 'https://github.com/org/from-env.git'; + try { + const file = await writeYaml( + 'env-var.eval.yaml', + ` +workspace: + repos: + - path: ./repo + source: + type: git + url: \${{ TEST_REPO_URL }} +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(1); + expect(result.repos[0].url).toBe('https://github.com/org/from-env.git'); + } finally { + if (originalEnv === undefined) { + delete process.env.TEST_REPO_URL; + } else { + process.env.TEST_REPO_URL = originalEnv; + } + } + }); + + it('collects malformed YAML as error without crashing', async () => { + const file = await writeYaml('bad-yaml.eval.yaml', ':\n bad: yaml: [unclosed'); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(0); + expect(result.errors).toHaveLength(1); + expect(result.errors[0].file).toBe(file); + }); + + it('collects error for broken external workspace file reference', async () => { + const file = await writeYaml( + 'broken-ref.eval.yaml', + ` +workspace: ./nonexistent-workspace.yaml +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(0); + expect(result.errors).toHaveLength(1); + expect(result.errors[0].file).toBe(file); + }); + + it('deduplicates URLs with and without trailing .git', async () => { + const file = await writeYaml( + 'url-normalize.eval.yaml', + ` +workspace: + repos: + - path: ./repo-a + source: + type: git + url: https://github.com/org/repo.git + checkout: + ref: main + - path: ./repo-b + source: + type: git + url: https://github.com/org/repo + checkout: + ref: main +tests: + - id: test-1 + input: hello + criteria: world +`, + ); + + const result = await scanRepoDeps([file]); + expect(result.repos).toHaveLength(1); + }); }); From 8debee3d1c0c72c898727470c017831fdda456a7 Mon Sep 17 00:00:00 2001 From: Christopher Date: Tue, 7 Apr 2026 05:28:59 +0000 Subject: [PATCH 5/6] style: replace delete operator with undefined assignment (biome lint) Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/core/test/evaluation/workspace/deps-scanner.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/test/evaluation/workspace/deps-scanner.test.ts b/packages/core/test/evaluation/workspace/deps-scanner.test.ts index 47154057b..0b082a50e 100644 --- a/packages/core/test/evaluation/workspace/deps-scanner.test.ts +++ b/packages/core/test/evaluation/workspace/deps-scanner.test.ts @@ -394,7 +394,7 @@ tests: expect(result.repos[0].url).toBe('https://github.com/org/from-env.git'); } finally { if (originalEnv === undefined) { - delete process.env.TEST_REPO_URL; + process.env.TEST_REPO_URL = undefined; } else { process.env.TEST_REPO_URL = originalEnv; } From fd2a0ca009ee56df149636647e8ea3166647898f Mon Sep 17 00:00:00 2001 From: Christopher Date: Tue, 7 Apr 2026 05:38:43 +0000 Subject: [PATCH 6/6] feat(workspace): omit used_by by default, add --used-by flag Keep the default JSON output minimal for CI piping. The --used-by flag includes the list of eval files that reference each repo when needed for debugging. Co-Authored-By: Claude Opus 4.6 (1M context) --- apps/cli/src/commands/workspace/deps.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/cli/src/commands/workspace/deps.ts b/apps/cli/src/commands/workspace/deps.ts index ee1b30291..21a67c24f 100644 --- a/apps/cli/src/commands/workspace/deps.ts +++ b/apps/cli/src/commands/workspace/deps.ts @@ -1,5 +1,5 @@ import path from 'node:path'; -import { command, restPositionals, string } from 'cmd-ts'; +import { command, flag, restPositionals, string } from 'cmd-ts'; import { scanRepoDeps } from '@agentv/core'; @@ -14,8 +14,12 @@ export const depsCommand = command({ displayName: 'eval-paths', description: 'Path(s) or glob(s) to evaluation .yaml file(s)', }), + usedBy: flag({ + long: 'used-by', + description: 'Include list of eval files that reference each repo', + }), }, - handler: async ({ evalPaths }) => { + handler: async ({ evalPaths, usedBy }) => { if (evalPaths.length === 0) { console.error('Usage: agentv workspace deps '); process.exit(1); @@ -37,7 +41,7 @@ export const depsCommand = command({ ...(r.ref !== undefined && { ref: r.ref }), ...(r.clone !== undefined && { clone: r.clone }), ...(r.checkout !== undefined && { checkout: r.checkout }), - used_by: r.usedBy.map((p) => path.relative(cwd, p)), + ...(usedBy && { used_by: r.usedBy.map((p) => path.relative(cwd, p)) }), })), };