diff --git a/apps/cli/src/commands/results/studio-config.ts b/apps/cli/src/commands/results/studio-config.ts index 4be3c56a..b9777e59 100644 --- a/apps/cli/src/commands/results/studio-config.ts +++ b/apps/cli/src/commands/results/studio-config.ts @@ -21,8 +21,8 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import path from 'node:path'; -import { DEFAULT_THRESHOLD } from '@agentv/core'; -import { parse as parseYaml, stringify as stringifyYaml } from 'yaml'; +import { DEFAULT_THRESHOLD, parseYamlValue } from '@agentv/core'; +import { stringify as stringifyYaml } from 'yaml'; export interface StudioConfig { threshold: number; @@ -47,7 +47,7 @@ export function loadStudioConfig(agentvDir: string): StudioConfig { } const raw = readFileSync(configPath, 'utf-8'); - const parsed = parseYaml(raw); + const parsed = parseYamlValue(raw); if (!parsed || typeof parsed !== 'object') { return { ...DEFAULTS }; @@ -89,7 +89,7 @@ export function saveStudioConfig(agentvDir: string, config: StudioConfig): void let existing: Record = {}; if (existsSync(configPath)) { const raw = readFileSync(configPath, 'utf-8'); - const parsed = parseYaml(raw); + const parsed = parseYamlValue(raw); if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { existing = parsed as Record; } diff --git a/packages/core/src/benchmarks.ts b/packages/core/src/benchmarks.ts index 39fa6b3c..a6036c69 100644 --- a/packages/core/src/benchmarks.ts +++ b/packages/core/src/benchmarks.ts @@ -32,8 +32,9 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs'; import path from 'node:path'; -import { parse as parseYaml, stringify as stringifyYaml } from 'yaml'; +import { stringify as stringifyYaml } from 'yaml'; +import { parseYamlValue } from './evaluation/yaml-loader.js'; import { getAgentvConfigDir } from './paths.js'; // ── Types ─────────────────────────────────────────────────────────────── @@ -101,7 +102,7 @@ export function loadBenchmarkRegistry(): BenchmarkRegistry { } try { const raw = readFileSync(registryPath, 'utf-8'); - const parsed = parseYaml(raw); + const parsed = parseYamlValue(raw) as { benchmarks?: unknown } | null | undefined; if (!parsed || typeof parsed !== 'object') { return { benchmarks: [] }; } diff --git a/packages/core/src/evaluation/loaders/case-file-loader.ts b/packages/core/src/evaluation/loaders/case-file-loader.ts index 0568d132..a14265fa 100644 --- a/packages/core/src/evaluation/loaders/case-file-loader.ts +++ b/packages/core/src/evaluation/loaders/case-file-loader.ts @@ -1,11 +1,11 @@ import { readFile, readdir, stat } from 'node:fs/promises'; import path from 'node:path'; import fg from 'fast-glob'; -import { parse as parseYaml } from 'yaml'; import { interpolateEnv } from '../interpolation.js'; import type { JsonObject, JsonValue } from '../types.js'; import { isJsonObject } from '../types.js'; +import { parseYamlValue } from '../yaml-loader.js'; const ANSI_YELLOW = '\u001b[33m'; const ANSI_RESET = '\u001b[0m'; @@ -38,7 +38,7 @@ function isGlobPattern(filePath: string): boolean { * Expects the file to contain an array of test objects. */ function parseYamlCases(content: string, filePath: string): JsonObject[] { - const raw = parseYaml(content) as unknown; + const raw = parseYamlValue(content); const parsed = interpolateEnv(raw, process.env); if (!Array.isArray(parsed)) { throw new Error( @@ -206,7 +206,7 @@ export async function loadCasesFromDirectory(dirPath: string): Promise; + const workspace = (parseYamlValue(workspaceContent) ?? {}) as Record; const cwd = String(workspace.cwd ?? ''); diff --git a/packages/core/src/evaluation/providers/targets-file.ts b/packages/core/src/evaluation/providers/targets-file.ts index 7e7e366f..f17729db 100644 --- a/packages/core/src/evaluation/providers/targets-file.ts +++ b/packages/core/src/evaluation/providers/targets-file.ts @@ -1,8 +1,8 @@ import { constants } from 'node:fs'; import { access, readFile } from 'node:fs/promises'; import path from 'node:path'; -import { parse } from 'yaml'; +import { parseYamlValue } from '../yaml-loader.js'; import { TARGETS_SCHEMA_V2 } from './types.js'; import type { TargetDefinition } from './types.js'; @@ -62,7 +62,7 @@ export async function readTargetDefinitions( } const raw = await readFile(absolutePath, 'utf8'); - const parsed = parse(raw) as unknown; + const parsed = parseYamlValue(raw); if (!isRecord(parsed)) { throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with a 'targets' field`); diff --git a/packages/core/src/evaluation/validation/cases-validator.ts b/packages/core/src/evaluation/validation/cases-validator.ts index d88a7a23..acea351d 100644 --- a/packages/core/src/evaluation/validation/cases-validator.ts +++ b/packages/core/src/evaluation/validation/cases-validator.ts @@ -1,7 +1,7 @@ import { readFile } from 'node:fs/promises'; import path from 'node:path'; -import { parse } from 'yaml'; +import { parseYamlValue } from '../yaml-loader.js'; import type { ValidationError, ValidationResult } from './types.js'; type JsonValue = string | number | boolean | null | JsonObject | JsonArray; @@ -26,7 +26,7 @@ export async function validateCasesFile(filePath: string): Promise { try { const content = await readFile(filePath, 'utf8'); - const parsed = parse(content) as unknown; + const parsed = parseYamlValue(content); // YAML array root → cases file (array of test case objects) if (Array.isArray(parsed)) { diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index 7d50eb73..5e1ce818 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -1,6 +1,5 @@ import { readFile } from 'node:fs/promises'; import path from 'node:path'; -import { parse } from 'yaml'; import { CLI_PLACEHOLDERS, @@ -8,6 +7,7 @@ import { findDeprecatedCamelCaseTargetWarnings, } from '../providers/targets.js'; import { KNOWN_PROVIDERS, PROVIDER_ALIASES } from '../providers/types.js'; +import { parseYamlValue } from '../yaml-loader.js'; import type { ValidationError, ValidationResult } from './types.js'; type JsonValue = string | number | boolean | null | JsonObject | JsonArray; @@ -280,7 +280,7 @@ export async function validateTargetsFile(filePath: string): Promise { const content = await readFile(filePath, 'utf8'); - const parsed = interpolateEnv(parse(content), process.env); + const parsed = interpolateEnv(parseYamlValue(content), process.env); if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return []; const obj = parsed as Record; const evalFileDir = path.dirname(path.resolve(filePath)); @@ -148,7 +148,7 @@ async function extractReposFromWorkspaceRaw(raw: unknown, evalFileDir: string): // External workspace file reference const workspaceFilePath = path.resolve(evalFileDir, raw); const content = await readFile(workspaceFilePath, 'utf8'); - const parsed = interpolateEnv(parse(content), process.env); + const parsed = interpolateEnv(parseYamlValue(content), process.env); if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return []; return extractReposFromObject(parsed as Record); } diff --git a/packages/core/src/evaluation/yaml-loader.ts b/packages/core/src/evaluation/yaml-loader.ts new file mode 100644 index 00000000..02d56845 --- /dev/null +++ b/packages/core/src/evaluation/yaml-loader.ts @@ -0,0 +1,32 @@ +/** + * Shared YAML parse boundary for AgentV configs. + * + * Why this exists: + * - We use the `yaml` package (eemeli/yaml). In its YAML 1.2 default mode it + * leaves the `<<` merge key as a literal sibling key instead of merging the + * referenced map into the parent. That leaks `<<` into downstream consumers + * (e.g. JSONL `metadata.governance` artifacts). The YAML 1.1 merge-key + * behavior must be opted into via `{ merge: true }`. + * - Every `*.eval.yaml`, `agentv.config.yaml`, workspace YAML, etc. is parsed + * here so behavior is uniform across loaders. Do NOT call `parse` from the + * `yaml` package directly elsewhere — funnel through these helpers. + * + * To extend: + * - For new YAML inputs that should support anchors + merge keys, import + * `parseYaml` (object form) or `parseYamlValue` (any-shape form) from here. + * - Do not duplicate the `{ merge: true }` option at call sites. + */ +import { parse } from 'yaml'; + +/** Options forwarded to the `yaml` package. `merge: true` is always set. */ +const PARSE_OPTIONS = { merge: true } as const; + +/** + * Parse a YAML document and return its top-level value as `unknown`. + * + * Use this when the document may be any shape (string, array, object, etc.). + * Anchor merges (`<<: *anchor`) are unwrapped into sibling keys. + */ +export function parseYamlValue(content: string): unknown { + return parse(content, PARSE_OPTIONS) as unknown; +} diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 29909476..444ec1ce 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -1,7 +1,6 @@ import { readFile, stat } from 'node:fs/promises'; import path from 'node:path'; import micromatch from 'micromatch'; -import { parse } from 'yaml'; import { collectResolvedInputFilePaths } from './input-message-utils.js'; import { interpolateEnv } from './interpolation.js'; @@ -61,6 +60,7 @@ import type { } from './types.js'; import { isJsonObject, isTestMessage } from './types.js'; import { parseRepoConfig } from './workspace/repo-config-parser.js'; +import { parseYamlValue } from './yaml-loader.js'; // Re-export public APIs from modules export { buildPromptInputs, type PromptInputs } from './formatting/prompt-builder.js'; @@ -172,7 +172,7 @@ export async function readTestSuiteMetadata(testFilePath: string): Promise<{ try { const absolutePath = path.resolve(testFilePath); const content = await readFile(absolutePath, 'utf8'); - const parsed = interpolateEnv(parse(content), process.env) as unknown; + const parsed = interpolateEnv(parseYamlValue(content), process.env) as unknown; if (!isJsonObject(parsed)) { return {}; @@ -308,7 +308,7 @@ async function loadTestsFromYaml( const config = await loadConfig(absoluteTestPath, repoRootPath); const rawFile = await readFile(absoluteTestPath, 'utf8'); - const interpolated = interpolateEnv(parse(rawFile), process.env) as unknown; + const interpolated = interpolateEnv(parseYamlValue(rawFile), process.env) as unknown; if (!isJsonObject(interpolated)) { throw new Error(`Invalid test file format: ${evalFilePath}`); } @@ -783,7 +783,7 @@ async function resolveWorkspaceConfig( } catch { throw new Error(`Workspace file not found: ${raw} (resolved to ${workspaceFilePath})`); } - const parsed = interpolateEnv(parse(content), process.env) as unknown; + const parsed = interpolateEnv(parseYamlValue(content), process.env) as unknown; if (!isJsonObject(parsed)) { throw new Error( `Invalid workspace file format: ${workspaceFilePath} (expected a YAML object)`, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d77011e4..3a7edf09 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,6 +1,7 @@ export * from './evaluation/content.js'; export * from './evaluation/types.js'; export * from './evaluation/trace.js'; +export { parseYamlValue } from './evaluation/yaml-loader.js'; export * from './evaluation/yaml-parser.js'; export { isAgentSkillsFormat, diff --git a/packages/core/test/evaluation/yaml-loader.test.ts b/packages/core/test/evaluation/yaml-loader.test.ts new file mode 100644 index 00000000..b971482f --- /dev/null +++ b/packages/core/test/evaluation/yaml-loader.test.ts @@ -0,0 +1,43 @@ +/** + * Regression test: YAML merge keys (`<<: *anchor`) are unwrapped at the parse + * boundary so the literal `<<` key never reaches downstream consumers. + * + * Bug history: PR #1166's red-team eval used `governance: { <<: *gov, ... }`. + * Because the `yaml` package leaves `<<` as a literal key in YAML 1.2 mode, + * `<<` leaked into JSONL `metadata.governance`. The fix funnels all parses + * through `parseYamlValue`, which sets `{ merge: true }`. + */ +import { describe, expect, it } from 'bun:test'; + +import { parseYamlValue } from '../../src/evaluation/yaml-loader.js'; + +describe('parseYamlValue', () => { + it('unwraps `<<: *anchor` merge keys without leaving `<<` as a sibling key', () => { + const yaml = ` +gov: &gov + schema_version: "1.0" + owasp_llm_top_10_2025: [LLM01] + risk_tier: high + +case: + <<: *gov + owasp_llm_top_10_2025: [LLM01, LLM06] +`; + + const parsed = parseYamlValue(yaml) as { case: Record }; + + // The `<<` key must NOT survive the parse — it should be unwrapped. + expect(Object.keys(parsed.case).sort()).toEqual([ + 'owasp_llm_top_10_2025', + 'risk_tier', + 'schema_version', + ]); + + // Anchor fields are merged into the case. + expect(parsed.case.schema_version).toBe('1.0'); + expect(parsed.case.risk_tier).toBe('high'); + + // Case-level overrides win over anchor values for the same key. + expect(parsed.case.owasp_llm_top_10_2025).toEqual(['LLM01', 'LLM06']); + }); +});