From aecd75569b14b0a79a94402b89604c45512598f6 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 27 Apr 2026 08:42:42 +0200 Subject: [PATCH] feat(core): optional governance metadata on EvalMetadata and EvalTest Adds an optional `governance` block (OWASP LLM Top 10 / OWASP Agentic / MITRE ATLAS / cross-framework controls / EU AI Act risk tier / owner) to suite-level EvalMetadata and case-level EvalTest.metadata. The shape is permissive: every field is optional, custom prefixes are first-class, and value validation is a soft warning, never an error. Existing evals without the block validate and run unchanged. Case-level blocks merge with suite-level (arrays concat with dedupe, scalars override). Result metadata is surfaced into the JSONL artifact so reports and `jq` pipelines can aggregate by control. Closes #1161 --- apps/cli/src/commands/eval/artifact-writer.ts | 4 + packages/core/src/evaluation/metadata.ts | 73 ++++++++- packages/core/src/evaluation/orchestrator.ts | 7 + packages/core/src/evaluation/types.ts | 6 + .../evaluation/validation/eval-validator.ts | 138 ++++++++++++++++++ packages/core/src/evaluation/yaml-parser.ts | 73 ++++++++- .../core/test/evaluation/metadata.test.ts | 46 ++++++ .../validation/eval-validator.test.ts | 120 +++++++++++++++ .../evaluation/yaml-parser-metadata.test.ts | 67 +++++++++ 9 files changed, 526 insertions(+), 8 deletions(-) diff --git a/apps/cli/src/commands/eval/artifact-writer.ts b/apps/cli/src/commands/eval/artifact-writer.ts index 755d36a1d..e005d786f 100644 --- a/apps/cli/src/commands/eval/artifact-writer.ts +++ b/apps/cli/src/commands/eval/artifact-writer.ts @@ -160,6 +160,8 @@ export interface IndexArtifactEntry { readonly output_path?: string; readonly input_path?: string; readonly response_path?: string; + /** Case-level metadata pass-through (governance taxonomies, skill tags, etc.). */ + readonly metadata?: Record; } export type ResultIndexArtifact = IndexArtifactEntry; @@ -573,6 +575,7 @@ export function buildIndexArtifactEntry( input_path: options.inputPath ? toRelativeArtifactPath(options.outputDir, options.inputPath) : undefined, + metadata: result.metadata, }; } @@ -606,6 +609,7 @@ export function buildResultIndexArtifact(result: EvaluationResult): ResultIndexA response_path: hasResponse ? path.posix.join(artifactSubdir, 'outputs', 'response.md') : undefined, + metadata: result.metadata, }; } diff --git a/packages/core/src/evaluation/metadata.ts b/packages/core/src/evaluation/metadata.ts index a66e99ffd..bb7400425 100644 --- a/packages/core/src/evaluation/metadata.ts +++ b/packages/core/src/evaluation/metadata.ts @@ -1,12 +1,56 @@ import { z } from 'zod'; import type { JsonObject } from './types.js'; +/** + * Optional governance block on suite-level `EvalMetadata` and case-level `EvalTest.metadata`. + * + * The schema is intentionally permissive: every field is optional, unknown fields pass through, + * and value validation is delegated to a soft-warning lint in `eval-validator.ts`. The block + * captures convergence on public AI-governance taxonomies (NIST AI RMF, ISO/IEC 42001, EU AI Act, + * OWASP LLM Top 10, MITRE ATLAS) without prescribing a workflow or hard-coding ID lists. + * + * Versioning lives in field names (`owasp_llm_top_10_2025`) so that when a standard revises and + * redefines IDs (OWASP LLM Top 10 v2025 vs v1.1), agentv ships a new field rather than + * silently changing the meaning of existing tags. + * + * To extend with a new versioned taxonomy: add an optional `string[]` field here, document it in + * the README under examples/red-team/, and propagate through the `agentv eval` JSONL output. + */ +const GovernanceMetadataSchema = z + .object({ + /** Schema version of this governance block itself (lets the block evolve). */ + schema_version: z.string().optional(), + /** OWASP LLM Top 10 v2025 IDs (LLM01..LLM10). */ + owasp_llm_top_10_2025: z.array(z.string()).optional(), + /** OWASP Top 10 for Agentic Applications v2025 (T1..T10). */ + owasp_agentic_top_10_2025: z.array(z.string()).optional(), + /** MITRE ATLAS technique IDs (e.g. AML.T0051, AML.T0075). */ + mitre_atlas: z.array(z.string()).optional(), + /** + * Cross-framework controls. String format: `-:`. + * Custom prefixes are first-class (e.g. `INTERNAL-AI-POLICY-3.2:CTRL-7`). + */ + controls: z.array(z.string()).optional(), + /** + * Risk vocabulary anchored to EU AI Act terminology by default. + * Allowed values: `prohibited | high | limited | minimal`. + * Other strings (e.g. NIST 800-30 `low | moderate | high`) are accepted with a soft warning. + */ + risk_tier: z.string().optional(), + /** Human-readable owner (team name, group). */ + owner: z.string().optional(), + }) + .passthrough(); + +export type GovernanceMetadata = z.infer; + const MetadataSchema = z.object({ name: z .string() .min(1) .max(64) - .regex(/^[a-z0-9-]+$/), + .regex(/^[a-z0-9-]+$/) + .optional(), description: z.string().min(1).max(1024).optional(), version: z.string().optional(), author: z.string().optional(), @@ -17,17 +61,35 @@ const MetadataSchema = z.object({ agentv: z.string().optional(), }) .optional(), + governance: GovernanceMetadataSchema.optional(), }); export type EvalMetadata = z.infer; +/** + * Extract the governance block from a suite-level YAML. Accepts either: + * - top-level `governance:` (consistent with `description`, `tags`, etc.) + * - nested `metadata.governance:` (matches the case-level shape) + * Top-level wins if both are present. + */ +function extractGovernance(suite: JsonObject): unknown { + if (suite.governance !== undefined) { + return suite.governance; + } + const wrapper = suite.metadata; + if (wrapper && typeof wrapper === 'object' && !Array.isArray(wrapper)) { + return (wrapper as Record).governance; + } + return undefined; +} + export function parseMetadata(suite: JsonObject): EvalMetadata | undefined { const hasName = typeof suite.name === 'string'; - const hasDescription = typeof suite.description === 'string'; + const governanceRaw = extractGovernance(suite); - // Only trigger metadata parsing when `name` is present. - // `description` alone doesn't trigger it since it's also used as a regular suite field. - if (!hasName) { + // Trigger metadata parsing when `name` is present, OR when a governance block exists + // (so authors can attach governance to suites that don't have a name). + if (!hasName && governanceRaw === undefined) { return undefined; } @@ -39,5 +101,6 @@ export function parseMetadata(suite: JsonObject): EvalMetadata | undefined { tags: suite.tags, license: suite.license, requires: suite.requires, + governance: governanceRaw, }); } diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index 3f494f697..0a72197cb 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -1369,6 +1369,13 @@ export async function runEvaluation( beforeAllOutputAttached = true; } + // Surface case-level metadata (e.g. governance taxonomies) on the result so + // it round-trips into the JSONL artifact and downstream consumers (reports, + // jq pipelines, attestation exports). Already-set metadata wins. + if (evalCase.metadata && !result.metadata) { + result = { ...result, metadata: evalCase.metadata }; + } + if (onProgress) { await onProgress({ workerId, diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index 6a7739216..53828126e 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -1160,6 +1160,12 @@ export interface EvaluationResult { readonly failureReasonCode?: string; /** Structured error detail (only when executionStatus === 'execution_error') */ readonly executionError?: ExecutionError; + /** + * Pass-through of `EvalTest.metadata` so case-level information (e.g. governance taxonomies, + * skill-name tags) flows into the JSONL artifact and downstream consumers without each + * surface having to thread the EvalTest separately. + */ + readonly metadata?: Record; } export type EvaluationVerdict = 'pass' | 'fail' | 'skip'; diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 309e5bfc5..4aec7132a 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -51,6 +51,8 @@ const KNOWN_TOP_LEVEL_FIELDS = new Set([ 'evaluators', 'preprocessors', 'workspace', + 'metadata', + 'governance', ]); /** @@ -195,6 +197,10 @@ export async function validateEvalFile(filePath: string): Promise-:` control string. Custom prefixes are first-class + * (e.g. `INTERNAL-AI-POLICY-3.2:CTRL-7`) — only the *shape* is checked. Returns true if the + * string has the required `:` separator AND the framework segment ends with a version-looking + * token (digit-or-dot suffix, e.g. `1.0`, `2024`, `3.2`). Misses on this heuristic produce + * a soft warning, never an error. + */ +function isWellFormedControlId(value: string): boolean { + const colonIdx = value.indexOf(':'); + if (colonIdx <= 0 || colonIdx === value.length - 1) { + return false; + } + const prefix = value.slice(0, colonIdx); + const lastSegment = prefix.split('-').pop() ?? ''; + // Version-looking: starts with a digit or contains a dot. + return /[0-9]/.test(lastSegment.charAt(0)) || lastSegment.includes('.'); +} + +/** Top-level `governance:` wins; falls back to nested `metadata.governance:`. */ +function extractGovernanceBlock(parsed: JsonObject): JsonValue | undefined { + if (parsed.governance !== undefined) { + return parsed.governance; + } + if (isObject(parsed.metadata)) { + return (parsed.metadata as JsonObject).governance; + } + return undefined; +} + +function validateGovernance( + block: JsonValue | undefined, + location: string, + filePath: string, + errors: ValidationError[], +): void { + if (block === undefined) return; + if (!isObject(block)) { + errors.push({ + severity: 'warning', + filePath, + location, + message: `'${location}' must be an object; got ${Array.isArray(block) ? 'array' : typeof block}.`, + }); + return; + } + + for (const key of Object.keys(block)) { + if (!KNOWN_GOVERNANCE_FIELDS.has(key)) { + errors.push({ + severity: 'warning', + filePath, + location: `${location}.${key}`, + message: `Unknown governance field '${key}'. Known fields: ${[...KNOWN_GOVERNANCE_FIELDS].join(', ')}.`, + }); + } + } + + const controls = block.controls; + if (controls !== undefined) { + if (!Array.isArray(controls)) { + errors.push({ + severity: 'warning', + filePath, + location: `${location}.controls`, + message: "'controls' should be an array of '-:' strings.", + }); + } else { + for (let i = 0; i < controls.length; i++) { + const entry = controls[i]; + if (typeof entry !== 'string') { + errors.push({ + severity: 'warning', + filePath, + location: `${location}.controls[${i}]`, + message: 'Control entries must be strings.', + }); + } else if (!isWellFormedControlId(entry)) { + errors.push({ + severity: 'warning', + filePath, + location: `${location}.controls[${i}]`, + message: `Malformed control '${entry}'. Expected '-:' (e.g. NIST-AI-RMF-1.0:MEASURE-2.7). Custom prefixes are allowed.`, + }); + } + } + } + } + + const riskTier = block.risk_tier; + if ( + riskTier !== undefined && + typeof riskTier === 'string' && + !EU_AI_ACT_RISK_TIERS.has(riskTier) + ) { + errors.push({ + severity: 'warning', + filePath, + location: `${location}.risk_tier`, + message: `'risk_tier: ${riskTier}' is outside EU AI Act vocabulary (prohibited | high | limited | minimal). Other vocabularies (e.g. NIST 800-30) are accepted but flagged.`, + }); + } +} diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 88ae9bc47..29909476a 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -79,7 +79,7 @@ export { } from './loaders/config-loader.js'; export type { AgentVConfig, CacheConfig, ExecutionDefaults } from './loaders/config-loader.js'; export { detectFormat } from './loaders/jsonl-parser.js'; -export type { EvalMetadata } from './metadata.js'; +export type { EvalMetadata, GovernanceMetadata } from './metadata.js'; const ANSI_YELLOW = '\u001b[33m'; const ANSI_RED = '\u001b[31m'; @@ -361,6 +361,10 @@ async function loadTestsFromYaml( const suiteWorkspace = await resolveWorkspaceConfig(suite.workspace, evalFileDir); + // Suite-level governance block (top-level `governance:` wins over `metadata.governance:`). + // Per-case `metadata.governance` merges with this — arrays concatenate, scalars override. + const suiteGovernance = extractSuiteGovernance(suite); + // Resolve suite-level input (prepended to each test's input messages) const suiteInputMessages = expandInputShorthand(suite.input); @@ -543,10 +547,12 @@ async function loadTestsFromYaml( const caseWorkspace = await resolveWorkspaceConfig(testCaseConfig.workspace, evalFileDir); const mergedWorkspace = mergeWorkspaceConfigs(suiteWorkspace, caseWorkspace); - // Parse per-case metadata - const metadata = isJsonObject(testCaseConfig.metadata) + // Parse per-case metadata, then merge suite-level governance onto the case. + // Arrays concatenate (suite controls + case controls), scalars on the case win. + const rawCaseMetadata = isJsonObject(testCaseConfig.metadata) ? (testCaseConfig.metadata as Record) : undefined; + const metadata = mergeCaseGovernance(rawCaseMetadata, suiteGovernance); // Extract per-test targets override (matrix evaluation) const caseTargets = extractTargetsFromTestCase(testCaseConfig as JsonObject); @@ -926,6 +932,67 @@ function asString(value: unknown): string | undefined { return typeof value === 'string' ? value : undefined; } +/** + * Pull the optional `governance` block out of a suite YAML. Top-level `governance:` wins + * over the nested `metadata.governance:` form so that authors who already use top-level + * suite metadata fields (`name`, `description`, `tags`) can keep their existing layout. + */ +function extractSuiteGovernance(suite: RawTestSuite): Record | undefined { + const top = (suite as JsonObject).governance; + if (isJsonObject(top)) { + return top as Record; + } + const wrapper = (suite as JsonObject).metadata; + if (isJsonObject(wrapper)) { + const nested = (wrapper as JsonObject).governance; + if (isJsonObject(nested)) { + return nested as Record; + } + } + return undefined; +} + +/** + * Merge suite-level governance into a case's metadata. Arrays concatenate (suite controls + + * case controls, deduplicated); scalar fields on the case override the suite. The case's + * other metadata keys are preserved verbatim. Returns undefined if neither side has anything. + */ +function mergeCaseGovernance( + caseMetadata: Record | undefined, + suiteGovernance: Record | undefined, +): Record | undefined { + const caseGovernance = isJsonObject(caseMetadata?.governance) + ? (caseMetadata?.governance as Record) + : undefined; + + if (!suiteGovernance && !caseGovernance) { + return caseMetadata; + } + + const merged: Record = { ...(suiteGovernance ?? {}) }; + if (caseGovernance) { + for (const [key, caseValue] of Object.entries(caseGovernance)) { + const suiteValue = merged[key]; + if (Array.isArray(suiteValue) && Array.isArray(caseValue)) { + const seen = new Set(); + const out: unknown[] = []; + for (const v of [...suiteValue, ...caseValue]) { + const key = typeof v === 'string' ? v : JSON.stringify(v); + if (!seen.has(key)) { + seen.add(key); + out.push(v); + } + } + merged[key] = out; + } else { + merged[key] = caseValue; + } + } + } + + return { ...(caseMetadata ?? {}), governance: merged }; +} + function logWarning(message: string, details?: readonly string[]): void { if (details && details.length > 0) { const detailBlock = details.join('\n'); diff --git a/packages/core/test/evaluation/metadata.test.ts b/packages/core/test/evaluation/metadata.test.ts index 0858cdcb7..96cec9ab8 100644 --- a/packages/core/test/evaluation/metadata.test.ts +++ b/packages/core/test/evaluation/metadata.test.ts @@ -52,4 +52,50 @@ describe('parseMetadata', () => { description: 'A simple eval', }); }); + + it('parses an optional governance block at the top level', () => { + const result = parseMetadata({ + name: 'red-team', + governance: { + schema_version: '1.0', + owasp_llm_top_10_2025: ['LLM01'], + controls: ['NIST-AI-RMF-1.0:MEASURE-2.7'], + risk_tier: 'high', + }, + }); + expect(result?.governance).toEqual({ + schema_version: '1.0', + owasp_llm_top_10_2025: ['LLM01'], + controls: ['NIST-AI-RMF-1.0:MEASURE-2.7'], + risk_tier: 'high', + }); + }); + + it('parses governance from nested metadata.governance form', () => { + const result = parseMetadata({ + name: 'red-team', + metadata: { + governance: { owasp_llm_top_10_2025: ['LLM06'], owner: 'security-team' }, + }, + }); + expect(result?.governance).toEqual({ + owasp_llm_top_10_2025: ['LLM06'], + owner: 'security-team', + }); + }); + + it('returns metadata when only governance is present (no name)', () => { + const result = parseMetadata({ + governance: { risk_tier: 'high' }, + }); + expect(result).toEqual({ governance: { risk_tier: 'high' } }); + }); + + it('passes unknown governance keys through (custom taxonomies extend without forking)', () => { + const result = parseMetadata({ + name: 'red-team', + governance: { custom_company_taxonomy: ['X-1'] }, + }); + expect(result?.governance).toEqual({ custom_company_taxonomy: ['X-1'] }); + }); }); diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 7992160b8..975be6a00 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -495,6 +495,126 @@ tests: }); }); + describe('governance metadata validation', () => { + it('passes a well-formed governance block with custom prefixes (no warnings)', async () => { + const filePath = path.join(tempDir, 'governance-ok.yaml'); + await writeFile( + filePath, + `name: red-team +governance: + schema_version: "1.0" + owasp_llm_top_10_2025: [LLM01] + controls: + - NIST-AI-RMF-1.0:MEASURE-2.7 + - INTERNAL-AI-POLICY-3.2:CTRL-7 + risk_tier: high +tests: + - id: case-1 + input: "Query" +`, + ); + + const result = await validateEvalFile(filePath); + expect(result.valid).toBe(true); + const warnings = result.errors.filter((e) => e.severity === 'warning'); + expect(warnings).toHaveLength(0); + }); + + it('warns on unknown governance fields without erroring', async () => { + const filePath = path.join(tempDir, 'governance-typo.yaml'); + await writeFile( + filePath, + `name: red-team +governance: + owasp_lm_top_10_2025: [LLM01] +tests: + - id: case-1 + input: "Query" +`, + ); + + const result = await validateEvalFile(filePath); + expect(result.valid).toBe(true); + const warnings = result.errors.filter((e) => e.severity === 'warning'); + expect( + warnings.some((w) => w.message.includes("Unknown governance field 'owasp_lm_top_10_2025'")), + ).toBe(true); + }); + + it('warns on malformed control strings (missing version segment)', async () => { + const filePath = path.join(tempDir, 'governance-malformed-control.yaml'); + await writeFile( + filePath, + `name: red-team +governance: + controls: ["NIST-AI-RMF:MEASURE-2.7"] +tests: + - id: case-1 + input: "Query" +`, + ); + + const result = await validateEvalFile(filePath); + expect(result.valid).toBe(true); + const warnings = result.errors.filter((e) => e.severity === 'warning'); + expect(warnings.some((w) => w.message.includes('Malformed control'))).toBe(true); + }); + + it('warns on risk_tier outside EU AI Act vocabulary', async () => { + const filePath = path.join(tempDir, 'governance-risk.yaml'); + await writeFile( + filePath, + `name: red-team +governance: + risk_tier: critical +tests: + - id: case-1 + input: "Query" +`, + ); + + const result = await validateEvalFile(filePath); + expect(result.valid).toBe(true); + const warnings = result.errors.filter((e) => e.severity === 'warning'); + expect(warnings.some((w) => w.message.includes('EU AI Act vocabulary'))).toBe(true); + }); + + it('warns on case-level governance typos', async () => { + const filePath = path.join(tempDir, 'governance-case.yaml'); + await writeFile( + filePath, + `tests: + - id: case-1 + input: "Query" + metadata: + governance: + owasp_lm_top_10_2025: [LLM01] +`, + ); + + const result = await validateEvalFile(filePath); + expect(result.valid).toBe(true); + const warnings = result.errors.filter((e) => e.severity === 'warning'); + expect(warnings.some((w) => w.location?.includes('tests[0].metadata.governance'))).toBe(true); + }); + + it('does not warn when no governance block is present', async () => { + const filePath = path.join(tempDir, 'governance-absent.yaml'); + await writeFile( + filePath, + `tests: + - id: case-1 + input: "Query" +`, + ); + + const result = await validateEvalFile(filePath); + expect(result.valid).toBe(true); + const warnings = result.errors.filter((e) => e.severity === 'warning'); + expect(warnings.filter((w) => w.location?.startsWith('governance'))).toHaveLength(0); + }); + }); + describe('tests as string path', () => { it('validates tests string has valid extension', async () => { const filePath = path.join(tempDir, 'tests-bad-ext.yaml'); diff --git a/packages/core/test/evaluation/yaml-parser-metadata.test.ts b/packages/core/test/evaluation/yaml-parser-metadata.test.ts index 0c4253055..bb27f8c3c 100644 --- a/packages/core/test/evaluation/yaml-parser-metadata.test.ts +++ b/packages/core/test/evaluation/yaml-parser-metadata.test.ts @@ -132,4 +132,71 @@ tests: const suite = await loadTestSuite(filePath, dir); expect(suite.metadata?.tags).toEqual(['unit', 'integration', 'smoke']); }); + + it('parses suite-level governance from top-level governance:', async () => { + const { filePath, dir } = createTempYaml(` +name: red-team +governance: + schema_version: "1.0" + owasp_llm_top_10_2025: [LLM01] + controls: + - NIST-AI-RMF-1.0:MEASURE-2.7 + risk_tier: high + owner: security-team +tests: + - id: case-1 + criteria: "Refuses" + input: "Query" +`); + + const suite = await loadTestSuite(filePath, dir); + expect(suite.metadata?.governance).toEqual({ + schema_version: '1.0', + owasp_llm_top_10_2025: ['LLM01'], + controls: ['NIST-AI-RMF-1.0:MEASURE-2.7'], + risk_tier: 'high', + owner: 'security-team', + }); + }); + + it('merges case-level governance into suite-level (arrays concat, scalars override)', async () => { + const { filePath, dir } = createTempYaml(` +name: red-team +governance: + owasp_llm_top_10_2025: [LLM01] + controls: + - NIST-AI-RMF-1.0:MEASURE-2.7 + risk_tier: high +tests: + - id: case-1 + criteria: "Refuses" + input: "Query" + metadata: + governance: + owasp_llm_top_10_2025: [LLM06] + risk_tier: limited +`); + + const suite = await loadTestSuite(filePath, dir); + const govern = suite.tests[0].metadata?.governance as Record; + expect(govern.owasp_llm_top_10_2025).toEqual(['LLM01', 'LLM06']); + expect(govern.controls).toEqual(['NIST-AI-RMF-1.0:MEASURE-2.7']); + expect(govern.risk_tier).toBe('limited'); + }); + + it('keeps suite governance on cases that have no metadata of their own', async () => { + const { filePath, dir } = createTempYaml(` +governance: + owasp_llm_top_10_2025: [LLM01] +tests: + - id: case-1 + criteria: "Refuses" + input: "Query" +`); + + const suite = await loadTestSuite(filePath, dir); + expect(suite.tests[0].metadata?.governance).toEqual({ + owasp_llm_top_10_2025: ['LLM01'], + }); + }); });