diff --git a/apps/web/src/content/docs/docs/evaluation/eval-cases.mdx b/apps/web/src/content/docs/docs/evaluation/eval-cases.mdx index afe5cae07..2f69246c2 100644 --- a/apps/web/src/content/docs/docs/evaluation/eval-cases.mdx +++ b/apps/web/src/content/docs/docs/evaluation/eval-cases.mdx @@ -328,6 +328,19 @@ tests: # No assertions → default llm-grader evaluates against criteria ``` +Suite-level `preprocessors` also apply to this implicit grader. That matters when the agent output is a `ContentFile` block rather than plain text: + +```yaml +preprocessors: + - type: xlsx + command: ["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"] + +tests: + - id: spreadsheet-eval + criteria: Output includes the revenue rows + input: Generate the spreadsheet report +``` + ### `assertions` present — explicit evaluators only When `assertions` is defined, only the declared evaluators run. No implicit grader is added. Graders that are declared (such as `llm-grader`, `code-grader`, or `rubrics`) receive `criteria` as input automatically. @@ -353,6 +366,26 @@ tests: value: "fix" ``` +When you need a custom file conversion for only one grader, add `preprocessors` directly to that evaluator: + +```yaml +preprocessors: + - type: xlsx + command: ["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"] + +tests: + - id: mixed-eval + criteria: Response is helpful and mentions the fix + input: "Debug this function..." + assertions: + - type: llm-grader + preprocessors: + - type: xlsx + command: ["bun", "run", "scripts/preprocessors/xlsx-to-json.ts"] + - type: contains + value: "fix" +``` + ## Metadata Pass additional context to evaluators via the `metadata` field: diff --git a/apps/web/src/content/docs/docs/evaluation/examples.mdx b/apps/web/src/content/docs/docs/evaluation/examples.mdx index d7c33e7e6..d6346512f 100644 --- a/apps/web/src/content/docs/docs/evaluation/examples.mdx +++ b/apps/web/src/content/docs/docs/evaluation/examples.mdx @@ -103,6 +103,35 @@ tests: } ``` +## File Output Preprocessing + +Convert a binary file output into text before the `llm-grader` sees it: + +```yaml +description: Grade spreadsheet output via a preprocessor + +preprocessors: + - type: xlsx + command: ["bun", "run", "../scripts/preprocessors/xlsx-to-csv.ts"] + +execution: + target: file_output + +tests: + - id: spreadsheet-output + input: Generate the spreadsheet report + criteria: The extracted spreadsheet content includes the revenue rows + assertions: + - name: file-check + type: llm-grader + prompt: | + Check whether the transformed spreadsheet text contains the revenue rows: + + {{ output }} +``` + +See [`examples/features/preprocessors/`](../../../../examples/features/preprocessors/) for a runnable end-to-end example with a file-producing target and custom grader target. + ## Tool Trajectory Validate that an agent uses specific tools during execution: diff --git a/apps/web/src/content/docs/docs/evaluators/llm-graders.mdx b/apps/web/src/content/docs/docs/evaluators/llm-graders.mdx index 9e17027c8..51d0db372 100644 --- a/apps/web/src/content/docs/docs/evaluators/llm-graders.mdx +++ b/apps/web/src/content/docs/docs/evaluators/llm-graders.mdx @@ -143,6 +143,42 @@ assertions: The `config` object is available as `ctx.config` inside the template function. +## Preprocessing File Outputs + +If an agent returns a `ContentFile` block instead of plain text, you can preprocess that file into text before `llm-grader` builds the candidate prompt. + +AgentV always tries a default UTF-8 text read first. That is enough for text-based formats such as CSV, JSON, SQL, Markdown, YAML, HTML, XML, and plain text. For binary formats such as `.xlsx`, `.pdf`, or `.docx`, add a preprocessor command: + +```yaml +preprocessors: + - type: xlsx + command: ["bun", "run", "scripts/preprocessors/xlsx-to-csv.ts"] + +tests: + - id: spreadsheet-output + criteria: Output includes the revenue rows + input: Generate the spreadsheet report + assertions: + - name: spreadsheet-check + type: llm-grader + prompt: | + Check whether the transformed spreadsheet text contains the revenue rows: + + {{ output }} +``` + +`type` accepts either a short alias such as `xlsx` or a full MIME type such as `application/vnd.openxmlformats-officedocument.spreadsheetml.sheet`. + +Resolution order: + +- per-evaluator `preprocessors` override suite-level entries +- if no preprocessor matches, AgentV falls back to a UTF-8 text read +- if the fallback read looks binary or invalid, the grader receives a warning note instead of failing the test run + +The implicit default `llm-grader` also inherits suite-level `preprocessors`, so you can omit `assertions` and still preprocess file outputs before grading. + +See [`examples/features/preprocessors/`](../../../../examples/features/preprocessors/) for a runnable example with a file-producing target and a custom preprocessor script. + ## Available Context Fields TypeScript templates receive a context object with these fields: diff --git a/examples/features/README.md b/examples/features/README.md index f3eb92fa5..72d50e39d 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -21,6 +21,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [composite](composite/) | Safety gate and weighted aggregation patterns | | [threshold-evaluator](threshold-evaluator/) | Pass a test if a configurable percentage of sub-evaluators pass | | [multi-turn-conversation](multi-turn-conversation/) | Grade a multi-turn conversation with per-turn score breakdowns | +| [preprocessors](preprocessors/) | Convert `ContentFile` outputs into grader-readable text before `llm-grader` runs | --- @@ -159,6 +160,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [matrix-evaluation](matrix-evaluation/) | Benchmarking | | [multi-turn-conversation](multi-turn-conversation/) | LLM grading | | [nlp-metrics](nlp-metrics/) | Deterministic assertions | +| [preprocessors](preprocessors/) | LLM grading | | [prompt-template-sdk](prompt-template-sdk/) | TypeScript SDK | | [repo-lifecycle](repo-lifecycle/) | Workspace & targets | | [rubric](rubric/) | LLM grading | diff --git a/examples/features/preprocessors/.agentv/providers/file-output.ts b/examples/features/preprocessors/.agentv/providers/file-output.ts new file mode 100644 index 000000000..a85b4ed61 --- /dev/null +++ b/examples/features/preprocessors/.agentv/providers/file-output.ts @@ -0,0 +1,30 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; + +const outputFile = process.argv[2]; +if (!outputFile) { + throw new Error('missing output file path'); +} + +const generatedDir = path.join(process.cwd(), 'generated'); +mkdirSync(generatedDir, { recursive: true }); +writeFileSync(path.join(generatedDir, 'report.xlsx'), Buffer.from([0, 159, 146, 150])); + +writeFileSync( + outputFile, + JSON.stringify({ + output: [ + { + role: 'assistant', + content: [ + { + type: 'file', + media_type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + path: 'generated/report.xlsx', + }, + ], + }, + ], + }), + 'utf8', +); diff --git a/examples/features/preprocessors/.agentv/providers/grader-check.ts b/examples/features/preprocessors/.agentv/providers/grader-check.ts new file mode 100644 index 000000000..130991ffc --- /dev/null +++ b/examples/features/preprocessors/.agentv/providers/grader-check.ts @@ -0,0 +1,30 @@ +import { readFileSync, writeFileSync } from 'node:fs'; + +const promptFile = process.argv[2]; +const outputFile = process.argv[3]; + +if (!promptFile || !outputFile) { + throw new Error('missing args'); +} + +const prompt = readFileSync(promptFile, 'utf8'); +const passed = prompt.includes('spreadsheet: revenue,total') && prompt.includes('Q1,42'); + +writeFileSync( + outputFile, + JSON.stringify({ + text: JSON.stringify({ + score: passed ? 1 : 0, + assertions: [ + { + text: 'preprocessed file content reached the llm grader', + passed, + evidence: passed + ? 'found transformed spreadsheet text in prompt' + : 'transformed spreadsheet text missing from prompt', + }, + ], + }), + }), + 'utf8', +); diff --git a/examples/features/preprocessors/.agentv/targets.yaml b/examples/features/preprocessors/.agentv/targets.yaml new file mode 100644 index 000000000..4afc8a559 --- /dev/null +++ b/examples/features/preprocessors/.agentv/targets.yaml @@ -0,0 +1,12 @@ +$schema: agentv-targets-v2.2 +targets: + - name: file_output + provider: file-output + command: bun run .agentv/providers/file-output.ts {OUTPUT_FILE} + cwd: .. + grader_target: grader_check + + - name: grader_check + provider: grader-check + command: bun run .agentv/providers/grader-check.ts {PROMPT_FILE} {OUTPUT_FILE} + cwd: .. diff --git a/examples/features/preprocessors/README.md b/examples/features/preprocessors/README.md new file mode 100644 index 000000000..ad095b2cb --- /dev/null +++ b/examples/features/preprocessors/README.md @@ -0,0 +1,27 @@ +# Content Preprocessors + +Demonstrates how `llm-grader` preprocessors turn `ContentFile` outputs into text before grading. + +## What This Shows + +- top-level `preprocessors:` shared by all graders in an eval +- an agent target returning a `ContentFile` block instead of plain text +- an `llm-grader` receiving transformed spreadsheet text +- relative `ContentFile.path` resolution against the target workspace + +## Running + +```bash +# From repository root +bun apps/cli/src/cli.ts eval examples/features/preprocessors/evals/dataset.eval.yaml --target file_output +``` + +Expected result: the eval passes because the grader sees the transformed spreadsheet text from `generated/report.xlsx`. + +## Key Files + +- `evals/dataset.eval.yaml` - eval with top-level `preprocessors` +- `.agentv/targets.yaml` - custom file-producing target and custom grader target +- `.agentv/providers/file-output.ts` - emits a relative `ContentFile` path +- `.agentv/providers/grader-check.ts` - passes only when transformed text reaches the grader prompt +- `scripts/preprocessors/xlsx-to-csv.ts` - example spreadsheet preprocessor script diff --git a/examples/features/preprocessors/evals/dataset.eval.yaml b/examples/features/preprocessors/evals/dataset.eval.yaml new file mode 100644 index 000000000..af4f0efb3 --- /dev/null +++ b/examples/features/preprocessors/evals/dataset.eval.yaml @@ -0,0 +1,17 @@ +description: Convert file outputs to grader-readable text before llm grading + +preprocessors: + - type: xlsx + command: ["bun", "run", "../scripts/preprocessors/xlsx-to-csv.ts"] + +tests: + - id: spreadsheet-output + input: Generate the spreadsheet report + criteria: The extracted spreadsheet content includes the revenue rows + assertions: + - name: file-check + type: llm-grader + prompt: | + Check whether the answer contains the transformed spreadsheet text: + + {{ output }} diff --git a/examples/features/preprocessors/scripts/preprocessors/xlsx-to-csv.ts b/examples/features/preprocessors/scripts/preprocessors/xlsx-to-csv.ts new file mode 100644 index 000000000..c1dd4860f --- /dev/null +++ b/examples/features/preprocessors/scripts/preprocessors/xlsx-to-csv.ts @@ -0,0 +1,12 @@ +import { readFileSync } from 'node:fs'; + +const payload = JSON.parse(readFileSync(0, 'utf8')) as { path?: string }; + +if (!payload.path) { + throw new Error('missing file path'); +} + +// Example-only placeholder transformation. Copy this script into your project +// and replace it with real spreadsheet extraction logic. +console.log('spreadsheet: revenue,total'); +console.log('Q1,42'); diff --git a/packages/core/src/evaluation/content-preprocessor.ts b/packages/core/src/evaluation/content-preprocessor.ts new file mode 100644 index 000000000..ab5fb295c --- /dev/null +++ b/packages/core/src/evaluation/content-preprocessor.ts @@ -0,0 +1,210 @@ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { execFileWithStdin } from '../runtime/exec.js'; +import type { Content, ContentFile } from './content.js'; +import type { ContentPreprocessorConfig } from './types.js'; + +const MIME_TYPE_ALIASES: Record = { + csv: 'text/csv', + docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + htm: 'text/html', + html: 'text/html', + json: 'application/json', + markdown: 'text/markdown', + md: 'text/markdown', + pdf: 'application/pdf', + sql: 'application/sql', + txt: 'text/plain', + xhtml: 'application/xhtml+xml', + xls: 'application/vnd.ms-excel', + xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + xml: 'application/xml', + yaml: 'application/yaml', + yml: 'application/yaml', +}; + +const REPLACEMENT_CHAR = '\ufffd'; + +export interface FilePreprocessingWarning { + readonly file: string; + readonly mediaType: string; + readonly reason: string; +} + +export interface ExtractedContentText { + readonly text: string; + readonly warnings: readonly FilePreprocessingWarning[]; +} + +export async function extractTextWithPreprocessors( + content: string | readonly Content[] | undefined, + preprocessors: readonly ContentPreprocessorConfig[] | undefined, + options: { readonly basePath?: string } = {}, +): Promise { + if (typeof content === 'string') { + return { text: content, warnings: [] }; + } + if (!content || content.length === 0) { + return { text: '', warnings: [] }; + } + + const parts: string[] = []; + const warnings: FilePreprocessingWarning[] = []; + + for (const block of content) { + if (block.type === 'text') { + parts.push(block.text); + continue; + } + if (block.type !== 'file') { + continue; + } + + const result = await preprocessContentFile(block, preprocessors, options.basePath); + if (result.text) { + parts.push(result.text); + } + warnings.push(...result.warnings); + } + + return { text: parts.join('\n'), warnings }; +} + +async function preprocessContentFile( + block: ContentFile, + preprocessors: readonly ContentPreprocessorConfig[] | undefined, + basePath?: string, +): Promise { + const mediaType = normalizePreprocessorType(block.media_type); + const resolvedPath = resolveLocalFilePath(block.path, basePath); + + if (!resolvedPath) { + return { + text: '', + warnings: [ + { + file: block.path, + mediaType: block.media_type, + reason: 'remote file paths are not supported for preprocessing', + }, + ], + }; + } + + const preprocessor = preprocessors?.find( + (entry) => normalizePreprocessorType(entry.type) === mediaType, + ); + if (preprocessor) { + return runContentPreprocessor(block, resolvedPath, preprocessor); + } + + try { + const buffer = await readFile(resolvedPath); + const text = buffer.toString('utf8').replace(/\r\n/g, '\n'); + if (buffer.includes(0) || text.includes(REPLACEMENT_CHAR)) { + return { + text: '', + warnings: [ + { + file: block.path, + mediaType: block.media_type, + reason: 'default UTF-8 read produced binary or invalid text; configure a preprocessor', + }, + ], + }; + } + return { text: formatFileText(block.path, text), warnings: [] }; + } catch (error) { + return { + text: '', + warnings: [ + { + file: block.path, + mediaType: block.media_type, + reason: error instanceof Error ? error.message : String(error), + }, + ], + }; + } +} + +async function runContentPreprocessor( + block: ContentFile, + resolvedPath: string, + preprocessor: ContentPreprocessorConfig, +): Promise { + try { + const argv = preprocessor.resolvedCommand ?? preprocessor.command; + const { stdout, stderr, exitCode } = await execFileWithStdin( + argv, + JSON.stringify({ + path: resolvedPath, + original_path: block.path, + media_type: block.media_type, + }), + ); + + if (exitCode !== 0) { + return { + text: '', + warnings: [ + { + file: block.path, + mediaType: block.media_type, + reason: stderr.trim() || `preprocessor exited with code ${exitCode}`, + }, + ], + }; + } + + return { text: formatFileText(block.path, stdout.trim()), warnings: [] }; + } catch (error) { + return { + text: '', + warnings: [ + { + file: block.path, + mediaType: block.media_type, + reason: error instanceof Error ? error.message : String(error), + }, + ], + }; + } +} + +export function appendPreprocessingWarnings( + text: string, + warnings: readonly FilePreprocessingWarning[], +): string { + if (warnings.length === 0) { + return text; + } + + const notes = warnings.map( + (warning) => + `[file preprocessing warning] ${warning.file} (${warning.mediaType}): ${warning.reason}`, + ); + + return [text, ...notes].filter((part) => part.length > 0).join('\n'); +} + +export function normalizePreprocessorType(value: string): string { + const normalized = value.trim().toLowerCase(); + return MIME_TYPE_ALIASES[normalized] ?? normalized; +} + +function resolveLocalFilePath(value: string, basePath?: string): string | undefined { + if (value.startsWith('file://')) { + return fileURLToPath(value); + } + if (/^[a-z]+:\/\//i.test(value)) { + return undefined; + } + return basePath ? path.resolve(basePath, value) : path.resolve(value); +} + +function formatFileText(filePath: string, text: string): string { + return `[[ file: ${filePath} ]]\n${text}`; +} diff --git a/packages/core/src/evaluation/evaluators/llm-grader.ts b/packages/core/src/evaluation/evaluators/llm-grader.ts index d53599358..cc6e482c7 100644 --- a/packages/core/src/evaluation/evaluators/llm-grader.ts +++ b/packages/core/src/evaluation/evaluators/llm-grader.ts @@ -4,6 +4,10 @@ import path from 'node:path'; import { generateText, stepCountIs, tool } from 'ai'; import { z } from 'zod'; +import { + appendPreprocessingWarnings, + extractTextWithPreprocessors, +} from '../content-preprocessor.js'; import type { ContentImage } from '../content.js'; import { isContentArray } from '../content.js'; import type { Message, Provider, ProviderResponse } from '../providers/types.js'; @@ -151,6 +155,24 @@ interface StructuredGenerationResult { readonly tokenUsage?: TokenUsage; } +function resolveContentBasePath(context: EvaluationContext): string | undefined { + if (context.workspacePath) { + return context.workspacePath; + } + + if ( + 'config' in context.target && + context.target.config && + typeof context.target.config === 'object' && + 'cwd' in context.target.config && + typeof context.target.config.cwd === 'string' + ) { + return context.target.config.cwd; + } + + return undefined; +} + export class LlmGraderEvaluator implements Evaluator { readonly kind = 'llm-grader'; @@ -172,33 +194,61 @@ export class LlmGraderEvaluator implements Evaluator { } async evaluate(context: EvaluationContext): Promise { + const preparedContext = await this.prepareContext(context); + // Delegate mode: grader target provider is an agent provider — send prompt via invoke() if (this.graderTargetProvider) { - return this.evaluateWithGraderTarget(context); + return this.evaluateWithGraderTarget(preparedContext); } - const graderProvider = await this.resolveGraderProvider(context); + const graderProvider = await this.resolveGraderProvider(preparedContext); if (!graderProvider) { throw new Error('No grader provider available for LLM grading'); } // Built-in agent mode: agentv provider → AI SDK generateText with filesystem tools if (graderProvider.kind === 'agentv') { - return this.evaluateBuiltIn(context, graderProvider); + return this.evaluateBuiltIn(preparedContext, graderProvider); } // Delegate mode: resolved provider is an agent provider → send prompt via invoke() if (isAgentProvider(graderProvider)) { - return this.evaluateWithDelegatedAgent(context, graderProvider); + return this.evaluateWithDelegatedAgent(preparedContext, graderProvider); } // LLM mode: structured JSON evaluation - const config = context.evaluator; + const config = preparedContext.evaluator; if (config?.type === 'llm-grader' && config.rubrics && config.rubrics.length > 0) { - return this.evaluateWithRubrics(context, graderProvider, config.rubrics); + return this.evaluateWithRubrics(preparedContext, graderProvider, config.rubrics); } - return this.evaluateFreeform(context, graderProvider); + return this.evaluateFreeform(preparedContext, graderProvider); + } + + private async prepareContext(context: EvaluationContext): Promise { + const config = context.evaluator; + if (config?.type !== 'llm-grader' || !context.output) { + return context; + } + + const lastAssistant = [...context.output] + .reverse() + .find((message) => message.role === 'assistant' && message.content !== undefined); + if (!lastAssistant || typeof lastAssistant.content === 'string') { + return context; + } + + const extracted = await extractTextWithPreprocessors( + lastAssistant.content, + config.preprocessors, + { + basePath: resolveContentBasePath(context), + }, + ); + return { + ...context, + candidate: appendPreprocessingWarnings(extracted.text, extracted.warnings), + }; } // --------------------------------------------------------------------------- diff --git a/packages/core/src/evaluation/loaders/evaluator-parser.ts b/packages/core/src/evaluation/loaders/evaluator-parser.ts index 76cfae0e7..79385fe81 100644 --- a/packages/core/src/evaluation/loaders/evaluator-parser.ts +++ b/packages/core/src/evaluation/loaders/evaluator-parser.ts @@ -1,7 +1,14 @@ import path from 'node:path'; +import { normalizePreprocessorType } from '../content-preprocessor.js'; import type { ToolTrajectoryEvaluatorConfig, ToolTrajectoryExpectedItem } from '../trace.js'; -import type { EvaluatorConfig, EvaluatorKind, JsonObject, JsonValue } from '../types.js'; +import type { + ContentPreprocessorConfig, + EvaluatorConfig, + EvaluatorKind, + JsonObject, + JsonValue, +} from '../types.js'; import { isEvaluatorKind } from '../types.js'; import { validateCustomPromptContent } from '../validation/prompt-validator.js'; import { resolveFileReference } from './file-resolver.js'; @@ -48,6 +55,7 @@ export async function parseEvaluators( globalExecution: JsonObject | undefined, searchRoots: readonly string[], evalId: string, + defaultPreprocessors?: readonly ContentPreprocessorConfig[], ): Promise { const execution = rawEvalCase.execution; const executionObject = isJsonObject(execution) ? execution : undefined; @@ -66,9 +74,19 @@ export async function parseEvaluators( : (globalExecution?.assertions ?? globalExecution?.assert ?? globalExecution?.evaluators); // deprecated: use assertions // Parse case-level evaluators - const parsedCase = await parseEvaluatorList(caseEvaluators, searchRoots, evalId); + const parsedCase = await parseEvaluatorList( + caseEvaluators, + searchRoots, + evalId, + defaultPreprocessors, + ); // Parse root-level evaluators (appended after case-level) - const parsedRoot = await parseEvaluatorList(rootEvaluators, searchRoots, evalId); + const parsedRoot = await parseEvaluatorList( + rootEvaluators, + searchRoots, + evalId, + defaultPreprocessors, + ); if (!parsedCase && !parsedRoot) { return undefined; @@ -87,6 +105,7 @@ async function parseEvaluatorList( candidateEvaluators: JsonValue | undefined, searchRoots: readonly string[], evalId: string, + defaultPreprocessors?: readonly ContentPreprocessorConfig[], ): Promise { if (candidateEvaluators === undefined) { return undefined; @@ -175,6 +194,13 @@ async function parseEvaluatorList( } const negate = rawEvaluator.negate === true ? true : undefined; + const mergedPreprocessors = await parseMergedPreprocessors( + rawEvaluator.preprocessors as JsonValue | undefined, + defaultPreprocessors, + searchRoots, + name, + evalId, + ); // Custom assertion types — store with their type name for registry dispatch if (isCustomType) { @@ -297,6 +323,7 @@ async function parseEvaluatorList( 'cwd', 'weight', 'target', + 'preprocessors', 'required', 'negate', ]); @@ -318,6 +345,7 @@ async function parseEvaluatorList( ...(min_score !== undefined ? { min_score } : {}), ...(negate !== undefined ? { negate } : {}), ...(Object.keys(config).length > 0 ? { config } : {}), + ...(mergedPreprocessors ? { preprocessors: mergedPreprocessors } : {}), ...(targetConfig !== undefined ? { target: targetConfig } : {}), }); continue; @@ -1236,6 +1264,7 @@ async function parseEvaluatorList( ...(required !== undefined ? { required } : {}), ...(min_score !== undefined ? { min_score } : {}), ...(negate !== undefined ? { negate } : {}), + ...(mergedPreprocessors ? { preprocessors: mergedPreprocessors } : {}), }); continue; } @@ -1346,6 +1375,7 @@ async function parseEvaluatorList( ...(required !== undefined ? { required } : {}), ...(min_score !== undefined ? { min_score } : {}), ...(negate !== undefined ? { negate } : {}), + ...(mergedPreprocessors ? { preprocessors: mergedPreprocessors } : {}), }); continue; } @@ -1375,6 +1405,7 @@ async function parseEvaluatorList( 'max_steps', 'maxSteps', 'temperature', + 'preprocessors', ]); const config: Record = {}; for (const [key, value] of Object.entries(rawEvaluator)) { @@ -1422,12 +1453,92 @@ async function parseEvaluatorList( ...(finalConfig ? { config: finalConfig } : {}), ...(llmMaxSteps !== undefined ? { max_steps: llmMaxSteps } : {}), ...(llmTemperature !== undefined ? { temperature: llmTemperature } : {}), + ...(mergedPreprocessors ? { preprocessors: mergedPreprocessors } : {}), }); } return evaluators.length > 0 ? evaluators : undefined; } +async function parseMergedPreprocessors( + rawValue: JsonValue | undefined, + defaultPreprocessors: readonly ContentPreprocessorConfig[] | undefined, + searchRoots: readonly string[], + evaluatorName: string, + evalId: string, +): Promise { + const parsedDefaults = defaultPreprocessors ?? []; + const parsedOverrides = await parsePreprocessors(rawValue, searchRoots, evaluatorName, evalId); + + if (parsedDefaults.length === 0 && (!parsedOverrides || parsedOverrides.length === 0)) { + return undefined; + } + + const merged = new Map(); + for (const entry of parsedDefaults) { + merged.set(normalizePreprocessorType(entry.type), entry); + } + for (const entry of parsedOverrides ?? []) { + merged.set(normalizePreprocessorType(entry.type), entry); + } + + return [...merged.values()]; +} + +export async function parsePreprocessors( + rawValue: JsonValue | undefined, + searchRoots: readonly string[], + evaluatorName: string, + evalId: string, +): Promise { + if (rawValue === undefined) { + return undefined; + } + if (!Array.isArray(rawValue)) { + throw new Error(`Evaluator '${evaluatorName}' in '${evalId}': preprocessors must be an array`); + } + + const preprocessors: ContentPreprocessorConfig[] = []; + for (const rawEntry of rawValue) { + if (!isJsonObject(rawEntry)) { + throw new Error( + `Evaluator '${evaluatorName}' in '${evalId}': each preprocessor must be an object`, + ); + } + + const type = asString(rawEntry.type)?.trim(); + if (!type) { + throw new Error(`Evaluator '${evaluatorName}' in '${evalId}': preprocessor.type is required`); + } + + const command = asStringArray( + rawEntry.command, + `preprocessor command for evaluator '${evaluatorName}' in '${evalId}'`, + ); + if (!command || command.length === 0) { + throw new Error( + `Evaluator '${evaluatorName}' in '${evalId}': preprocessor '${type}' requires command`, + ); + } + + const commandPath = command[command.length - 1]; + const resolved = await resolveFileReference(commandPath, searchRoots); + if (!resolved.resolvedPath) { + throw new Error( + `Evaluator '${evaluatorName}' in '${evalId}': preprocessor command file not found: ${resolved.displayPath}`, + ); + } + + preprocessors.push({ + type, + command, + resolvedCommand: [...command.slice(0, -1), path.resolve(resolved.resolvedPath)], + }); + } + + return preprocessors; +} + /** Assertion evaluator types that support auto-generated names. */ const ASSERTION_TYPES = new Set([ 'skill-trigger', diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index a10a73025..416fa1ba2 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -58,6 +58,7 @@ import type { FailureStage, JsonObject, JsonValue, + LlmGraderEvaluatorConfig, TrialResult, TrialsConfig, WorkspaceHookConfig, @@ -2287,6 +2288,10 @@ async function runEvaluatorsForCase(options: { if (!activeEvaluator) { throw new Error(`No evaluator registered for kind '${evaluatorKind}'`); } + const implicitEvaluator = + evaluatorKind === 'llm-grader' && !evalCase.assertions + ? buildImplicitLlmGraderConfig(evalCase) + : undefined; const score = await activeEvaluator.evaluate({ evalCase, @@ -2308,11 +2313,24 @@ async function runEvaluatorsForCase(options: { availableTargets, fileChanges, workspacePath, + ...(implicitEvaluator ? { evaluator: implicitEvaluator } : {}), }); return { score }; } +function buildImplicitLlmGraderConfig(evalCase: EvalTest): LlmGraderEvaluatorConfig | undefined { + if (!evalCase.preprocessors || evalCase.preprocessors.length === 0) { + return undefined; + } + + return { + name: 'llm-grader', + type: 'llm-grader', + preprocessors: evalCase.preprocessors, + }; +} + async function runEvaluatorList(options: { readonly evalCase: EvalTest; readonly evaluators: readonly EvaluatorConfig[]; diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index fb4a83ba7..ae21d986b 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -311,6 +311,8 @@ export type CodeEvaluatorConfig = { readonly config?: JsonObject; /** When present, enables target access via local proxy */ readonly target?: TargetAccessConfig; + /** Optional content preprocessors inherited from suite/evaluator config */ + readonly preprocessors?: readonly ContentPreprocessorConfig[]; }; /** @@ -326,6 +328,15 @@ export type PromptScriptConfig = { readonly config?: Record; }; +export type ContentPreprocessorConfig = { + /** MIME type or short alias such as "xlsx" or "html" */ + readonly type: string; + /** Command array to execute (stdin JSON payload -> stdout text) */ + readonly command: readonly string[]; + /** Resolved absolute path for the command script (last argv element) */ + readonly resolvedCommand?: readonly string[]; +}; + export type LlmGraderEvaluatorConfig = { readonly name: string; readonly type: 'llm-grader'; @@ -351,6 +362,8 @@ export type LlmGraderEvaluatorConfig = { readonly max_steps?: number; /** Temperature override for grader calls */ readonly temperature?: number; + /** Optional content preprocessors for ContentFile blocks in assistant output */ + readonly preprocessors?: readonly ContentPreprocessorConfig[]; }; /** @deprecated Use `LlmGraderEvaluatorConfig` instead */ @@ -830,6 +843,8 @@ export interface EvalTest { readonly criteria: string; readonly evaluator?: EvaluatorKind; readonly assertions?: readonly EvaluatorConfig[]; + /** Suite-level preprocessors used by the implicit default llm-grader. */ + readonly preprocessors?: readonly ContentPreprocessorConfig[]; /** Workspace configuration (merged from suite-level and case-level) */ readonly workspace?: WorkspaceConfig; /** Arbitrary metadata passed to workspace scripts via stdin */ diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 0e1fef17d..e0af2feee 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -55,6 +55,11 @@ const PromptSchema = z.union([ }), ]); +const PreprocessorSchema = z.object({ + type: z.string().min(1), + command: z.union([z.string(), z.array(z.string())]), +}); + /** Score range for analytic rubrics */ const ScoreRangeSchema = z.object({ score_range: z.tuple([z.number().int().min(0).max(10), z.number().int().min(0).max(10)]), @@ -81,6 +86,7 @@ const CodeGraderSchema = EvaluatorCommonSchema.extend({ cwd: z.string().optional(), target: z.union([z.boolean(), z.object({ max_calls: z.number().optional() })]).optional(), config: z.record(z.unknown()).optional(), + preprocessors: z.array(PreprocessorSchema).optional(), }); const LlmGraderSchema = EvaluatorCommonSchema.extend({ @@ -92,6 +98,7 @@ const LlmGraderSchema = EvaluatorCommonSchema.extend({ config: z.record(z.unknown()).optional(), max_steps: z.number().int().min(1).max(50).optional(), temperature: z.number().min(0).max(2).optional(), + preprocessors: z.array(PreprocessorSchema).optional(), }); /** Aggregator configs for composite evaluator */ @@ -383,6 +390,8 @@ export const EvalFileSchema = z.object({ execution: ExecutionSchema.optional(), // Suite-level assertions assertions: z.array(EvaluatorSchema).optional(), + // Suite-level content preprocessors shared by evaluators + preprocessors: z.array(PreprocessorSchema).optional(), // Workspace (inline object or path to external workspace YAML file) workspace: z.union([WorkspaceSchema, z.string()]).optional(), }); diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index dc552c2b5..72ef09b2e 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -23,6 +23,7 @@ import { coerceEvaluator, parseEvaluators, parseInlineRubrics, + parsePreprocessors, warnUnconsumedCriteria, } from './loaders/evaluator-parser.js'; import { buildSearchRoots, resolveToAbsolutePath } from './loaders/file-resolver.js'; @@ -95,6 +96,7 @@ type RawTestSuite = JsonObject & { readonly execution?: JsonValue; readonly workspace?: JsonValue; readonly assertions?: JsonValue; + readonly preprocessors?: JsonValue; /** @deprecated Use `assertions` instead */ readonly assert?: JsonValue; readonly input?: JsonValue; @@ -283,6 +285,12 @@ async function loadTestsFromYaml( const rawTestCases = resolveTests(suite); const globalEvaluator = coerceEvaluator(suite.evaluator, 'global') ?? 'llm-grader'; + const suitePreprocessors = await parsePreprocessors( + suite.preprocessors, + searchRoots, + '', + absoluteTestPath, + ); // Parse suite-level workspace config (default for all cases) const evalFileDir = path.dirname(absoluteTestPath); @@ -456,6 +464,7 @@ async function loadTestsFromYaml( globalExecution, searchRoots, id ?? 'unknown', + suitePreprocessors, ); } catch (error) { // Skip entire test if evaluator validation fails @@ -503,6 +512,7 @@ async function loadTestsFromYaml( criteria: outcome ?? '', evaluator: testCaseEvaluatorKind, assertions: evaluators, + ...(suitePreprocessors ? { preprocessors: suitePreprocessors } : {}), workspace: mergedWorkspace, metadata, targets: caseTargets, diff --git a/packages/core/test/evaluation/content-preprocessor.test.ts b/packages/core/test/evaluation/content-preprocessor.test.ts new file mode 100644 index 000000000..22c562feb --- /dev/null +++ b/packages/core/test/evaluation/content-preprocessor.test.ts @@ -0,0 +1,112 @@ +import { afterEach, describe, expect, it } from 'bun:test'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { + appendPreprocessingWarnings, + extractTextWithPreprocessors, + normalizePreprocessorType, +} from '../../src/evaluation/content-preprocessor.js'; + +describe('content preprocessors', () => { + const tempDirs: string[] = []; + + afterEach(async () => { + await Promise.all(tempDirs.map((dir) => rm(dir, { recursive: true, force: true }))); + tempDirs.length = 0; + }); + + it('reads text files as UTF-8 by default', async () => { + const dir = await mkdtemp(join(tmpdir(), 'agentv-preprocessor-')); + tempDirs.push(dir); + const filePath = join(dir, 'report.txt'); + await writeFile(filePath, 'alpha\nbeta\n', 'utf8'); + + const result = await extractTextWithPreprocessors( + [{ type: 'file', media_type: 'text/plain', path: filePath }], + undefined, + ); + + expect(result.warnings).toEqual([]); + expect(result.text).toContain('[[ file:'); + expect(result.text).toContain('alpha\nbeta'); + }); + + it('uses configured preprocessors for matching file types', async () => { + const dir = await mkdtemp(join(tmpdir(), 'agentv-preprocessor-')); + tempDirs.push(dir); + const filePath = join(dir, 'report.xlsx'); + const scriptPath = join(dir, 'xlsx-to-text.js'); + await writeFile(filePath, 'unused', 'utf8'); + await writeFile( + scriptPath, + `const fs = require('node:fs'); +const payload = JSON.parse(fs.readFileSync(0, 'utf8')); +console.log('sheet:' + payload.original_path.split('/').pop());`, + 'utf8', + ); + + const result = await extractTextWithPreprocessors( + [ + { + type: 'file', + media_type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + path: filePath, + }, + ], + [{ type: 'xlsx', command: [process.execPath, scriptPath] }], + ); + + expect(result.warnings).toEqual([]); + expect(result.text).toContain('sheet:report.xlsx'); + }); + + it('resolves relative file paths against the provided base path', async () => { + const dir = await mkdtemp(join(tmpdir(), 'agentv-preprocessor-')); + tempDirs.push(dir); + const nestedDir = join(dir, 'workspace'); + await mkdir(nestedDir, { recursive: true }); + await writeFile(join(nestedDir, 'report.txt'), 'from workspace', 'utf8'); + + const result = await extractTextWithPreprocessors( + [{ type: 'file', media_type: 'text/plain', path: 'report.txt' }], + undefined, + { basePath: nestedDir }, + ); + + expect(result.warnings).toEqual([]); + expect(result.text).toContain('from workspace'); + }); + + it('records a warning when default UTF-8 extraction looks binary', async () => { + const dir = await mkdtemp(join(tmpdir(), 'agentv-preprocessor-')); + tempDirs.push(dir); + const filePath = join(dir, 'report.pdf'); + await writeFile(filePath, Buffer.from([0, 159, 146, 150])); + + const result = await extractTextWithPreprocessors( + [{ type: 'file', media_type: 'application/pdf', path: filePath }], + undefined, + ); + + expect(result.text).toBe(''); + expect(result.warnings).toHaveLength(1); + expect(result.warnings[0]?.reason).toContain('configure a preprocessor'); + }); + + it('appends warnings to extracted text for grader prompts', () => { + const text = appendPreprocessingWarnings('body', [ + { file: '/tmp/report.pdf', mediaType: 'application/pdf', reason: 'failed to extract' }, + ]); + expect(text).toContain('body'); + expect(text).toContain('[file preprocessing warning]'); + }); + + it('normalizes short aliases to MIME types', () => { + expect(normalizePreprocessorType('xlsx')).toBe( + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ); + expect(normalizePreprocessorType('text/html')).toBe('text/html'); + }); +}); diff --git a/packages/core/test/evaluation/llm-grader-multimodal.test.ts b/packages/core/test/evaluation/llm-grader-multimodal.test.ts index 1ff035a02..bbf6e8c04 100644 --- a/packages/core/test/evaluation/llm-grader-multimodal.test.ts +++ b/packages/core/test/evaluation/llm-grader-multimodal.test.ts @@ -9,7 +9,10 @@ * - Images in non-assistant messages are ignored */ -import { beforeEach, describe, expect, it, mock } from 'bun:test'; +import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import type { ResolvedTarget } from '../../src/evaluation/providers/targets.js'; import type { Message } from '../../src/evaluation/providers/types.js'; @@ -194,10 +197,19 @@ describe('extractImageBlocks', () => { // --------------------------------------------------------------------------- describe('LlmGraderEvaluator multimodal', () => { + let tempDir: string | undefined; + beforeEach(() => { capturedGenerateTextArgs = undefined; }); + afterEach(async () => { + if (tempDir) { + await rm(tempDir, { recursive: true, force: true }); + tempDir = undefined; + } + }); + it('sends plain text prompt when output has no images', async () => { const provider = createLmProvider(); @@ -353,4 +365,53 @@ describe('LlmGraderEvaluator multimodal', () => { expect(capturedGenerateTextArgs?.prompt).toBeTypeOf('string'); expect(capturedGenerateTextArgs?.messages).toBeUndefined(); }); + + it('injects preprocessed file text into the plain prompt', async () => { + tempDir = await mkdtemp(join(tmpdir(), 'agentv-llm-file-')); + const filePath = join(tempDir, 'report.xlsx'); + const scriptPath = join(tempDir, 'xlsx-to-text.js'); + await writeFile(filePath, 'unused', 'utf8'); + await writeFile( + scriptPath, + `const fs = require('node:fs'); +const payload = JSON.parse(fs.readFileSync(0, 'utf8')); +console.log('spreadsheet:' + payload.original_path.split('/').pop());`, + 'utf8', + ); + + const provider = createLmProvider(); + const evaluator = new LlmGraderEvaluator({ + resolveGraderProvider: async () => provider, + }); + + await evaluator.evaluate({ + evalCase: baseTestCase, + candidate: '', + target: baseTarget, + provider, + attempt: 0, + promptInputs: { question: 'Describe the image' }, + now: new Date(), + evaluator: { + name: 'grade', + type: 'llm-grader', + preprocessors: [{ type: 'xlsx', command: [process.execPath, scriptPath] }], + }, + output: [ + { + role: 'assistant', + content: [ + { + type: 'file', + media_type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + path: filePath, + }, + ], + }, + ], + }); + + expect(capturedGenerateTextArgs?.prompt).toBeTypeOf('string'); + expect(String(capturedGenerateTextArgs?.prompt)).toContain('spreadsheet:report.xlsx'); + }); }); diff --git a/packages/core/test/evaluation/orchestrator.test.ts b/packages/core/test/evaluation/orchestrator.test.ts index 183ad2258..14ff78e5b 100644 --- a/packages/core/test/evaluation/orchestrator.test.ts +++ b/packages/core/test/evaluation/orchestrator.test.ts @@ -179,6 +179,78 @@ describe('runTestCase', () => { expect(result.failureReasonCode).toBeUndefined(); }); + it('applies suite-level preprocessors to the implicit default llm-grader', async () => { + const tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-orchestrator-preprocessor-')); + const reportPath = path.join(tempDir, 'report.xlsx'); + const scriptPath = path.join(tempDir, 'xlsx-to-text.js'); + writeFileSync(reportPath, Buffer.from([0, 159, 146, 150])); + writeFileSync( + scriptPath, + `const fs = require('node:fs'); +const payload = JSON.parse(fs.readFileSync(0, 'utf8')); +if (!payload.path) throw new Error('missing path'); +console.log('spreadsheet: revenue,total\\nQ1,42');`, + 'utf8', + ); + + const answerProvider = new SequenceProvider('file-output', { + responses: [ + { + output: [ + { + role: 'assistant', + content: [ + { + type: 'file', + media_type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + path: reportPath, + }, + ], + }, + ], + }, + ], + }); + const graderProvider = new CapturingGraderProvider('grader', { + output: [ + { + role: 'assistant', + content: JSON.stringify({ + score: 1, + assertions: [{ text: 'ok', passed: true }], + }), + }, + ], + }); + + const evalCase: EvalTest = { + ...baseTestCase, + id: 'implicit-preprocessors', + assertions: undefined, + preprocessors: [{ type: 'xlsx', command: [process.execPath, scriptPath] }], + }; + + const results = await runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: tempDir, + target: { ...baseTarget, name: 'file-output', graderTarget: 'grader' }, + targets: [ + { name: 'grader', provider: 'mock' }, + { name: 'file-output', provider: 'mock', grader_target: 'grader' }, + ], + providerFactory: (target) => { + if (target.name === 'grader') return graderProvider; + return answerProvider; + }, + evaluators: undefined, + evalCases: [evalCase], + }); + + expect(results[0]?.score).toBe(1); + expect(graderProvider.lastRequest?.question).toContain('spreadsheet: revenue,total'); + expect(graderProvider.lastRequest?.question).toContain('Q1,42'); + }); + it('reuses cached provider response when available', async () => { const provider = new SequenceProvider('mock', { responses: [ diff --git a/packages/core/test/evaluation/preprocessors-yaml.test.ts b/packages/core/test/evaluation/preprocessors-yaml.test.ts new file mode 100644 index 000000000..92e4be24e --- /dev/null +++ b/packages/core/test/evaluation/preprocessors-yaml.test.ts @@ -0,0 +1,93 @@ +import { afterEach, describe, expect, it } from 'bun:test'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { loadTests } from '../../src/evaluation/yaml-parser.js'; + +describe('eval YAML preprocessors', () => { + const tempDirs: string[] = []; + + afterEach(async () => { + await Promise.all(tempDirs.map((dir) => rm(dir, { recursive: true, force: true }))); + tempDirs.length = 0; + }); + + it('merges suite-level preprocessors into llm-graders and resolves command paths', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'agentv-yaml-preprocessors-')); + tempDirs.push(dir); + + await writeFile(path.join(dir, 'xlsx-default.js'), 'console.log("default")', 'utf8'); + await writeFile(path.join(dir, 'xlsx-override.js'), 'console.log("override")', 'utf8'); + await writeFile( + path.join(dir, 'suite.eval.yaml'), + `preprocessors: + - type: xlsx + command: ["node", "xlsx-default.js"] +tests: + - id: report + input: "grade this" + criteria: "works" + assertions: + - name: grade + type: llm-grader + prompt: "Evaluate {{ output }}" + preprocessors: + - type: xlsx + command: ["node", "xlsx-override.js"] +`, + 'utf8', + ); + + const tests = await loadTests(path.join(dir, 'suite.eval.yaml'), dir); + const evaluator = tests[0]?.assertions?.[0]; + + expect(evaluator?.type).toBe('llm-grader'); + if (!evaluator || evaluator.type !== 'llm-grader') { + throw new Error('expected llm-grader evaluator'); + } + + expect(evaluator.preprocessors).toHaveLength(1); + expect(evaluator.preprocessors?.[0]?.resolvedCommand?.[1]).toBe( + path.join(dir, 'xlsx-override.js'), + ); + }); + + it('lets alias-based evaluator overrides replace MIME-typed suite defaults', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'agentv-yaml-preprocessors-')); + tempDirs.push(dir); + + await writeFile(path.join(dir, 'xlsx-default.js'), 'console.log("default")', 'utf8'); + await writeFile(path.join(dir, 'xlsx-override.js'), 'console.log("override")', 'utf8'); + await writeFile( + path.join(dir, 'suite.eval.yaml'), + `preprocessors: + - type: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet + command: ["node", "xlsx-default.js"] +tests: + - id: report + input: "grade this" + criteria: "works" + assertions: + - name: grade + type: llm-grader + prompt: "Evaluate {{ output }}" + preprocessors: + - type: xlsx + command: ["node", "xlsx-override.js"] +`, + 'utf8', + ); + + const tests = await loadTests(path.join(dir, 'suite.eval.yaml'), dir); + const evaluator = tests[0]?.assertions?.[0]; + if (!evaluator || evaluator.type !== 'llm-grader') { + throw new Error('expected llm-grader evaluator'); + } + + expect(evaluator.preprocessors).toHaveLength(1); + expect(evaluator.preprocessors?.[0]?.resolvedCommand?.[1]).toBe( + path.join(dir, 'xlsx-override.js'), + ); + }); +}); diff --git a/plugins/agentv-dev/skills/agentv-eval-writer/references/eval-schema.json b/plugins/agentv-dev/skills/agentv-eval-writer/references/eval-schema.json index 7d2f54e93..bb340e350 100644 --- a/plugins/agentv-dev/skills/agentv-eval-writer/references/eval-schema.json +++ b/plugins/agentv-dev/skills/agentv-eval-writer/references/eval-schema.json @@ -294,6 +294,33 @@ "config": { "type": "object", "additionalProperties": {} + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type", "command"], @@ -455,6 +482,33 @@ "type": "number", "minimum": 0, "maximum": 2 + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type"], @@ -1368,6 +1422,33 @@ "config": { "type": "object", "additionalProperties": {} + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type", "command"], @@ -1529,6 +1610,33 @@ "type": "number", "minimum": 0, "maximum": 2 + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type"], @@ -2459,6 +2567,33 @@ "config": { "type": "object", "additionalProperties": {} + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type", "command"], @@ -2620,6 +2755,33 @@ "type": "number", "minimum": 0, "maximum": 2 + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type"], @@ -3533,6 +3695,33 @@ "config": { "type": "object", "additionalProperties": {} + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type", "command"], @@ -3694,6 +3883,33 @@ "type": "number", "minimum": 0, "maximum": 2 + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type"], @@ -5030,6 +5246,33 @@ "config": { "type": "object", "additionalProperties": {} + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type", "command"], @@ -5191,6 +5434,33 @@ "type": "number", "minimum": 0, "maximum": 2 + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type"], @@ -6104,6 +6374,33 @@ "config": { "type": "object", "additionalProperties": {} + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type", "command"], @@ -6265,6 +6562,33 @@ "type": "number", "minimum": 0, "maximum": 2 + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type"], @@ -7195,6 +7519,33 @@ "config": { "type": "object", "additionalProperties": {} + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type", "command"], @@ -7356,6 +7707,33 @@ "type": "number", "minimum": 0, "maximum": 2 + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type"], @@ -8269,6 +8647,33 @@ "config": { "type": "object", "additionalProperties": {} + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type", "command"], @@ -8430,6 +8835,33 @@ "type": "number", "minimum": 0, "maximum": 2 + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type"], @@ -9672,6 +10104,33 @@ "config": { "type": "object", "additionalProperties": {} + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type", "command"], @@ -9833,6 +10292,33 @@ "type": "number", "minimum": 0, "maximum": 2 + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type"], @@ -10746,6 +11232,33 @@ "config": { "type": "object", "additionalProperties": {} + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type", "command"], @@ -10907,6 +11420,33 @@ "type": "number", "minimum": 0, "maximum": 2 + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type"], @@ -11871,6 +12411,33 @@ "config": { "type": "object", "additionalProperties": {} + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type", "command"], @@ -12032,6 +12599,33 @@ "type": "number", "minimum": 0, "maximum": 2 + }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } } }, "required": ["type"], @@ -12857,6 +13451,33 @@ ] } }, + "preprocessors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + }, + "required": ["type", "command"], + "additionalProperties": false + } + }, "workspace": { "anyOf": [ {