diff --git a/package-lock.json b/package-lock.json index 1615779..85f49da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,12 +10,12 @@ "dependencies": { "@actions/core": "^1.11.1", "@actions/github": "^6.0.1", - "@ai-sdk/anthropic": "^3.0.71", - "@ai-sdk/google": "^3.0.64", - "@ai-sdk/mistral": "^3.0.30", - "@ai-sdk/openai": "^3.0.53", - "@ai-sdk/openai-compatible": "^2.0.41", - "ai": "^6.0.168", + "@ai-sdk/anthropic": "^3.0.79", + "@ai-sdk/google": "^3.0.79", + "@ai-sdk/mistral": "^3.0.37", + "@ai-sdk/openai": "^3.0.65", + "@ai-sdk/openai-compatible": "^2.0.48", + "ai": "^6.0.191", "minimatch": "^10.1.1", "zod": "^3.25.76" }, @@ -74,13 +74,13 @@ "license": "MIT" }, "node_modules/@ai-sdk/anthropic": { - "version": "3.0.71", - "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-3.0.71.tgz", - "integrity": "sha512-bUWOzrzR0gJKJO/PLGMR4uH2dqEgqGhrsCV+sSpk4KtOEnUQlfjZI/F7BFlqSvVpFbjdgYRRLysAeEZpJ6S1lg==", + "version": "3.0.79", + "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-3.0.79.tgz", + "integrity": "sha512-saEX+h5JDOkT9P/+REKDyikbnJiToFuLipgNcsmu4Zr3GW5kW1m9HhvrPK+vj63itIOsoZU6tmVIjkrePOlIUA==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "3.0.8", - "@ai-sdk/provider-utils": "4.0.23" + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" }, "engines": { "node": ">=18" @@ -90,13 +90,13 @@ } }, "node_modules/@ai-sdk/gateway": { - "version": "3.0.104", - "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.104.tgz", - "integrity": "sha512-ZKX5n74io8VIRlhIMSLWVlvT3sXC8Z7cZ9GHuWBWZDVi96+62AIsWuLGvMfcBA1STYuSoDrp6rIziZmvrTq0TA==", + "version": "3.0.120", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.120.tgz", + "integrity": "sha512-MYKAeD2q7/sa1ZdqtL2tw0Me0B8Tok6Q/fhkJDhJl39dG8u+VBlWO9yk9lcdm784bM418o1EKObo4aOxs6+18Q==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "3.0.8", - "@ai-sdk/provider-utils": "4.0.23", + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27", "@vercel/oidc": "3.2.0" }, "engines": { @@ -107,13 +107,13 @@ } }, "node_modules/@ai-sdk/google": { - "version": "3.0.64", - "resolved": "https://registry.npmjs.org/@ai-sdk/google/-/google-3.0.64.tgz", - "integrity": "sha512-CbR82EgGPNrj/6q0HtclwuCqe0/pDShyv3nWDP/A9DroujzWXnLMlUJVrgPOsg4b40zQCwwVs2XSKCxvt/4QaA==", + "version": "3.0.79", + "resolved": "https://registry.npmjs.org/@ai-sdk/google/-/google-3.0.79.tgz", + "integrity": "sha512-QWVAvYeA7JzEX2wkSyXOWv/I9PD9kvTzdykkSTLi+Eu8RyJ6gA0tdPIGa8esEtOcHE//G5vy6FTB70qQw8l/uw==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "3.0.8", - "@ai-sdk/provider-utils": "4.0.23" + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" }, "engines": { "node": ">=18" @@ -123,13 +123,13 @@ } }, "node_modules/@ai-sdk/mistral": { - "version": "3.0.30", - "resolved": "https://registry.npmjs.org/@ai-sdk/mistral/-/mistral-3.0.30.tgz", - "integrity": "sha512-+j4IXRSk9E661cFSafmIr+XHOzwjFagawwzMOlSqwL6U4Sq4PCFLDF+oHbX5NUqNjUL7FD1zi/9lBIfa41pUvw==", + "version": "3.0.37", + "resolved": "https://registry.npmjs.org/@ai-sdk/mistral/-/mistral-3.0.37.tgz", + "integrity": "sha512-KkdaMjs4C2y+vrZWJE990E3ZxBFiOTHQ94ZlquuIttpphcqJTMxNoIpnKT/4UzMVWXL0BUEE2vs+1UEVXkN8Kg==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "3.0.8", - "@ai-sdk/provider-utils": "4.0.23" + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" }, "engines": { "node": ">=18" @@ -139,13 +139,13 @@ } }, "node_modules/@ai-sdk/openai": { - "version": "3.0.53", - "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-3.0.53.tgz", - "integrity": "sha512-Wld+Rbc05KaUn08uBt06eEuwcgalcIFtIl32Yp+GxuZXUQwOb6YeAuq+C6da4ch6BurFoqEaLemJVwjBb7x+PQ==", + "version": "3.0.65", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-3.0.65.tgz", + "integrity": "sha512-ZlVoWH+zrdiYDiUt6n/xvfCsk33mzsB81TUQkBRVx79rxU1FKZqVH9J/QCtEpSLqx0cUzjvtIw9l9p7EbUv+dw==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "3.0.8", - "@ai-sdk/provider-utils": "4.0.23" + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" }, "engines": { "node": ">=18" @@ -155,13 +155,13 @@ } }, "node_modules/@ai-sdk/openai-compatible": { - "version": "2.0.41", - "resolved": "https://registry.npmjs.org/@ai-sdk/openai-compatible/-/openai-compatible-2.0.41.tgz", - "integrity": "sha512-kNAGINk71AlOXx10Dq/PXw4t/9XjdK8uxfpVElRwtSFMdeSiLVt58p9TPx4/FJD+hxZuVhvxYj9r42osxWq79g==", + "version": "2.0.48", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai-compatible/-/openai-compatible-2.0.48.tgz", + "integrity": "sha512-z9MC6M4Oh/yUY/F/eszOtO8wc2nMz99XmZQKd2gWTtyIfe716xTfrKe3aYZKg20NZDtyjqPPKPSR+wqz7q1T7Q==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "3.0.8", - "@ai-sdk/provider-utils": "4.0.23" + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27" }, "engines": { "node": ">=18" @@ -171,9 +171,9 @@ } }, "node_modules/@ai-sdk/provider": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.8.tgz", - "integrity": "sha512-oGMAgGoQdBXbZqNG0Ze56CHjDZ1IDYOwGYxYjO5KLSlz5HiNQ9udIXsPZ61VWaHGZ5XW/jyjmr6t2xz2jGVwbQ==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.10.tgz", + "integrity": "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw==", "license": "Apache-2.0", "dependencies": { "json-schema": "^0.4.0" @@ -183,14 +183,14 @@ } }, "node_modules/@ai-sdk/provider-utils": { - "version": "4.0.23", - "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.23.tgz", - "integrity": "sha512-z8GlDaCmRSDlqkMF2f4/RFgWxdarvIbyuk+m6WXT1LYgsnGiXRJGTD2Z1+SDl3LqtFuRtGX1aghYvQLoHL/9pg==", + "version": "4.0.27", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.27.tgz", + "integrity": "sha512-ubkAJ+xODouwtmN1tYlvTPphH1hPOBfZaEQe8U7skGvFAnIRs9PPpsq57bC2+Ky/MB4yzhd6YOsxTAx9sGpazw==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/provider": "3.0.8", + "@ai-sdk/provider": "3.0.10", "@standard-schema/spec": "^1.1.0", - "eventsource-parser": "^3.0.6" + "eventsource-parser": "^3.0.8" }, "engines": { "node": ">=18" @@ -391,15 +391,15 @@ } }, "node_modules/ai": { - "version": "6.0.168", - "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.168.tgz", - "integrity": "sha512-2HqCJuO+1V2aV7vfYs5LFEUfxbkGX+5oa54q/gCCTL7KLTdbxcCu5D7TdLA5kwsrs3Szgjah9q6D9tpjHM3hUQ==", + "version": "6.0.191", + "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.191.tgz", + "integrity": "sha512-zAxvjKebQE7YkSyyNIl0OM7i6/zygnKeF+yNUjD4nWOelYrG+LpDd6RnH6mjySI4zUpZ7o4wbnmAy8jc6u98vQ==", "license": "Apache-2.0", "dependencies": { - "@ai-sdk/gateway": "3.0.104", - "@ai-sdk/provider": "3.0.8", - "@ai-sdk/provider-utils": "4.0.23", - "@opentelemetry/api": "1.9.0" + "@ai-sdk/gateway": "3.0.120", + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27", + "@opentelemetry/api": "^1.9.0" }, "engines": { "node": ">=18" @@ -442,9 +442,9 @@ "license": "ISC" }, "node_modules/eventsource-parser": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.7.tgz", - "integrity": "sha512-zwxwiQqexizSXFZV13zMiEtW1E3lv7RlUv+1f5FBiR4x7wFhEjm3aFTyYkZQWzyN08WnPdox015GoRH5D/E5YA==", + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.8.tgz", + "integrity": "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ==", "license": "MIT", "engines": { "node": ">=18.0.0" diff --git a/package.json b/package.json index 467eaa4..797b6e6 100644 --- a/package.json +++ b/package.json @@ -16,12 +16,12 @@ "dependencies": { "@actions/core": "^1.11.1", "@actions/github": "^6.0.1", - "@ai-sdk/anthropic": "^3.0.71", - "@ai-sdk/google": "^3.0.64", - "@ai-sdk/mistral": "^3.0.30", - "@ai-sdk/openai": "^3.0.53", - "@ai-sdk/openai-compatible": "^2.0.41", - "ai": "^6.0.168", + "@ai-sdk/anthropic": "^3.0.79", + "@ai-sdk/google": "^3.0.79", + "@ai-sdk/mistral": "^3.0.37", + "@ai-sdk/openai": "^3.0.65", + "@ai-sdk/openai-compatible": "^2.0.48", + "ai": "^6.0.191", "minimatch": "^10.1.1", "zod": "^3.25.76" } diff --git a/src/agents.js b/src/agents.js index fe4fba1..c73090a 100644 --- a/src/agents.js +++ b/src/agents.js @@ -1,7 +1,152 @@ const { z } = require('zod'); const { configureRuntime, runStructuredWithRepair } = require('./model-runtime'); -const plannerOutputSchema = z.object({ +const SEVERITIES = ['critical', 'high', 'medium', 'low']; +const SIDES = ['LEFT', 'RIGHT', 'FILE']; + +function coerceString(value) { + if (value === undefined || value === null) { + return undefined; + } + return String(value).trim(); +} + +function coerceStringArray(value, maxItems = Infinity) { + const source = Array.isArray(value) + ? value.flat(Infinity) + : (value === undefined || value === null ? [] : [value]); + const seen = new Set(); + const out = []; + + for (const item of source) { + if (item === undefined || item === null) { + continue; + } + const text = (typeof item === 'object' ? JSON.stringify(item) : String(item)).trim(); + if (!text || seen.has(text)) { + continue; + } + seen.add(text); + out.push(text); + if (out.length >= maxItems) { + break; + } + } + + return out; +} + +function coerceBoolean(value) { + if (value === undefined || value === null) { + return undefined; + } + if (typeof value === 'boolean') { + return value; + } + if (typeof value === 'number') { + return value !== 0; + } + + const text = String(value).trim().toLowerCase(); + if (['true', '1', 'yes', 'y', 'done'].includes(text)) { + return true; + } + if (['false', '0', 'no', 'n', 'pending'].includes(text)) { + return false; + } + return false; +} + +function coerceSeverity(value) { + const text = String(coerceString(value) || '').toLowerCase().replace(/[\s_-]+/g, ''); + if (SEVERITIES.includes(text)) { + return text; + } + if (text.startsWith('crit') || text === 'blocker' || text === 'blocking') { + return 'critical'; + } + if (text === 'hi' || text === 'major') { + return 'high'; + } + if (text.startsWith('med') || text === 'warn' || text === 'warning') { + return 'medium'; + } + if (text.startsWith('lo') || text === 'minor' || text === 'info' || text === 'informational' || text === 'hint') { + return 'low'; + } + return 'medium'; +} + +function coerceSide(value) { + const text = String(coerceString(value) || '').toUpperCase(); + if (SIDES.includes(text)) { + return text; + } + if (['L', 'OLD', 'REMOVED', 'DELETED', 'DELETION'].includes(text)) { + return 'LEFT'; + } + if (['R', 'NEW', 'ADDED', 'ADDITION'].includes(text)) { + return 'RIGHT'; + } + if (['NONE', 'GENERAL', 'OVERALL'].includes(text)) { + return 'FILE'; + } + return 'RIGHT'; +} + +function coercePositiveIntegerOrNull(value) { + if (value === undefined || value === null || value === '') { + return null; + } + + const text = String(value).trim(); + const normalized = text.replace(/^[LR#:\s]+/i, ''); + const numberValue = typeof value === 'number' ? value : Number(normalized); + return Number.isInteger(numberValue) && numberValue > 0 ? numberValue : null; +} + +function coerceConfidence(value) { + if (value === undefined || value === null || value === '') { + return null; + } + + const text = String(value).trim(); + const numberValue = Number.parseFloat(text.replace('%', '')); + if (!Number.isFinite(numberValue)) { + return null; + } + + const normalized = text.includes('%') ? numberValue / 100 : numberValue; + return Math.min(1, Math.max(0, normalized)); +} + +function coerceObjectArray(value, itemSchema) { + const source = Array.isArray(value) + ? value + : (value === undefined || value === null ? [] : [value]); + const out = []; + + for (const item of source) { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + continue; + } + const parsed = itemSchema.safeParse(item); + if (parsed.success) { + out.push(parsed.data); + } + } + + return out; +} + +const requiredStringSchema = z.preprocess(coerceString, z.string().min(1)); +const optionalStringSchema = (defaultValue = '') => z.preprocess(coerceString, z.string().default(defaultValue)); +const stringArraySchema = (maxItems = Infinity) => z.preprocess( + (value) => coerceStringArray(value, maxItems), + z.array(z.string()).default([]) +); + +const plannerGenerationSchema = z.object({ batches: z .array( z.object({ @@ -15,12 +160,27 @@ const plannerOutputSchema = z.object({ notes: z.string().default('') }); -const findingSchema = z.object({ +const plannerBatchSchema = z.object({ + focus: optionalStringSchema('general'), + filePaths: stringArraySchema(), + reason: optionalStringSchema('') +}); + +const plannerOutputSchema = z.object({ + batches: z.preprocess( + (value) => coerceObjectArray(value, plannerBatchSchema), + z.array(plannerBatchSchema).default([]) + ), + done: z.preprocess(coerceBoolean, z.boolean().default(false)), + notes: optionalStringSchema('') +}); + +const findingGenerationSchema = z.object({ title: z.string().min(1), - severity: z.enum(['critical', 'high', 'medium', 'low']), + severity: z.enum(SEVERITIES), category: z.string().default('general'), path: z.string().min(1), - side: z.enum(['LEFT', 'RIGHT', 'FILE']).default('RIGHT'), + side: z.enum(SIDES).default('RIGHT'), line: z.number().int().positive().nullable().default(null), confidence: z.number().min(0).max(1).nullable().optional().default(null), evidence: z.array(z.string().min(1)).default([]), @@ -30,7 +190,25 @@ const findingSchema = z.object({ risk: z.string().default('') }); -const fileConclusionSchema = z.object({ +const findingSchema = z.object({ + title: requiredStringSchema, + severity: z.preprocess(coerceSeverity, z.enum(SEVERITIES)), + category: optionalStringSchema('general'), + path: requiredStringSchema, + side: z.preprocess(coerceSide, z.enum(SIDES).default('RIGHT')), + line: z.preprocess(coercePositiveIntegerOrNull, z.number().int().positive().nullable().default(null)), + confidence: z.preprocess(coerceConfidence, z.number().min(0).max(1).nullable().default(null)), + evidence: stringArraySchema(), + fingerprint: z.preprocess( + (value) => String(coerceString(value) || '').slice(0, 120), + z.string().max(120).default('') + ), + summary: requiredStringSchema, + suggestion: optionalStringSchema(''), + risk: optionalStringSchema('') +}); + +const fileConclusionGenerationSchema = z.object({ path: z.string().min(1), conclusion: z.string().min(1), risks: z.array(z.string()).default([]), @@ -38,10 +216,18 @@ const fileConclusionSchema = z.object({ note: z.string().default('') }); -const reviewOutputSchema = z.object({ +const fileConclusionSchema = z.object({ + path: requiredStringSchema, + conclusion: requiredStringSchema, + risks: stringArraySchema(), + testSuggestions: stringArraySchema(), + note: optionalStringSchema('') +}); + +const reviewGenerationSchema = z.object({ overall: z.string().min(1), - findings: z.array(findingSchema).default([]), - fileConclusions: z.array(fileConclusionSchema).default([]), + findings: z.array(findingGenerationSchema).default([]), + fileConclusions: z.array(fileConclusionGenerationSchema).default([]), recommendedExtraDimensions: z.array(z.string()).default([]), recommendationReason: z.string().default(''), actionableSuggestions: z.array(z.string()).default([]), @@ -49,6 +235,23 @@ const reviewOutputSchema = z.object({ testSuggestions: z.array(z.string()).default([]) }); +const reviewOutputSchema = z.object({ + overall: requiredStringSchema, + findings: z.preprocess( + (value) => coerceObjectArray(value, findingSchema), + z.array(findingSchema).default([]) + ), + fileConclusions: z.preprocess( + (value) => coerceObjectArray(value, fileConclusionSchema), + z.array(fileConclusionSchema).default([]) + ), + recommendedExtraDimensions: stringArraySchema(), + recommendationReason: optionalStringSchema(''), + actionableSuggestions: stringArraySchema(), + potentialRisks: stringArraySchema(), + testSuggestions: stringArraySchema() +}); + function buildProjectGuidanceInstructions(projectGuidance) { if (!projectGuidance || !projectGuidance.content) { return ''; @@ -118,7 +321,8 @@ Output must follow the required JSON contract exactly.`; name: 'Review Planner', model, instructions, - schema: plannerOutputSchema, + schema: plannerGenerationSchema, + parseSchema: plannerOutputSchema, responseName: 'planner_output', outputContractPrompt: buildPlannerOutputContractPrompt(), opts: { @@ -164,7 +368,8 @@ Output must follow the required JSON contract exactly.`; model, modelInstance: modelInstance || null, instructions, - schema: reviewOutputSchema, + schema: reviewGenerationSchema, + parseSchema: reviewOutputSchema, responseName: `${dimension}_review_output`, outputContractPrompt: buildReviewerOutputContractPrompt(), opts: { @@ -333,5 +538,15 @@ module.exports = { buildPlannerInput, buildBatchReviewInput, plannerOutputSchema, - reviewOutputSchema + reviewOutputSchema, + __private: { + plannerGenerationSchema, + reviewGenerationSchema, + coerceBoolean, + coerceSeverity, + coerceSide, + coercePositiveIntegerOrNull, + coerceConfidence, + coerceStringArray + } }; diff --git a/src/model-runtime.js b/src/model-runtime.js index 92da5ac..af741f7 100644 --- a/src/model-runtime.js +++ b/src/model-runtime.js @@ -87,22 +87,70 @@ async function requestStructuredOutput({ agent, input, repairContext }) { const userPrompt = buildUserInput(agent, input, repairContext); const model = agent.modelInstance || runtimeState.model; + try { + const result = await generateText({ + model, + system: agent.instructions, + prompt: userPrompt, + output: Output.object({ schema: agent.schema }) + }); + + return parseGenerateTextResult(agent, result); + } catch (error) { + if (!shouldFallbackToTextJson(error)) { + throw error; + } + + try { + return await requestTextJsonOutput({ agent, userPrompt, model }); + } catch (fallbackError) { + const wrapped = new Error( + `structured_text_fallback_failed: ${compactErrorMessage(fallbackError)}; original_error: ${compactErrorMessage(error)}` + ); + wrapped.code = 'structured_text_fallback_failed'; + wrapped.preview = fallbackError.preview; + throw wrapped; + } + } +} + +async function requestTextJsonOutput({ agent, userPrompt, model }) { const result = await generateText({ model, system: agent.instructions, - prompt: userPrompt, - output: Output.object({ schema: agent.schema }) + prompt: userPrompt }); - // AI SDK sets output to the parsed object when successful + return parseGenerateTextResult(agent, result); +} + +function parseGenerateTextResult(agent, result) { + // AI SDK sets output to the parsed object when successful. if (result.output !== undefined && result.output !== null) { - return result.output; + return parseStructuredObject(agent, result.output); + } + + return parseStructuredText(agent, result.text || ''); +} + +function parseStructuredObject(agent, outputObject) { + const schema = agent.parseSchema || agent.schema; + const parsed = schema.safeParse(outputObject); + if (!parsed.success) { + const error = new Error(`schema_validation_failed: ${formatIssues(parsed.error.issues)}`); + error.code = 'schema_validation_failed'; + error.preview = JSON.stringify(outputObject).slice(0, 400); + throw error; } + return parsed.data; +} + +function parseStructuredText(agent, rawText) { // Fallback: some providers may return text but fail structured parsing. // AI SDK sets output=null when schema validation fails internally, // so we attempt manual JSON extraction from the raw text as a safety net. - const text = (result.text || '').trim(); + const text = String(rawText || '').trim(); if (!text) { const error = new Error('Model returned empty output text.'); error.code = 'empty_output'; @@ -119,7 +167,8 @@ async function requestStructuredOutput({ agent, input, repairContext }) { throw wrapped; } - const parsed = agent.schema.safeParse(parsedObject); + const schema = agent.parseSchema || agent.schema; + const parsed = schema.safeParse(parsedObject); if (!parsed.success) { const error = new Error(`schema_validation_failed: ${formatIssues(parsed.error.issues)}`); error.code = 'schema_validation_failed'; @@ -130,6 +179,21 @@ async function requestStructuredOutput({ agent, input, repairContext }) { return parsed.data; } +function shouldFallbackToTextJson(error) { + const message = compactErrorMessage(error).toLowerCase(); + return ( + message.includes('responseformat') || + message.includes('response format') || + message.includes('structured output') || + message.includes('structured-output') || + message.includes('no object generated') || + message.includes('response did not match schema') || + (message.includes('schema') && message.includes('object generated')) || + (message.includes('not supported') && message.includes('json')) || + (message.includes('unsupported') && message.includes('json')) + ); +} + function stripCodeFences(text) { const trimmed = String(text || '').trim(); const fenced = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i); @@ -200,6 +264,11 @@ module.exports = { __private: { buildUserInput, requestStructuredOutput, + requestTextJsonOutput, + parseGenerateTextResult, + parseStructuredObject, + parseStructuredText, + shouldFallbackToTextJson, formatIssues, compactErrorMessage } diff --git a/test/agents.test.js b/test/agents.test.js index 3c3ebd7..1c1149b 100644 --- a/test/agents.test.js +++ b/test/agents.test.js @@ -6,7 +6,7 @@ function loadAgents() { return require('../src/agents'); } -test('createReviewerAgent schema accepts nullable/omitted confidence and rejects invalid confidence', () => { +test('createReviewerAgent schema normalizes nullable, omitted, and string confidence', () => { const { createReviewerAgent } = loadAgents(); const agent = createReviewerAgent({ dimension: 'general', @@ -60,39 +60,308 @@ test('createReviewerAgent schema accepts nullable/omitted confidence and rejects }); assert.equal(parsedNumeric.findings[0].confidence, 0.9); - assert.throws( - () => schema.parse({ + const parsedString = schema.parse({ + overall: 'ok', + findings: [ + { + title: 'String confidence', + severity: 'low', + path: 'src/a.js', + summary: 'desc', + confidence: '0.9', + evidence: ['e1'] + } + ] + }); + assert.equal(parsedString.findings[0].confidence, 0.9); + + const confidenceCases = [ + [0, 0], + [0.5, 0.5], + [0.99, 0.99], + [1, 1], + [1.01, 1], + [1.5, 1], + [50, 1], + [100, 1], + [150, 1], + [-0.5, 0], + [Number.NaN, null], + [Infinity, null], + ['not-a-number', null] + ]; + + for (const [input, expected] of confidenceCases) { + const parsed = schema.parse({ overall: 'ok', findings: [ { - title: 'String confidence', + title: `Confidence ${String(input)}`, severity: 'low', path: 'src/a.js', summary: 'desc', - confidence: '0.9', + confidence: input, evidence: ['e1'] } ] - }), - /Expected number, received string/ - ); + }); + assert.equal(parsed.findings[0].confidence, expected); + } - assert.throws( - () => schema.parse({ + const parsedPercent = schema.parse({ + overall: 'ok', + findings: [ + { + title: 'Percent confidence', + severity: 'low', + path: 'src/a.js', + summary: 'desc', + confidence: '90%', + evidence: ['e1'] + } + ] + }); + assert.equal(parsedPercent.findings[0].confidence, 0.9); + + const parsedOutOfRange = schema.parse({ + overall: 'ok', + findings: [ + { + title: 'Out-of-range confidence', + severity: 'low', + path: 'src/a.js', + summary: 'desc', + confidence: 120, + evidence: ['e1'] + } + ] + }); + assert.equal(parsedOutOfRange.findings[0].confidence, 1); +}); + +test('createPlannerAgent schema normalizes planner batches and done values', () => { + const { createPlannerAgent } = loadAgents(); + const agent = createPlannerAgent({ model: 'gpt-test', projectGuidance: null }); + + const parsed = agent.opts.outputType.parse({ + batches: [ + { + focus: 123, + filePaths: 'src/a.js', + reason: null + }, + { + focus: 'empty', + filePaths: [null, '', 'src/b.js', 'src/b.js'], + reason: 'keep one path' + } + ], + done: 'true', + notes: 42 + }); + + assert.equal(parsed.done, true); + assert.equal(parsed.notes, '42'); + assert.deepEqual(parsed.batches[0], { + focus: '123', + filePaths: ['src/a.js'], + reason: '' + }); + assert.deepEqual(parsed.batches[1].filePaths, ['src/b.js']); + + // done boundary tests + const doneCases = [ + ['false', false], + [0, false], + ['', false], + ['no', false], + ['n', false], + ['pending', false], + [null, false], + [undefined, false] + ]; + for (const [input, expected] of doneCases) { + const result = agent.opts.outputType.parse({ + batches: [], + done: input + }); + assert.equal(result.done, expected, `done=${JSON.stringify(input)} should be ${expected}`); + } +}); + +test('createReviewerAgent schema normalizes common model field drift', () => { + const { createReviewerAgent } = loadAgents(); + const agent = createReviewerAgent({ + dimension: 'general', + model: 'gpt-test', + language: 'English', + projectGuidance: null + }); + + const parsed = agent.opts.outputType.parse({ + overall: 123, + findings: [ + { + title: 456, + severity: 'Warning', + category: null, + path: 'src/a.js', + side: 'right', + line: 'R42', + confidence: '90%', + evidence: 'single evidence', + fingerprint: 'x'.repeat(140), + summary: 'summary', + suggestion: 789, + risk: null + }, + { + title: 'Hint severity', + severity: 'hint', + path: 'src/b.js', + summary: 'hint should be low', + evidence: ['e2'] + }, + { + title: 'missing path should be dropped', + severity: 'high', + summary: 'invalid finding' + } + ], + fileConclusions: [ + { + path: 'src/a.js', + conclusion: 100, + risks: 'risk text', + testSuggestions: [null, 'test one'], + note: null + } + ], + recommendedExtraDimensions: 'security', + recommendationReason: null, + actionableSuggestions: ['do this', 99], + potentialRisks: null, + testSuggestions: 'add tests' + }); + + assert.equal(parsed.overall, '123'); + assert.equal(parsed.findings.length, 2); + assert.deepEqual(parsed.findings[0], { + title: '456', + severity: 'medium', + category: 'general', + path: 'src/a.js', + side: 'RIGHT', + line: 42, + confidence: 0.9, + evidence: ['single evidence'], + fingerprint: 'x'.repeat(120), + summary: 'summary', + suggestion: '789', + risk: '' + }); + assert.equal(parsed.findings[1].severity, 'low'); + assert.equal(parsed.findings[1].title, 'Hint severity'); + assert.deepEqual(parsed.fileConclusions[0], { + path: 'src/a.js', + conclusion: '100', + risks: ['risk text'], + testSuggestions: ['test one'], + note: '' + }); + assert.deepEqual(parsed.recommendedExtraDimensions, ['security']); + assert.equal(parsed.recommendationReason, ''); + assert.deepEqual(parsed.actionableSuggestions, ['do this', '99']); + assert.deepEqual(parsed.potentialRisks, []); + assert.deepEqual(parsed.testSuggestions, ['add tests']); + + // side boundary tests + const sideCases = [ + ['left', 'LEFT'], + ['LEFT', 'LEFT'], + ['l', 'LEFT'], + ['FILE', 'FILE'], + ['file', 'FILE'], + ['none', 'FILE'], + ['general', 'FILE'], + ['center', 'RIGHT'], + ['both', 'RIGHT'], + ['', 'RIGHT'] + ]; + for (const [input, expected] of sideCases) { + const sideResult = agent.opts.outputType.parse({ overall: 'ok', findings: [ { - title: 'Out-of-range confidence', + title: `Side ${input}`, severity: 'low', path: 'src/a.js', summary: 'desc', - confidence: 1.2, + side: input, evidence: ['e1'] } ] + }); + assert.equal(sideResult.findings[0].side, expected, `side=${JSON.stringify(input)} should be ${expected}`); + } +}); + +test('createReviewerAgent exposes strict generation schema and tolerant parse schema', () => { + const { createReviewerAgent } = loadAgents(); + const agent = createReviewerAgent({ + dimension: 'general', + model: 'gpt-test', + language: 'English', + projectGuidance: null + }); + + assert.notEqual(agent.schema, agent.parseSchema); + assert.throws( + () => agent.schema.parse({ + overall: 'ok', + findings: [ + { + title: 'String confidence', + severity: 'low', + path: 'src/a.js', + summary: 'desc', + confidence: '0.9' + } + ] }), - /Number must be less than or equal to 1/ + /Expected number, received string/ ); + for (const invalidConfidence of [1.2, -0.5]) { + assert.throws( + () => agent.schema.parse({ + overall: 'ok', + findings: [ + { + title: 'Out-of-range confidence', + severity: 'low', + path: 'src/a.js', + summary: 'desc', + confidence: invalidConfidence + } + ] + }), + /Number must be (less than or equal to 1|greater than or equal to 0)/ + ); + } + + const parsed = agent.parseSchema.parse({ + overall: 'ok', + findings: [ + { + title: 'String confidence', + severity: 'low', + path: 'src/a.js', + summary: 'desc', + confidence: '0.9' + } + ] + }); + assert.equal(parsed.findings[0].confidence, 0.9); }); test('buildBatchReviewInput keeps additional file with truncation at boundary', () => { @@ -207,3 +476,134 @@ test('buildBatchReviewInput preserves line anchors for code starting with +++ an assert.match(result.prompt, /\[L1\|R-\] - ---old/); assert.match(result.prompt, /\[L-\|R1\] \+ \+\+\+new/); }); + +// --- Direct coerce function unit tests --- + +test('coerceSeverity maps standard values and common aliases', () => { + const { __private } = loadAgents(); + const cs = __private.coerceSeverity; + assert.equal(cs('critical'), 'critical'); + assert.equal(cs('high'), 'high'); + assert.equal(cs('medium'), 'medium'); + assert.equal(cs('low'), 'low'); + assert.equal(cs('CRITICAL'), 'critical'); + assert.equal(cs('High'), 'high'); + assert.equal(cs('blocker'), 'critical'); + assert.equal(cs('blocking'), 'critical'); + assert.equal(cs('major'), 'high'); + assert.equal(cs('hi'), 'high'); + assert.equal(cs('warning'), 'medium'); + assert.equal(cs('warn'), 'medium'); + assert.equal(cs('minor'), 'low'); + assert.equal(cs('info'), 'low'); + assert.equal(cs('informational'), 'low'); + assert.equal(cs('hint'), 'low'); + assert.equal(cs('unknown'), 'medium'); + assert.equal(cs(''), 'medium'); +}); + +test('coerceSide maps standard values and common aliases', () => { + const { __private } = loadAgents(); + const cs = __private.coerceSide; + assert.equal(cs('LEFT'), 'LEFT'); + assert.equal(cs('RIGHT'), 'RIGHT'); + assert.equal(cs('FILE'), 'FILE'); + assert.equal(cs('left'), 'LEFT'); + assert.equal(cs('right'), 'RIGHT'); + assert.equal(cs('file'), 'FILE'); + assert.equal(cs('L'), 'LEFT'); + assert.equal(cs('OLD'), 'LEFT'); + assert.equal(cs('REMOVED'), 'LEFT'); + assert.equal(cs('DELETED'), 'LEFT'); + assert.equal(cs('DELETION'), 'LEFT'); + assert.equal(cs('R'), 'RIGHT'); + assert.equal(cs('NEW'), 'RIGHT'); + assert.equal(cs('ADDED'), 'RIGHT'); + assert.equal(cs('ADDITION'), 'RIGHT'); + assert.equal(cs('NONE'), 'FILE'); + assert.equal(cs('GENERAL'), 'FILE'); + assert.equal(cs('OVERALL'), 'FILE'); + assert.equal(cs('center'), 'RIGHT'); + assert.equal(cs('both'), 'RIGHT'); + assert.equal(cs(''), 'RIGHT'); +}); + +test('coercePositiveIntegerOrNull handles various inputs', () => { + const { __private } = loadAgents(); + const cp = __private.coercePositiveIntegerOrNull; + assert.equal(cp(42), 42); + assert.equal(cp(1), 1); + assert.equal(cp('42'), 42); + assert.equal(cp('R42'), 42); + assert.equal(cp('L42'), 42); + assert.equal(cp('R:42'), 42); + assert.equal(cp('#42'), 42); + assert.equal(cp(0), null); + assert.equal(cp(-1), null); + assert.equal(cp('R0'), null); + assert.equal(cp('R-1'), null); + assert.equal(cp('abc'), null); + assert.equal(cp(''), null); + assert.equal(cp(null), null); + assert.equal(cp(undefined), null); + assert.equal(cp(3.14), null); + assert.equal(cp('42R'), null); +}); + +test('coerceConfidence handles numeric, string, percent, and edge cases', () => { + const { __private } = loadAgents(); + const cc = __private.coerceConfidence; + assert.equal(cc(0), 0); + assert.equal(cc(0.5), 0.5); + assert.equal(cc(1), 1); + assert.equal(cc(1.5), 1); + assert.equal(cc(100), 1); + assert.equal(cc('0.9'), 0.9); + assert.equal(cc('90%'), 0.9); + assert.equal(cc('100%'), 1); + assert.equal(cc('150%'), 1); + assert.equal(cc(-0.5), 0); + assert.equal(cc(null), null); + assert.equal(cc(undefined), null); + assert.equal(cc(''), null); + assert.equal(cc('not-a-number'), null); + assert.equal(cc(Infinity), null); + assert.equal(cc(NaN), null); +}); + +test('coerceBoolean handles various truthy and falsy inputs', () => { + const { __private } = loadAgents(); + const cb = __private.coerceBoolean; + assert.equal(cb(true), true); + assert.equal(cb(false), false); + assert.equal(cb(1), true); + assert.equal(cb(0), false); + assert.equal(cb('true'), true); + assert.equal(cb('1'), true); + assert.equal(cb('yes'), true); + assert.equal(cb('y'), true); + assert.equal(cb('done'), true); + assert.equal(cb('false'), false); + assert.equal(cb('0'), false); + assert.equal(cb('no'), false); + assert.equal(cb('n'), false); + assert.equal(cb('pending'), false); + assert.equal(cb('complete'), false); + assert.equal(cb('finished'), false); + assert.equal(cb(''), false); + assert.equal(cb(undefined), undefined); + assert.equal(cb(null), undefined); +}); + +test('coerceStringArray deduplicates, flattens, and filters', () => { + const { __private } = loadAgents(); + const csa = __private.coerceStringArray; + assert.deepEqual(csa(['a', 'b', 'a']), ['a', 'b']); + assert.deepEqual(csa(['a', null, '', 'b']), ['a', 'b']); + assert.deepEqual(csa('single'), ['single']); + assert.deepEqual(csa(null), []); + assert.deepEqual(csa(undefined), []); + assert.deepEqual(csa([null, undefined]), []); + assert.deepEqual(csa(['a', 'b', 'c', 'd'], 2), ['a', 'b']); + assert.deepEqual(csa([[1, 2], 'a']), ['1', '2', 'a']); +}); diff --git a/test/model-runtime.test.js b/test/model-runtime.test.js index 2b8667c..8ec4480 100644 --- a/test/model-runtime.test.js +++ b/test/model-runtime.test.js @@ -137,14 +137,32 @@ test('runStructuredWithRepair handles empty output text', async () => { assert.ok(result.error.message.includes('empty output')); }); -test('runStructuredWithRepair handles generateText throwing', async () => { - let callCount = 0; +test('runStructuredWithRepair does not fallback for non-structured generateText errors', async () => { const runtime = loadRuntimeWithMockedAI(async () => { - callCount += 1; - if (callCount === 1) { - throw new Error('API rate limit exceeded'); + throw new Error('API rate limit exceeded'); + }); + + runtime.configureRuntime({ model: createFakeModel() }); + const agent = createAgent(); + const result = await runtime.runStructuredWithRepair(agent, 'review'); + + assert.equal(result.ok, false); + assert.match(result.error.message, /API rate limit exceeded/); + assert.equal(result.calls, 2); + assert.equal(result.repaired, true); +}); + +test('runStructuredWithRepair falls back to plain text JSON when structured output is unsupported', async () => { + const calls = []; + const runtime = loadRuntimeWithMockedAI(async (opts) => { + calls.push(opts); + if (opts.output) { + throw new Error('No object generated: response did not match schema. The feature "responseFormat" is not supported.'); } - return { output: { overall: 'recovered' }, text: '{"overall":"recovered"}' }; + return { + output: undefined, + text: '```json\n{"overall":"from fallback"}\n```' + }; }); runtime.configureRuntime({ model: createFakeModel() }); @@ -152,8 +170,36 @@ test('runStructuredWithRepair handles generateText throwing', async () => { const result = await runtime.runStructuredWithRepair(agent, 'review'); assert.equal(result.ok, true); - assert.deepEqual(result.output, { overall: 'recovered' }); - assert.equal(result.repaired, true); + assert.deepEqual(result.output, { overall: 'from fallback' }); + assert.equal(result.calls, 1); + assert.equal(result.repaired, false); + assert.equal(calls.length, 2); + assert.ok(calls[0].output); + assert.equal(calls[1].output, undefined); +}); + +test('runStructuredWithRepair reports structured fallback failure when text JSON fallback also fails', async () => { + let callCount = 0; + const runtime = loadRuntimeWithMockedAI(async (opts) => { + callCount += 1; + if (opts.output) { + throw new Error('No object generated: response did not match schema.'); + } + return { + output: undefined, + text: 'not json' + }; + }); + + runtime.configureRuntime({ model: createFakeModel() }); + const agent = createAgent(); + const result = await runtime.runStructuredWithRepair(agent, 'review', { allowRepair: false }); + + assert.equal(result.ok, false); + assert.equal(result.error.code, 'structured_text_fallback_failed'); + assert.match(result.error.message, /structured_text_fallback_failed/); + assert.match(result.error.message, /original_error: No object generated/); + assert.equal(callCount, 2); }); test('configureRuntime throws when model is falsy', () => { @@ -214,6 +260,25 @@ test('requestStructuredOutput falls back to runtimeState.model when agent.modelI assert.equal(usedModel, defaultModel); }); +test('runStructuredWithRepair validates model output through parseSchema when provided', async () => { + const runtime = loadRuntimeWithMockedAI(async () => ({ + output: { overall: 123 }, + text: '{}' + })); + + runtime.configureRuntime({ model: createFakeModel() }); + const agent = { + ...createAgent(), + parseSchema: z.object({ + overall: z.preprocess((value) => String(value), z.string()) + }) + }; + const result = await runtime.runStructuredWithRepair(agent, 'review'); + + assert.equal(result.ok, true); + assert.deepEqual(result.output, { overall: '123' }); +}); + test('schema validation failure on first attempt triggers repair', async () => { let callCount = 0; const runtime = loadRuntimeWithMockedAI(async () => {