From 7ac3567eef5abcc6c998ea6e45db7b3750e603d6 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 10 Apr 2026 06:42:37 +0000 Subject: [PATCH] fix(core): accept string items in message content arrays isTestMessage previously rejected message content arrays whose items were not all JSON objects, so an eval whose content mixed an inline string and a {type: file, ...} block validated cleanly but was skipped at runtime as "incomplete". The validator already accepted this shape. Make the loader and prompt builder agree with the validator by treating plain string items inside a content array as inline text segments. The existing structured-block handling for type: text / file / image is unchanged. - Widen TestMessageContent to allow (string | JsonObject)[] arrays. - Update isTestMessage to accept mixed string/object content arrays. - Convert string items to {type:text,value:...} segments in extractContentSegments, processMessages, and processExpectedMessages. - Add unit tests covering the mixed-content shape end to end. Closes #1034 Co-Authored-By: Claude Opus 4.6 --- .../src/evaluation/input-message-utils.ts | 8 +++++ .../evaluation/loaders/message-processor.ts | 20 +++++++++++ packages/core/src/evaluation/types.ts | 13 ++++++-- .../loaders/message-processor.test.ts | 33 +++++++++++++++++++ .../loaders/shorthand-expansion.test.ts | 19 +++++++++++ 5 files changed, 91 insertions(+), 2 deletions(-) diff --git a/packages/core/src/evaluation/input-message-utils.ts b/packages/core/src/evaluation/input-message-utils.ts index f5b9e1a9e..35d4d89b5 100644 --- a/packages/core/src/evaluation/input-message-utils.ts +++ b/packages/core/src/evaluation/input-message-utils.ts @@ -53,6 +53,14 @@ export function extractContentSegments(content: TestMessageContent): JsonObject[ const segments: JsonObject[] = []; for (const segment of content) { + // Plain string items inside a content array are treated as text segments. + // This matches the validator, which accepts string items in content arrays. + if (typeof segment === 'string') { + if (segment.trim().length > 0) { + segments.push({ type: 'text', value: segment }); + } + continue; + } if (!isJsonObject(segment)) { continue; } diff --git a/packages/core/src/evaluation/loaders/message-processor.ts b/packages/core/src/evaluation/loaders/message-processor.ts index 1e1a9b48d..4aa2b4ea8 100644 --- a/packages/core/src/evaluation/loaders/message-processor.ts +++ b/packages/core/src/evaluation/loaders/message-processor.ts @@ -79,6 +79,18 @@ export async function processMessages(options: ProcessMessagesOptions): Promise< const processedContent: JsonObject[] = []; for (const rawSegment of content) { + // Plain string items inside a content array are treated as inline text segments. + // This matches the validator, which accepts string items alongside structured blocks. + if (typeof rawSegment === 'string') { + if (rawSegment.length > 0) { + processedContent.push({ type: 'text', value: rawSegment }); + if (textParts) { + textParts.push(rawSegment); + } + } + continue; + } + if (!isJsonObject(rawSegment)) { continue; } @@ -334,6 +346,14 @@ export async function processExpectedMessages( // Process content array, resolving file references const processedContent: JsonObject[] = []; for (const rawSegment of content) { + // Plain string items are treated as inline text segments (matches validator). + if (typeof rawSegment === 'string') { + if (rawSegment.length > 0) { + processedContent.push({ type: 'text', value: rawSegment }); + } + continue; + } + if (!isJsonObject(rawSegment)) { continue; } diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index 18144cb11..792d3bcd1 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -40,8 +40,12 @@ const TEST_MESSAGE_ROLE_SET: ReadonlySet = new Set(TEST_MESSAGE_ROLE_VAL /** * Text or structured payload attached to a message. + * + * Content arrays may mix plain string items with structured content blocks + * (e.g. `{ type: 'text' | 'file' | 'image', value: ... }`). Plain string items + * are treated as text segments by the loader and prompt builder. */ -export type TestMessageContent = string | JsonObject | readonly JsonObject[]; +export type TestMessageContent = string | JsonObject | readonly (string | JsonObject)[]; /** * System-authored instruction message. @@ -140,7 +144,12 @@ export function isTestMessage(value: unknown): value is TestMessage { if (typeof candidate.content === 'string') { return true; } - if (Array.isArray(candidate.content) && candidate.content.every(isJsonObject)) { + // Content arrays may mix plain string items with structured content blocks. + // The loader treats string items as text segments (see message-processor.ts). + if ( + Array.isArray(candidate.content) && + candidate.content.every((item) => typeof item === 'string' || isJsonObject(item)) + ) { return true; } // Allow messages with tool_calls but no content (for expected_output format) diff --git a/packages/core/test/evaluation/loaders/message-processor.test.ts b/packages/core/test/evaluation/loaders/message-processor.test.ts index 1c7f5510c..ccac4e490 100644 --- a/packages/core/test/evaluation/loaders/message-processor.test.ts +++ b/packages/core/test/evaluation/loaders/message-processor.test.ts @@ -162,6 +162,39 @@ describe('processMessages – image content', () => { } }); + it('converts plain string items in content arrays into text segments', async () => { + await setupFixtures(); + try { + const messages: TestMessage[] = [ + { + role: 'user', + content: ['Use the local file.', { type: 'file', value: './test-file.txt' }], + }, + ]; + + const textParts: string[] = []; + const result = await processMessages({ + messages, + searchRoots: [FIXTURE_DIR], + repoRootPath: FIXTURE_DIR, + textParts, + messageType: 'input', + verbose: false, + }); + + const items = result[0].content as Record[]; + expect(items).toHaveLength(2); + expect(items[0].type).toBe('text'); + expect(items[0].value).toBe('Use the local file.'); + expect(items[1].type).toBe('file'); + expect(items[1].text).toBe('hello world'); + // Inline string was also surfaced through textParts for prompt building. + expect(textParts).toContain('Use the local file.'); + } finally { + await cleanupFixtures(); + } + }); + it('preserves existing type: text and type: file behavior', async () => { await setupFixtures(); try { diff --git a/packages/core/test/evaluation/loaders/shorthand-expansion.test.ts b/packages/core/test/evaluation/loaders/shorthand-expansion.test.ts index 95f28200d..f0ec058a1 100644 --- a/packages/core/test/evaluation/loaders/shorthand-expansion.test.ts +++ b/packages/core/test/evaluation/loaders/shorthand-expansion.test.ts @@ -61,6 +61,25 @@ describe('expandInputShorthand', () => { const messages = [{ invalid: 'message' }, { also: 'invalid' }]; expect(expandInputShorthand(messages)).toBeUndefined(); }); + + it('accepts messages whose content array mixes plain strings and structured blocks', () => { + const messages = [ + { + role: 'user', + content: ['Use the local file.', { type: 'file', value: '/README.md' }], + }, + ]; + + const result = expandInputShorthand(messages); + + expect(result).toHaveLength(1); + const content = result?.[0].content; + expect(Array.isArray(content)).toBe(true); + const items = content as Array; + expect(items).toHaveLength(2); + expect(items[0]).toBe('Use the local file.'); + expect(items[1]).toEqual({ type: 'file', value: '/README.md' }); + }); }); describe('expandExpectedOutputShorthand', () => {