diff --git a/packages/core/src/evaluation/input-message-utils.ts b/packages/core/src/evaluation/input-message-utils.ts index f5b9e1a9e..35d4d89b5 100644 --- a/packages/core/src/evaluation/input-message-utils.ts +++ b/packages/core/src/evaluation/input-message-utils.ts @@ -53,6 +53,14 @@ export function extractContentSegments(content: TestMessageContent): JsonObject[ const segments: JsonObject[] = []; for (const segment of content) { + // Plain string items inside a content array are treated as text segments. + // This matches the validator, which accepts string items in content arrays. + if (typeof segment === 'string') { + if (segment.trim().length > 0) { + segments.push({ type: 'text', value: segment }); + } + continue; + } if (!isJsonObject(segment)) { continue; } diff --git a/packages/core/src/evaluation/loaders/message-processor.ts b/packages/core/src/evaluation/loaders/message-processor.ts index 1e1a9b48d..4aa2b4ea8 100644 --- a/packages/core/src/evaluation/loaders/message-processor.ts +++ b/packages/core/src/evaluation/loaders/message-processor.ts @@ -79,6 +79,18 @@ export async function processMessages(options: ProcessMessagesOptions): Promise< const processedContent: JsonObject[] = []; for (const rawSegment of content) { + // Plain string items inside a content array are treated as inline text segments. + // This matches the validator, which accepts string items alongside structured blocks. + if (typeof rawSegment === 'string') { + if (rawSegment.length > 0) { + processedContent.push({ type: 'text', value: rawSegment }); + if (textParts) { + textParts.push(rawSegment); + } + } + continue; + } + if (!isJsonObject(rawSegment)) { continue; } @@ -334,6 +346,14 @@ export async function processExpectedMessages( // Process content array, resolving file references const processedContent: JsonObject[] = []; for (const rawSegment of content) { + // Plain string items are treated as inline text segments (matches validator). + if (typeof rawSegment === 'string') { + if (rawSegment.length > 0) { + processedContent.push({ type: 'text', value: rawSegment }); + } + continue; + } + if (!isJsonObject(rawSegment)) { continue; } diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index 18144cb11..792d3bcd1 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -40,8 +40,12 @@ const TEST_MESSAGE_ROLE_SET: ReadonlySet = new Set(TEST_MESSAGE_ROLE_VAL /** * Text or structured payload attached to a message. + * + * Content arrays may mix plain string items with structured content blocks + * (e.g. `{ type: 'text' | 'file' | 'image', value: ... }`). Plain string items + * are treated as text segments by the loader and prompt builder. */ -export type TestMessageContent = string | JsonObject | readonly JsonObject[]; +export type TestMessageContent = string | JsonObject | readonly (string | JsonObject)[]; /** * System-authored instruction message. @@ -140,7 +144,12 @@ export function isTestMessage(value: unknown): value is TestMessage { if (typeof candidate.content === 'string') { return true; } - if (Array.isArray(candidate.content) && candidate.content.every(isJsonObject)) { + // Content arrays may mix plain string items with structured content blocks. + // The loader treats string items as text segments (see message-processor.ts). + if ( + Array.isArray(candidate.content) && + candidate.content.every((item) => typeof item === 'string' || isJsonObject(item)) + ) { return true; } // Allow messages with tool_calls but no content (for expected_output format) diff --git a/packages/core/test/evaluation/loaders/message-processor.test.ts b/packages/core/test/evaluation/loaders/message-processor.test.ts index 1c7f5510c..ccac4e490 100644 --- a/packages/core/test/evaluation/loaders/message-processor.test.ts +++ b/packages/core/test/evaluation/loaders/message-processor.test.ts @@ -162,6 +162,39 @@ describe('processMessages – image content', () => { } }); + it('converts plain string items in content arrays into text segments', async () => { + await setupFixtures(); + try { + const messages: TestMessage[] = [ + { + role: 'user', + content: ['Use the local file.', { type: 'file', value: './test-file.txt' }], + }, + ]; + + const textParts: string[] = []; + const result = await processMessages({ + messages, + searchRoots: [FIXTURE_DIR], + repoRootPath: FIXTURE_DIR, + textParts, + messageType: 'input', + verbose: false, + }); + + const items = result[0].content as Record[]; + expect(items).toHaveLength(2); + expect(items[0].type).toBe('text'); + expect(items[0].value).toBe('Use the local file.'); + expect(items[1].type).toBe('file'); + expect(items[1].text).toBe('hello world'); + // Inline string was also surfaced through textParts for prompt building. + expect(textParts).toContain('Use the local file.'); + } finally { + await cleanupFixtures(); + } + }); + it('preserves existing type: text and type: file behavior', async () => { await setupFixtures(); try { diff --git a/packages/core/test/evaluation/loaders/shorthand-expansion.test.ts b/packages/core/test/evaluation/loaders/shorthand-expansion.test.ts index 95f28200d..f0ec058a1 100644 --- a/packages/core/test/evaluation/loaders/shorthand-expansion.test.ts +++ b/packages/core/test/evaluation/loaders/shorthand-expansion.test.ts @@ -61,6 +61,25 @@ describe('expandInputShorthand', () => { const messages = [{ invalid: 'message' }, { also: 'invalid' }]; expect(expandInputShorthand(messages)).toBeUndefined(); }); + + it('accepts messages whose content array mixes plain strings and structured blocks', () => { + const messages = [ + { + role: 'user', + content: ['Use the local file.', { type: 'file', value: '/README.md' }], + }, + ]; + + const result = expandInputShorthand(messages); + + expect(result).toHaveLength(1); + const content = result?.[0].content; + expect(Array.isArray(content)).toBe(true); + const items = content as Array; + expect(items).toHaveLength(2); + expect(items[0]).toBe('Use the local file.'); + expect(items[1]).toEqual({ type: 'file', value: '/README.md' }); + }); }); describe('expandExpectedOutputShorthand', () => {