From 921d8341cb6f300b59e597f1d8a8032e622713d7 Mon Sep 17 00:00:00 2001 From: zerob13 Date: Fri, 27 Mar 2026 16:13:17 +0800 Subject: [PATCH 1/2] fix(chat): restore image generation blocks --- .../deepchatAgentPresenter/accumulator.ts | 15 +++++++++++++ .../providers/openAICompatibleProvider.ts | 15 ++++++++++++- src/shared/types/agent-interface.d.ts | 5 +++++ .../accumulator.test.ts | 22 +++++++++++++++++++ .../openAICompatibleProvider.test.ts | 18 ++++++++++++++- 5 files changed, 73 insertions(+), 2 deletions(-) diff --git a/src/main/presenter/deepchatAgentPresenter/accumulator.ts b/src/main/presenter/deepchatAgentPresenter/accumulator.ts index 141259676..addfa937d 100644 --- a/src/main/presenter/deepchatAgentPresenter/accumulator.ts +++ b/src/main/presenter/deepchatAgentPresenter/accumulator.ts @@ -119,6 +119,21 @@ export function accumulate(state: StreamState, event: LLMCoreStreamEvent): void } break } + case 'image_data': { + if (state.firstTokenTime === null) state.firstTokenTime = Date.now() + const block: AssistantMessageBlock = { + type: 'image', + status: 'pending', + timestamp: Date.now(), + image_data: { + data: event.image_data.data, + mimeType: event.image_data.mimeType + } + } + state.blocks.push(block) + state.dirty = true + break + } case 'usage': { state.metadata.inputTokens = event.usage.prompt_tokens state.metadata.outputTokens = event.usage.completion_tokens diff --git a/src/main/presenter/llmProviderPresenter/providers/openAICompatibleProvider.ts b/src/main/presenter/llmProviderPresenter/providers/openAICompatibleProvider.ts index eac2fafce..8aedd633b 100644 --- a/src/main/presenter/llmProviderPresenter/providers/openAICompatibleProvider.ts +++ b/src/main/presenter/llmProviderPresenter/providers/openAICompatibleProvider.ts @@ -66,6 +66,19 @@ const SUPPORTED_IMAGE_SIZES = { // Add list of models with configurable sizes const SIZE_CONFIGURABLE_MODELS = ['gpt-image-1', 'gpt-4o-image', 'gpt-4o-all'] +export function normalizeExtractedImageText(content: string): string { + const normalized = content + .replace(/\r\n/g, '\n') + .replace(/\n\s*\n/g, '\n') + .trim() + if (!normalized) { + return '' + } + + const semanticText = normalized.replace(/[\`*_~!\[\]\(\)]/g, '').trim() + return semanticText.length > 0 ? normalized : '' +} + function getOpenAIChatCachedTokens(usage: unknown): number | undefined { if (!usage || typeof usage !== 'object') { return undefined @@ -1294,7 +1307,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider { // 如果处理了图片，清理多余的空行并记录日志 if (hasImages) { // 清理移除图片后可能留下的多余空行 - processedCurrentContent = processedCurrentContent.replace(/\n\s*\n/g, '\n').trim() + processedCurrentContent = normalizeExtractedImageText(processedCurrentContent) console.log( `[handleChatCompletion] Processed ${currentContent.length} chars -> ${processedCurrentContent.length} chars (images removed)` ) diff --git a/src/shared/types/agent-interface.d.ts b/src/shared/types/agent-interface.d.ts index d3aac8af3..7aa6aa822 100644 --- a/src/shared/types/agent-interface.d.ts +++ b/src/shared/types/agent-interface.d.ts @@ -210,6 +210,7 @@ export type AssistantBlockType = | 'error' | 'tool_call' | 'action' + | 'image' export interface ToolCallBlockData { id?: string @@ -263,6 +264,10 @@ export interface AssistantMessageBlock { start: number end: number } + image_data?: { + data: string + mimeType: string + } tool_call?: ToolCallBlockData extra?: AssistantMessageExtra action_type?: 'tool_call_permission' | 'question_request' diff --git a/test/main/presenter/deepchatAgentPresenter/accumulator.test.ts b/test/main/presenter/deepchatAgentPresenter/accumulator.test.ts index 35bd3b6cf..36c322e2b 100644 --- a/test/main/presenter/deepchatAgentPresenter/accumulator.test.ts +++ b/test/main/presenter/deepchatAgentPresenter/accumulator.test.ts @@ -236,3 +236,25 @@ describe('accumulate', () => { expect(state.blocks.length).toBe(blocksBefore) }) }) + +it('creates image blocks for image_data events without empty text blocks', () => { + accumulate(state, { + type: 'image_data', + image_data: { + data: 'imgcache://generated/test.png', + mimeType: 'deepchat/image-url' + } + }) + + expect(state.blocks).toHaveLength(1) + expect(state.blocks[0]).toMatchObject({ + type: 'image', + status: 'pending', + image_data: { + data: 'imgcache://generated/test.png', + mimeType: 'deepchat/image-url' + } + }) + expect(state.blocks.some((block) => block.type === 'content')).toBe(false) + expect(state.dirty).toBe(true) +}) diff --git a/test/main/presenter/llmProviderPresenter/openAICompatibleProvider.test.ts b/test/main/presenter/llmProviderPresenter/openAICompatibleProvider.test.ts index c702108e7..e751dfc1d 100644 --- a/test/main/presenter/llmProviderPresenter/openAICompatibleProvider.test.ts +++ b/test/main/presenter/llmProviderPresenter/openAICompatibleProvider.test.ts @@ -7,7 +7,10 @@ import type { MCPToolDefinition, ModelConfig } from '../../../../src/shared/presenter' -import { OpenAICompatibleProvider } from '../../../../src/main/presenter/llmProviderPresenter/providers/openAICompatibleProvider' +import { + OpenAICompatibleProvider, + normalizeExtractedImageText +} from '../../../../src/main/presenter/llmProviderPresenter/providers/openAICompatibleProvider' import { OpenRouterProvider } from '../../../../src/main/presenter/llmProviderPresenter/providers/openRouterProvider' import { LLMProviderPresenter } from '../../../../src/main/presenter/llmProviderPresenter' @@ -335,3 +338,16 @@ describe('OpenAICompatibleProvider MCP runtime injection', () => { expect(requestParams.tools).toEqual(convertedTools) }) }) + +describe('normalizeExtractedImageText', () => { + it('keeps meaningful text after image markdown cleanup', () => { + expect(normalizeExtractedImageText(' Here is the updated image.\n\n')).toBe( + 'Here is the updated image.' + ) + }) + + it('drops markdown residue after image markdown cleanup', () => { + expect(normalizeExtractedImageText('`\n')).toBe('') + expect(normalizeExtractedImageText('[]()')).toBe('') + }) +}) From 4c95ee6a1c8575334cba16c961c095b8b59d7706 Mon Sep 17 00:00:00 2001 From: zerob13 Date: Fri, 27 Mar 2026 16:33:29 +0800 Subject: [PATCH 2/2] test(chat): scope image accumulator test --- .../accumulator.test.ts | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/test/main/presenter/deepchatAgentPresenter/accumulator.test.ts b/test/main/presenter/deepchatAgentPresenter/accumulator.test.ts index 36c322e2b..2d39f19c5 100644 --- a/test/main/presenter/deepchatAgentPresenter/accumulator.test.ts +++ b/test/main/presenter/deepchatAgentPresenter/accumulator.test.ts @@ -235,26 +235,26 @@ describe('accumulate', () => { accumulate(state, { type: 'permission', permission: {} } as any) expect(state.blocks.length).toBe(blocksBefore) }) -}) -it('creates image blocks for image_data events without empty text blocks', () => { - accumulate(state, { - type: 'image_data', - image_data: { - data: 'imgcache://generated/test.png', - mimeType: 'deepchat/image-url' - } - }) + it('creates image blocks for image_data events without empty text blocks', () => { + accumulate(state, { + type: 'image_data', + image_data: { + data: 'imgcache://generated/test.png', + mimeType: 'deepchat/image-url' + } + }) - expect(state.blocks).toHaveLength(1) - expect(state.blocks[0]).toMatchObject({ - type: 'image', - status: 'pending', - image_data: { - data: 'imgcache://generated/test.png', - mimeType: 'deepchat/image-url' - } + expect(state.blocks).toHaveLength(1) + expect(state.blocks[0]).toMatchObject({ + type: 'image', + status: 'pending', + image_data: { + data: 'imgcache://generated/test.png', + mimeType: 'deepchat/image-url' + } + }) + expect(state.blocks.some((block) => block.type === 'content')).toBe(false) + expect(state.dirty).toBe(true) }) - expect(state.blocks.some((block) => block.type === 'content')).toBe(false) - expect(state.dirty).toBe(true) })