Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions src/main/presenter/deepchatAgentPresenter/accumulator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,21 @@ export function accumulate(state: StreamState, event: LLMCoreStreamEvent): void
}
break
}
case 'image_data': {
if (state.firstTokenTime === null) state.firstTokenTime = Date.now()
const block: AssistantMessageBlock = {
type: 'image',
status: 'pending',
timestamp: Date.now(),
image_data: {
data: event.image_data.data,
mimeType: event.image_data.mimeType
}
}
state.blocks.push(block)
state.dirty = true
break
}
case 'usage': {
state.metadata.inputTokens = event.usage.prompt_tokens
state.metadata.outputTokens = event.usage.completion_tokens
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,19 @@
// Add list of models with configurable sizes
const SIZE_CONFIGURABLE_MODELS = ['gpt-image-1', 'gpt-4o-image', 'gpt-4o-all']

export function normalizeExtractedImageText(content: string): string {
const normalized = content
.replace(/\r\n/g, '\n')
.replace(/\n\s*\n/g, '\n')
.trim()
if (!normalized) {
return ''
}

const semanticText = normalized.replace(/[\`*_~!\[\]\(\)]/g, '').trim()

Check warning on line 78 in src/main/presenter/llmProviderPresenter/providers/openAICompatibleProvider.ts

View workflow job for this annotation

GitHub Actions / build-check (x64)

eslint(no-useless-escape)

Unnecessary escape character ')'

Check warning on line 78 in src/main/presenter/llmProviderPresenter/providers/openAICompatibleProvider.ts

View workflow job for this annotation

GitHub Actions / build-check (x64)

eslint(no-useless-escape)

Unnecessary escape character '('

Check warning on line 78 in src/main/presenter/llmProviderPresenter/providers/openAICompatibleProvider.ts

View workflow job for this annotation

GitHub Actions / build-check (x64)

eslint(no-useless-escape)

Unnecessary escape character '['

Check warning on line 78 in src/main/presenter/llmProviderPresenter/providers/openAICompatibleProvider.ts

View workflow job for this annotation

GitHub Actions / build-check (x64)

eslint(no-useless-escape)

Unnecessary escape character '`'
return semanticText.length > 0 ? normalized : ''
}

function getOpenAIChatCachedTokens(usage: unknown): number | undefined {
if (!usage || typeof usage !== 'object') {
return undefined
Expand Down Expand Up @@ -1294,7 +1307,7 @@
// 如果处理了图片,清理多余的空行并记录日志
if (hasImages) {
// 清理移除图片后可能留下的多余空行
processedCurrentContent = processedCurrentContent.replace(/\n\s*\n/g, '\n').trim()
processedCurrentContent = normalizeExtractedImageText(processedCurrentContent)
console.log(
`[handleChatCompletion] Processed ${currentContent.length} chars -> ${processedCurrentContent.length} chars (images removed)`
)
Expand Down
5 changes: 5 additions & 0 deletions src/shared/types/agent-interface.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ export type AssistantBlockType =
| 'error'
| 'tool_call'
| 'action'
| 'image'

export interface ToolCallBlockData {
id?: string
Expand Down Expand Up @@ -263,6 +264,10 @@ export interface AssistantMessageBlock {
start: number
end: number
}
image_data?: {
data: string
mimeType: string
}
tool_call?: ToolCallBlockData
extra?: AssistantMessageExtra
action_type?: 'tool_call_permission' | 'question_request'
Expand Down
22 changes: 22 additions & 0 deletions test/main/presenter/deepchatAgentPresenter/accumulator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -235,4 +235,26 @@ describe('accumulate', () => {
accumulate(state, { type: 'permission', permission: {} } as any)
expect(state.blocks.length).toBe(blocksBefore)
})

it('creates image blocks for image_data events without empty text blocks', () => {
accumulate(state, {
type: 'image_data',
image_data: {
data: 'imgcache://generated/test.png',
mimeType: 'deepchat/image-url'
}
})

expect(state.blocks).toHaveLength(1)
expect(state.blocks[0]).toMatchObject({
type: 'image',
status: 'pending',
image_data: {
data: 'imgcache://generated/test.png',
mimeType: 'deepchat/image-url'
}
})
expect(state.blocks.some((block) => block.type === 'content')).toBe(false)
expect(state.dirty).toBe(true)
})
})
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@ import type {
MCPToolDefinition,
ModelConfig
} from '../../../../src/shared/presenter'
import { OpenAICompatibleProvider } from '../../../../src/main/presenter/llmProviderPresenter/providers/openAICompatibleProvider'
import {
OpenAICompatibleProvider,
normalizeExtractedImageText
} from '../../../../src/main/presenter/llmProviderPresenter/providers/openAICompatibleProvider'
import { OpenRouterProvider } from '../../../../src/main/presenter/llmProviderPresenter/providers/openRouterProvider'
import { LLMProviderPresenter } from '../../../../src/main/presenter/llmProviderPresenter'

Expand Down Expand Up @@ -335,3 +338,16 @@ describe('OpenAICompatibleProvider MCP runtime injection', () => {
expect(requestParams.tools).toEqual(convertedTools)
})
})

describe('normalizeExtractedImageText', () => {
it('keeps meaningful text after image markdown cleanup', () => {
expect(normalizeExtractedImageText(' Here is the updated image.\n\n')).toBe(
'Here is the updated image.'
)
})

it('drops markdown residue after image markdown cleanup', () => {
expect(normalizeExtractedImageText('`\n')).toBe('')
expect(normalizeExtractedImageText('[]()')).toBe('')
})
})
Loading