From 0eec8282a54adb79bc2101e0bdd4a367c91e54d1 Mon Sep 17 00:00:00 2001 From: zerob13 Date: Thu, 26 Mar 2026 23:30:52 +0800 Subject: [PATCH 1/5] refactor(mcp): retire meeting server --- archives/code/dead-code-batch-3/README.md | 14 + .../inMemoryServers/meetingServer.ts | 0 scripts/generate-i18n-types.js | 4 +- src/main/events.ts | 5 - .../configPresenter/mcpConfHelper.ts | 38 +- .../mcpPresenter/inMemoryServers/builder.ts | 3 - .../settings/components/AcpSettings.vue | 15 +- src/renderer/src/events.ts | 5 - src/renderer/src/i18n/da-DK/mcp.json | 4 - src/renderer/src/i18n/en-US/mcp.json | 4 - src/renderer/src/i18n/fa-IR/mcp.json | 4 - src/renderer/src/i18n/fr-FR/mcp.json | 4 - src/renderer/src/i18n/he-IL/mcp.json | 4 - src/renderer/src/i18n/ja-JP/mcp.json | 4 - src/renderer/src/i18n/ko-KR/mcp.json | 4 - src/renderer/src/i18n/pt-BR/mcp.json | 4 - src/renderer/src/i18n/ru-RU/mcp.json | 4 - src/renderer/src/i18n/zh-CN/mcp.json | 4 - src/renderer/src/i18n/zh-HK/mcp.json | 4 - src/renderer/src/i18n/zh-TW/mcp.json | 4 - src/types/i18n.d.ts | 528 +++++++++++++++++- 21 files changed, 543 insertions(+), 117 deletions(-) create mode 100644 archives/code/dead-code-batch-3/README.md rename {src => archives/code/dead-code-batch-3/src}/main/presenter/mcpPresenter/inMemoryServers/meetingServer.ts (100%) diff --git a/archives/code/dead-code-batch-3/README.md b/archives/code/dead-code-batch-3/README.md new file mode 100644 index 000000000..bea975789 --- /dev/null +++ b/archives/code/dead-code-batch-3/README.md @@ -0,0 +1,14 @@ +# Dead Code Batch 3 + +- Purpose: archive retired MCP runtime code that is no longer part of the active in-memory server set. +- Archived at: 2026-03-26 +- Rationale: `meetingServer.ts` has been removed from live MCP registration and default config, but is retained in source form for precise rollback if the feature is rebuilt later. + +## Archived Paths + +- `src/main/presenter/mcpPresenter/inMemoryServers/meetingServer.ts` + +## Notes + +- This directory is not part of the runtime, build, typecheck, or test target set. +- Restore by moving files back to their original paths only if a future audit proves the retired MCP server is needed again. diff --git a/src/main/presenter/mcpPresenter/inMemoryServers/meetingServer.ts b/archives/code/dead-code-batch-3/src/main/presenter/mcpPresenter/inMemoryServers/meetingServer.ts similarity index 100% rename from src/main/presenter/mcpPresenter/inMemoryServers/meetingServer.ts rename to archives/code/dead-code-batch-3/src/main/presenter/mcpPresenter/inMemoryServers/meetingServer.ts diff --git a/scripts/generate-i18n-types.js b/scripts/generate-i18n-types.js index 9e24b7c3a..5c212e7c7 100644 --- a/scripts/generate-i18n-types.js +++ b/scripts/generate-i18n-types.js @@ -1,6 +1,6 @@ import fs from 'fs' import path from 'path' -import { fileURLToPath } from 'url' +import { fileURLToPath, pathToFileURL } from 'url' const __filename = fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) @@ -57,6 +57,6 @@ async function main() { } // 仅需要在本地开发时执行 -if (import.meta.url === `file://${process.argv[1]}`) { +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { main() } diff --git a/src/main/events.ts b/src/main/events.ts index 38c6ac140..77da1c471 100644 --- a/src/main/events.ts +++ b/src/main/events.ts @@ -219,11 +219,6 @@ export const TRAY_EVENTS = { CHECK_FOR_UPDATES: 'tray:check-for-updates' // 托盘检查更新 } -// MCP会议专用事件 -export const MEETING_EVENTS = { - INSTRUCTION: 'mcp:meeting-instruction' // 主进程向渲染进程发送指令 -} - // 悬浮按钮相关事件 export const FLOATING_BUTTON_EVENTS = { CLICKED: 'floating-button:clicked', // 悬浮按钮被点击 diff --git a/src/main/presenter/configPresenter/mcpConfHelper.ts b/src/main/presenter/configPresenter/mcpConfHelper.ts index b3c5f0a5b..d653133e7 100644 --- a/src/main/presenter/configPresenter/mcpConfHelper.ts +++ b/src/main/presenter/configPresenter/mcpConfHelper.ts @@ -258,16 +258,6 @@ const DEFAULT_INMEMORY_SERVERS: Record> env: {}, disable: false }, - 'deepchat-inmemory/meeting-server': { - args: [], - descriptions: 'DeepChat内置会议服务，用于组织多Agent讨论', - icons: '👥', - autoApprove: ['all'], - type: 'inmemory' as MCPServerType, - command: 'deepchat-inmemory/meeting-server', - env: {}, - disable: false - }, // Merge platform-specific services ...PLATFORM_SPECIFIC_SERVERS } @@ -384,15 +374,31 @@ export class McpConfHelper { private removeDeprecatedBuiltInServers( servers: Record ): Record { - const deprecatedBuiltInServers = ['powerpack'] + const deprecatedBuiltInServers = ['powerpack', 'deepchat-inmemory/meeting-server'] + let hasChanges = false + const removedBuiltInServers = new Set(this.getRemovedBuiltInServers()) + let removedListChanged = false for (const serverName of deprecatedBuiltInServers) { if (servers[serverName]) { console.log(`Removing deprecated built-in MCP service: ${serverName}`) delete servers[serverName] + hasChanges = true + } + + if (removedBuiltInServers.delete(serverName)) { + removedListChanged = true } } + if (hasChanges) { + this.mcpStore.set('mcpServers', servers) + } + + if (removedListChanged) { + this.setRemovedBuiltInServers(Array.from(removedBuiltInServers)) + } + return servers } @@ -913,15 +919,9 @@ export class McpConfHelper { } try { - const mcpServers = this.mcpStore.get('mcpServers') || {} - - if (mcpServers.powerpack) { - console.log('Removing deprecated powerpack MCP server') - delete mcpServers.powerpack - this.mcpStore.set('mcpServers', mcpServers) - } + this.removeDeprecatedBuiltInServers(this.mcpStore.get('mcpServers') || {}) } catch (error) { - console.error('Error occurred while removing deprecated powerpack server:', error) + console.error('Error occurred while removing deprecated built-in MCP servers:', error) } // 升级后检查并添加平台特有服务 diff --git a/src/main/presenter/mcpPresenter/inMemoryServers/builder.ts b/src/main/presenter/mcpPresenter/inMemoryServers/builder.ts index 9a02e88fb..42170938c 100644 --- a/src/main/presenter/mcpPresenter/inMemoryServers/builder.ts +++ b/src/main/presenter/mcpPresenter/inMemoryServers/builder.ts @@ -9,7 +9,6 @@ import { FastGptKnowledgeServer } from './fastGptKnowledgeServer' import { DeepResearchServer } from './deepResearchServer' import { AutoPromptingServer } from './autoPromptingServer' import { ConversationSearchServer } from './conversationSearchServer' -import { MeetingServer } from './meetingServer' import { BuiltinKnowledgeServer } from './builtinKnowledgeServer' import { BuiltinKnowledgeConfig } from '@shared/presenter' import { AppleServer } from './appleServer' @@ -79,8 +78,6 @@ export function getInMemoryServer( return new AutoPromptingServer() case 'deepchat-inmemory/conversation-search-server': return new ConversationSearchServer() - case 'deepchat-inmemory/meeting-server': - return new MeetingServer() case 'deepchat/apple-server': // 只在 macOS 上创建 AppleServer if (process.platform !== 'darwin') { diff --git a/src/renderer/settings/components/AcpSettings.vue b/src/renderer/settings/components/AcpSettings.vue index b90851bb4..80a11e9fe 100644 --- a/src/renderer/settings/components/AcpSettings.vue +++ b/src/renderer/settings/components/AcpSettings.vue @@ -75,14 +75,9 @@ {{ t('settings.acp.installedSectionDescription') }}

-
- - {{ t('settings.acp.installedCount', { count: installedRegistryAgents.length }) }} - - -
+ + {{ t('settings.acp.installedCount', { count: installedRegistryAgents.length }) }} +
{{ t('settings.acp.installedEmptyDescription') }}

-
diff --git a/src/renderer/src/events.ts b/src/renderer/src/events.ts index b59e5fbe4..d5ed808bc 100644 --- a/src/renderer/src/events.ts +++ b/src/renderer/src/events.ts @@ -98,11 +98,6 @@ export const MCP_EVENTS = { SAMPLING_CANCELLED: 'mcp:sampling-cancelled' } -// 新增会议相关事件 -export const MEETING_EVENTS = { - INSTRUCTION: 'mcp:meeting-instruction' // 监听来自主进程的指令 -} - // 同步相关事件 export const SYNC_EVENTS = { BACKUP_STARTED: 'sync:backup-started', diff --git a/src/renderer/src/i18n/da-DK/mcp.json b/src/renderer/src/i18n/da-DK/mcp.json index c51af4ba9..0a1a36fe8 100644 --- a/src/renderer/src/i18n/da-DK/mcp.json +++ b/src/renderer/src/i18n/da-DK/mcp.json @@ -83,10 +83,6 @@ "desc": "DeepChat indbyggede dybdeforskningsservice baseret på Bocha-søgning (bemærk, at det kræver en lang kontekstmodel for at kunne bruges; modeller med utilstrækkelig kontekst kan fejle).", "name": "DeepResearch" }, - "deepchat-inmemory/meeting-server": { - "desc": "DeepChat har en indbygget mødetjeneste, der understøtter igangsættelse og ledelse af diskussioner med flere intelligente agenter.", - "name": "Multi-agent møde" - }, "deepchat/apple-server": { "desc": "Lad modellen kunne betjene macOS' systemfunktioner som kalender, kontakter, mail, kort, noter og påmindelser.", "name": "macOS systemassistent" diff --git a/src/renderer/src/i18n/en-US/mcp.json b/src/renderer/src/i18n/en-US/mcp.json index 5e3728948..cd54de077 100644 --- a/src/renderer/src/i18n/en-US/mcp.json +++ b/src/renderer/src/i18n/en-US/mcp.json @@ -238,10 +238,6 @@ "desc": "DeepChat built-in knowledge base search for DeepChat docs and guides.", "name": "Built-in knowledge base search" }, - "deepchat-inmemory/meeting-server": { - "name": "Multi-Agent Meetings", - "desc": "DeepChat built-in meetings to host multi-agent discussions." - }, "deepchat/apple-server": { "desc": "Let models operate macOS apps like Calendar, Contacts, Mail, Maps, Notes, and Reminders.", "name": "macOS System Assistant" diff --git a/src/renderer/src/i18n/fa-IR/mcp.json b/src/renderer/src/i18n/fa-IR/mcp.json index 6f0a3f922..7538d59b4 100644 --- a/src/renderer/src/i18n/fa-IR/mcp.json +++ b/src/renderer/src/i18n/fa-IR/mcp.json @@ -195,10 +195,6 @@ "name": "جستجوی تاریخچه گفت‌وگو", "desc": "خدمات جستجوی تاریخچه گفت‌وگوی داخلی دیپ‌چت، میتواند در تاریخچه و محتوای پیام‌های گفت‌وگو جستجو کند" }, - "deepchat-inmemory/meeting-server": { - "name": "جلسه چند-عامل", - "desc": "سرویس جلسه داخلی DeepChat امکان میزبانی و مدیریت بحث‌های چند-عامله را فراهم می‌کند." - }, "builtinKnowledge": { "desc": "خدمات جستجوی پایه دانش داخلی Deepchat ، که می تواند محتوای پایگاه دانش داخلی Deepchat را جستجو کند", "name": "جستجوی پایگاه دانش داخلی" diff --git a/src/renderer/src/i18n/fr-FR/mcp.json b/src/renderer/src/i18n/fr-FR/mcp.json index 18b637e1b..f385c01e6 100644 --- a/src/renderer/src/i18n/fr-FR/mcp.json +++ b/src/renderer/src/i18n/fr-FR/mcp.json @@ -195,10 +195,6 @@ "name": "Recherche d’historique", "desc": "Rechercher les conversations et messages passés (intégré à DeepChat)" }, - "deepchat-inmemory/meeting-server": { - "name": "Réunion Multi-Agent", - "desc": "Le service de réunion intégré de DeepChat permet d’organiser et d’animer des discussions multi-agents." - }, "builtinKnowledge": { "desc": "Recherche dans la base de connaissances intégrée de DeepChat (docs, contenus intégrés).", "name": "Base de connaissances intégrée" diff --git a/src/renderer/src/i18n/he-IL/mcp.json b/src/renderer/src/i18n/he-IL/mcp.json index d44e64166..8a55ce13c 100644 --- a/src/renderer/src/i18n/he-IL/mcp.json +++ b/src/renderer/src/i18n/he-IL/mcp.json @@ -238,10 +238,6 @@ "desc": "חיפוש בסיס ידע מובנה של DeepChat עבור מסמכים ומדריכים של DeepChat.", "name": "חיפוש בסיס ידע מובנה" }, - "deepchat-inmemory/meeting-server": { - "name": "פגישות מרובות סוכנים (Multi-Agent)", - "desc": "פגישות מובנות ב-DeepChat לאירוח דיונים מרובי סוכנים." - }, "deepchat/apple-server": { "desc": "אפשר למודלים להפעיל אפליקציות macOS כמו יומן, אנשי קשר, דואר, מפות, פתקים ותזכורות.", "name": "עוזר מערכת macOS" diff --git a/src/renderer/src/i18n/ja-JP/mcp.json b/src/renderer/src/i18n/ja-JP/mcp.json index 510ab1f7a..7132d0025 100644 --- a/src/renderer/src/i18n/ja-JP/mcp.json +++ b/src/renderer/src/i18n/ja-JP/mcp.json @@ -195,10 +195,6 @@ "name": "会話履歴検索", "desc": "DeepChat内蔵の会話履歴検索サービス、過去の会話記録とメッセージ内容を検索できます" }, - "deepchat-inmemory/meeting-server": { - "name": "マルチエージェント会議", - "desc": "DeepChatの内蔵会議サービスは、マルチエージェントによる討論の開催と進行を可能にします。" - }, "builtinKnowledge": { "desc": "deepchatビルトインナレッジベース検索サービス。これは、deepchatビルトインナレッジベースのコンテンツを検索できる", "name": "組み込みのナレッジベース検索" diff --git a/src/renderer/src/i18n/ko-KR/mcp.json b/src/renderer/src/i18n/ko-KR/mcp.json index 2dd53254c..4d4ffa04d 100644 --- a/src/renderer/src/i18n/ko-KR/mcp.json +++ b/src/renderer/src/i18n/ko-KR/mcp.json @@ -195,10 +195,6 @@ "name": "대화 기록 검색", "desc": "DeepChat 내장 대화 기록 검색 서비스, 과거 대화 기록과 메시지 내용을 검색할 수 있습니다" }, - "deepchat-inmemory/meeting-server": { - "name": "멀티 에이전트 회의", - "desc": "DeepChat의 내장 회의 서비스는 다중 에이전트 토론의 주최와 진행을 지원합니다." - }, "builtinKnowledge": { "desc": "Deepchat 내장 지식 기반 검색 서비스, Deepchat 내장 지식 기반의 내용을 검색 할 수있는 Deepchat 내장 지식 기반 검색 서비스", "name": "내장 된 지식 기반 검색" diff --git a/src/renderer/src/i18n/pt-BR/mcp.json b/src/renderer/src/i18n/pt-BR/mcp.json index dab43a80c..9720d488d 100644 --- a/src/renderer/src/i18n/pt-BR/mcp.json +++ b/src/renderer/src/i18n/pt-BR/mcp.json @@ -199,10 +199,6 @@ "desc": "Pesquisa na base de conhecimento integrada do DeepChat para documentação e guias.", "name": "Pesquisa na base de conhecimento integrada" }, - "deepchat-inmemory/meeting-server": { - "name": "Reuniões Multi-Agente", - "desc": "Reuniões integradas ao DeepChat para hospedar discussões multi-agente." - }, "deepchat/apple-server": { "desc": "Permite que os modelos operem aplicativos do macOS como Calendário, Contatos, Mail, Mapas, Notas e Lembretes.", "name": "Assistente do Sistema macOS" diff --git a/src/renderer/src/i18n/ru-RU/mcp.json b/src/renderer/src/i18n/ru-RU/mcp.json index fe36cf186..505bd241c 100644 --- a/src/renderer/src/i18n/ru-RU/mcp.json +++ b/src/renderer/src/i18n/ru-RU/mcp.json @@ -195,10 +195,6 @@ "name": "Поиск истории разговоров", "desc": "Встроенная служба поиска истории разговоров DeepChat, может искать записи исторических разговоров и содержимое сообщений" }, - "deepchat-inmemory/meeting-server": { - "name": "Мультиагентная встреча", - "desc": "Встроенный сервис встреч DeepChat позволяет организовывать и проводить обсуждения между несколькими агентами." - }, "builtinKnowledge": { "desc": "Встроенная служба поиска базы знаний DeepChat, которая может искать содержание встроенной базы знаний DeepChat", "name": "Встроенный поиск базы знаний" diff --git a/src/renderer/src/i18n/zh-CN/mcp.json b/src/renderer/src/i18n/zh-CN/mcp.json index 5b554cd07..1f082ea67 100644 --- a/src/renderer/src/i18n/zh-CN/mcp.json +++ b/src/renderer/src/i18n/zh-CN/mcp.json @@ -223,10 +223,6 @@ "name": "对话历史搜索", "desc": "DeepChat内置对话历史搜索服务，可搜索历史对话记录和消息内容" }, - "deepchat-inmemory/meeting-server": { - "name": "多智能体会议", - "desc": "DeepChat 内置会议服务，支持发起和主持多智能体讨论" - }, "deepchat/apple-server": { "name": "macOS系统助手", "desc": "让模型能操作macOS的日历、联系人、邮件、地图、备忘录、提醒事项等系统功能" diff --git a/src/renderer/src/i18n/zh-HK/mcp.json b/src/renderer/src/i18n/zh-HK/mcp.json index df4825ec7..2bdf38fad 100644 --- a/src/renderer/src/i18n/zh-HK/mcp.json +++ b/src/renderer/src/i18n/zh-HK/mcp.json @@ -195,10 +195,6 @@ "name": "對話歷史搜尋", "desc": "DeepChat內建對話歷史搜尋服務，可搜尋歷史對話記錄和訊息內容" }, - "deepchat-inmemory/meeting-server": { - "name": "多智能體會議", - "desc": "DeepChat 內置的會議功能支援舉辦和主持多智能體討論。" - }, "builtinKnowledge": { "desc": "DeepChat內置知識庫檢索服務，可以對DeepChat內置知識庫內容進行檢索", "name": "內置知識庫檢索" diff --git a/src/renderer/src/i18n/zh-TW/mcp.json b/src/renderer/src/i18n/zh-TW/mcp.json index 13ab88526..b802c2496 100644 --- a/src/renderer/src/i18n/zh-TW/mcp.json +++ b/src/renderer/src/i18n/zh-TW/mcp.json @@ -219,10 +219,6 @@ "name": "對話歷史搜索", "desc": "DeepChat內建對話歷史搜索服務，可搜索歷史對話記錄和訊息內容" }, - "deepchat-inmemory/meeting-server": { - "name": "多智能體會議", - "desc": "DeepChat 內建的會議服務可用於發起與主持多智能體討論。" - }, "builtinKnowledge": { "desc": "DeepChat內置知識庫檢索服務，可以對DeepChat內置知識庫內容進行檢索", "name": "內置知識庫檢索" diff --git a/src/types/i18n.d.ts b/src/types/i18n.d.ts index 76523d9f3..268d6012c 100644 --- a/src/types/i18n.d.ts +++ b/src/types/i18n.d.ts @@ -87,6 +87,8 @@ declare module 'vue-i18n' { functionSwitch: string attach: string voiceInput: string + queue: string + stop: string fileSelect: string pasteFiles: string dropFiles: string @@ -115,6 +117,33 @@ declare module 'vue-i18n' { empty: string openSettings: string } + tools: { + badge: string + title: string + mcpSection: string + builtinSection: string + loading: string + builtinEmpty: string + groups: { + agentFilesystem: string + agentCore: string + agentSkills: string + deepchatSettings: string + yobrowser: string + } + } + } + pendingInput: { + steer: string + queueCount: string + resumeQueue: string + toSteer: string + locked: string + reorder: string + files: string + attachmentsOnly: string + empty: string + limitReached: string } features: { webSearch: string @@ -177,23 +206,6 @@ declare module 'vue-i18n' { generationComplete: string generationError: string } - navigation: { - title: string - searchPlaceholder: string - noResults: string - noMessages: string - totalMessages: string - searchResults: string - userMessage: string - assistantMessage: string - unknownMessage: string - } - mcpUi: { - title: string - badge: string - expand: string - collapse: string - } toolCall: { title: string calling: string @@ -225,9 +237,23 @@ declare module 'vue-i18n' { maxTokens: string thinkingBudget: string verbosity: string + forceInterleavedThinkingCompat: string + forceInterleavedThinkingCompatDescription: string verbosityPlaceholder: string currentCustomPrompt: string useDefault: string + decreaseValue: string + increaseValue: string + toggleValue: string + validation: { + finiteNumber: string + nonNegativeInteger: string + contextLengthAtLeastMaxTokens: string + maxTokensWithinContextLength: string + } + } + modelPicker: { + empty: string } audio: { play: string @@ -286,6 +312,12 @@ declare module 'vue-i18n' { successMessage: string failed: string } + card: { + scripts: string + env: string + pythonShort: string + nodeShort: string + } edit: { title: string placeholder: string @@ -301,6 +333,23 @@ declare module 'vue-i18n' { allowedToolsPlaceholder: string allowedToolsHint: string content: string + runtimeTitle: string + runtimeHint: string + pythonRuntime: string + nodeRuntime: string + envTitle: string + envWarning: string + scriptsTitle: string + scriptsHint: string + noScripts: string + scriptEnabled: string + scriptDescription: string + scriptDescriptionPlaceholder: string + runtime: { + auto: string + system: string + builtin: string + } files: string noFiles: string } @@ -403,6 +452,11 @@ declare module 'vue-i18n' { workspace: { title: string collapse: string + sections: { + files: string + git: string + artifacts: string + } plan: { section: string empty: string @@ -424,6 +478,12 @@ declare module 'vue-i18n' { insertPath: string } } + git: { + empty: string + clean: string + staged: string + unstaged: string + } browser: { section: string empty: string @@ -450,6 +510,45 @@ declare module 'vue-i18n' { } } } + newThread: { + title: string + } + floatingWidget: { + title: string + collapse: string + empty: string + executing: string + sessionCount: string + untitled: string + status: { + inProgress: string + done: string + error: string + } + } + permissionMode: { + default: string + fullAccess: string + } + sidebar: { + allAgents: string + expandSidebar: string + collapseSidebar: string + remoteControlDisabled: string + remoteControlStatus: { + disabled: string + stopped: string + starting: string + running: string + backoff: string + error: string + } + groupByDate: string + groupByProject: string + pinned: string + emptyTitle: string + emptyDescription: string + } loading: string copied: string paste: string @@ -519,11 +618,27 @@ declare module 'vue-i18n' { back: string forward: string reload: string + addressLabel: string addressPlaceholder: string enterUrlToStart: string enterUrlDescription: string name: string } + size: { + bytes: string + } + time: { + today: string + yesterday: string + lastWeek: string + older: string + } + project: { + select: string + none: string + recent: string + openFolder: string + } emojiPicker: { search: string smileys: string @@ -853,10 +968,6 @@ declare module 'vue-i18n' { name: string desc: string } - 'deepchat-inmemory/meeting-server': { - name: string - desc: string - } 'deepchat/apple-server': { name: string desc: string @@ -966,8 +1077,6 @@ declare module 'vue-i18n' { writing: string analysis: string } - greeting: string - prompt: string completed: string addTitle: string addDescription: string @@ -1061,6 +1170,7 @@ declare module 'vue-i18n' { 'settings-common': string 'settings-provider': string 'settings-mcp': string + 'settings-deepchat-agents': string 'settings-database': string 'settings-about': string 'settings-shortcut': string @@ -1071,6 +1181,9 @@ declare module 'vue-i18n' { 'settings-acp': string 'settings-skills': string 'settings-notifications-hooks': string + 'settings-dashboard': string + 'settings-environments': string + 'settings-remote': string common: { title: string resetData: string @@ -1080,6 +1193,18 @@ declare module 'vue-i18n' { searchEngineSelect: string searchPreview: string autoScrollEnabled: string + autoCompaction: { + title: string + enabled: string + description: string + thresholdLabel: string + thresholdMin: string + thresholdMax: string + thresholdDescription: string + retainPairsLabel: string + retainPairsValue: string + retainPairsDescription: string + } searchAssistantModel: string selectModel: string proxyMode: string @@ -1136,6 +1261,57 @@ declare module 'vue-i18n' { visionModel: string } } + deepchatAgents: { + title: string + description: string + builtIn: string + editTitle: string + createTitle: string + unnamed: string + name: string + namePlaceholder: string + enabledLabel: string + descriptionLabel: string + descriptionPlaceholder: string + avatarTitle: string + avatarDefault: string + avatarDefaultDesc: string + avatarLucide: string + avatarLucideDesc: string + avatarMonogram: string + avatarMonogramDesc: string + lucideIcon: string + lightColor: string + darkColor: string + monogramText: string + monogramPlaceholder: string + backgroundColor: string + modelsTitle: string + chatModel: string + assistantModel: string + visionModel: string + temperature: string + contextLength: string + maxTokens: string + thinkingBudget: string + reasoningEffort: string + verbosity: string + interleaved: string + systemPrompt: string + systemPromptPlaceholder: string + defaultProjectPath: string + defaultProjectPathPlaceholder: string + permissionMode: string + permissionFullAccess: string + permissionDefault: string + toolsTitle: string + compactionTitle: string + compactionEnabled: string + compactionDescription: string + compactionThreshold: string + compactionRetainPairs: string + deleteConfirm: string + } notificationsHooks: { title: string description: string @@ -1245,6 +1421,109 @@ declare module 'vue-i18n' { clearFailedDescription: string } } + dashboard: { + badge: string + title: string + description: string + actions: { + refresh: string + } + backfill: { + runningTitle: string + runningDescription: string + failedTitle: string + failedDescription: string + } + error: { + title: string + description: string + } + empty: { + title: string + description: string + historyNote: string + } + summary: { + totalTokens: string + totalTokensDescription: string + inputTokensLabel: string + outputTokensLabel: string + cachedTokens: string + cachedTokensDescription: string + cachedTokensCachedLabel: string + cachedTokensUncachedLabel: string + cacheHitRate: string + cacheHitRateDescription: string + estimatedCost: string + estimatedCostDescription: string + estimatedCostTrendLabel: string + estimatedCostTrendEmpty: string + recordingStartedAt: string + recordingStartedAtDescription: string + withDeepChatDaysLabel: string + withDeepChatDaysValue: string + withDeepChatDaysSentence: string + withDeepChatDaysDescription: string + withDeepChatDaysDescriptionUnavailable: string + tokenUsage: string + nostalgiaLabel: string + nostalgiaDaysValue: string + nostalgiaSessionsValue: string + nostalgiaMessagesValue: string + nostalgiaDaysDetailLabel: string + nostalgiaDaysDetail: string + nostalgiaSessionsDetailLabel: string + nostalgiaSessionsDetail: string + nostalgiaMessagesDetailLabel: string + nostalgiaMessagesDetail: string + nostalgiaMostActiveDayLabel: string + nostalgiaMostActiveDayDetail: string + } + calendar: { + title: string + description: string + legend: string + tooltip: string + } + breakdown: { + providerTitle: string + providerDescription: string + modelTitle: string + modelDescription: string + messages: string + empty: string + } + rtk: { + title: string + description: string + actions: { + retry: string + } + status: { + disabled: string + checking: string + healthy: string + unhealthy: string + } + descriptionDisabled: string + descriptionChecking: string + descriptionHealthy: string + descriptionUnhealthy: string + sourceLabel: string + source: { + bundled: string + system: string + none: string + } + summary: { + savedTokens: string + commands: string + avgSavingsPct: string + outputTokens: string + } + } + unavailable: string + } model: string provider: { select: string @@ -1607,6 +1886,8 @@ declare module 'vue-i18n' { useBuiltinRuntimeTitle: string useBuiltinRuntimeDescription: string enableToAccess: string + registryInstallEntry: string + registryInstallEntryDescription: string addCustomAgent: string customEmpty: string customDeleteConfirm: string @@ -1624,6 +1905,43 @@ declare module 'vue-i18n' { mcpAccessTitle: string mcpAccessEmpty: string mcpAccessBadge: string + sharedMcpTitle: string + sharedMcpDescription: string + installedSectionTitle: string + installedSectionDescription: string + installedCount: string + installedEmptyTitle: string + installedEmptyDescription: string + registryCount: string + registryEmpty: string + registryOverlayEmpty: string + registryRefresh: string + registryRepair: string + registryInstallAction: string + registryInstallTitle: string + registryInstallDescription: string + registryLearnMore: string + registryRepository: string + registrySearchPlaceholder: string + installFilters: { + all: string + installed: string + notInstalled: string + } + filters: { + all: string + enabled: string + installed: string + attention: string + } + envOverrideTitle: string + envOverridePlaceholder: string + installState: { + installed: string + installing: string + error: string + notInstalled: string + } loading: string none: string saveSuccess: string @@ -1699,6 +2017,9 @@ declare module 'vue-i18n' { title: string description: string entry: string + healthCheck: string + healthChecking: string + healthCheckFailed: string workdirPlaceholder: string close: string customMethod: string @@ -1743,6 +2064,138 @@ declare module 'vue-i18n' { resetToDefaultSuccess: string resetToDefaultFailed: string } + environments: { + title: string + description: string + default: { + title: string + description: string + empty: string + } + history: { + title: string + description: string + } + temp: { + title: string + description: string + } + actions: { + refresh: string + showMissing: string + open: string + setDefault: string + clearDefault: string + showTemp: string + hideTemp: string + } + badges: { + default: string + temp: string + missing: string + notInHistory: string + } + meta: { + sessions: string + lastUsed: string + never: string + } + empty: { + regular: string + temp: string + } + errors: { + openTitle: string + } + } + remote: { + title: string + description: string + telegram: { + title: string + description: string + botToken: string + botTokenPlaceholder: string + } + feishu: { + title: string + description: string + appId: string + appIdPlaceholder: string + appSecret: string + appSecretPlaceholder: string + verificationToken: string + verificationTokenPlaceholder: string + encryptKey: string + encryptKeyPlaceholder: string + botUser: string + bindings: string + pairedUserOpenIds: string + pairedUserOpenIdsPlaceholder: string + accessRulesDescription: string + accessRule1: string + accessRule2: string + } + sections: { + credentials: string + remoteControl: string + notifications: string + accessRules: string + } + remoteControl: { + description: string + allowedUserIds: string + allowedUserIdsPlaceholder: string + defaultAgent: string + defaultAgentPlaceholder: string + openPairDialog: string + manageBindings: string + pairCode: string + noPairCode: string + pairCodeExpiresAt: string + pairDialogTitle: string + pairDialogDescription: string + pairDialogInstructionTelegram: string + pairDialogInstructionFeishu: string + bindingsDialogTitle: string + bindingsDialogDescription: string + bindingsEmpty: string + pairingSuccessTitle: string + pairingSuccessDescription: string + } + hooks: { + title: string + description: string + chatId: string + chatIdPlaceholder: string + threadId: string + threadIdPlaceholder: string + } + status: { + title: string + botUser: string + bindings: string + states: { + disabled: string + stopped: string + starting: string + running: string + backoff: string + error: string + } + } + overview: { + telegram: string + feishu: string + hooksOn: string + hooksOff: string + } + bindingKinds: { + dm: string + group: string + topic: string + } + } success: string message: { toolbar: { @@ -1774,6 +2227,9 @@ declare module 'vue-i18n' { params: string responseData: string terminalOutput: string + badge: { + rtk: string + } replacementsCount: string fileOperation: string fileRead: string @@ -1798,6 +2254,10 @@ declare module 'vue-i18n' { checkUpdate: string downloading: string installNow: string + installUpdate: string + versionAvailable: string + autoUpdateFailed: string + topbarButton: string autoUpdate: string restarting: string alreadyUpToDate: string @@ -1829,5 +2289,27 @@ declare module 'vue-i18n' { next: string back: string } + page: { + title: string + description: string + browseProviders: string + connectAgent: string + acpTitle: string + acpDescription: string + providers: { + claude: string + openai: string + deepseek: string + gemini: string + ollama: string + openrouter: string + } + } + agentPage: { + title: string + deepchatType: string + acpType: string + manageAgents: string + } } } From a82cace2bba06a3d326e41e389a6c4527fa85a2f Mon Sep 17 00:00:00 2001 From: zerob13 Date: Thu, 26 Mar 2026 23:46:05 +0800 Subject: [PATCH 2/5] fix(agent): use session vision for screenshots --- .../deepchatAgentPresenter/dispatch.ts | 14 ++ .../presenter/deepchatAgentPresenter/index.ts | 189 +++++++++++++++++- .../presenter/deepchatAgentPresenter/types.ts | 10 +- .../deepchatAgentPresenter.test.ts | 134 ++++++++++++- .../deepchatAgentPresenter/dispatch.test.ts | 62 ++++++ 5 files changed, 400 insertions(+), 9 deletions(-) diff --git a/src/main/presenter/deepchatAgentPresenter/dispatch.ts b/src/main/presenter/deepchatAgentPresenter/dispatch.ts index 6874379db..37fdd809e 100644 --- a/src/main/presenter/deepchatAgentPresenter/dispatch.ts +++ b/src/main/presenter/deepchatAgentPresenter/dispatch.ts @@ -679,6 +679,20 @@ export async function executeTools( } } + if (hooks?.normalizeToolResult) { + toolRawData = { + ...toolRawData, + content: await hooks.normalizeToolResult({ + sessionId: io.sessionId, + toolCallId: tc.id, + toolName: tc.name, + toolArgs: tc.arguments, + content: toolRawData.content, + isError: toolRawData.isError === true + }) + } + } + const searchPayload = extractSearchPayload( toolRawData.content, toolContext.name, diff --git a/src/main/presenter/deepchatAgentPresenter/index.ts b/src/main/presenter/deepchatAgentPresenter/index.ts index 6971a592e..b95756c1b 100644 --- a/src/main/presenter/deepchatAgentPresenter/index.ts +++ b/src/main/presenter/deepchatAgentPresenter/index.ts @@ -1424,7 +1424,16 @@ export class DeepChatAgentPresenter implements IAgentImplementation { body: gap } }) - } + }, + normalizeToolResult: async (tool) => + await this.normalizeToolResultContent({ + sessionId: tool.sessionId, + toolCallId: tool.toolCallId, + toolName: tool.toolName, + toolArgs: tool.toolArgs, + content: tool.content, + isError: tool.isError + }) }, io: { sessionId, @@ -2867,7 +2876,15 @@ export class DeepChatAgentPresenter implements IAgentImplementation { permissionRequest: rawData.permissionRequest as PendingToolInteraction['permission'] } } - const responseText = this.toolContentToText(rawData.content) + const normalizedContent = await this.normalizeToolResultContent({ + sessionId, + toolCallId: toolCall.id || '', + toolName, + toolArgs: toolCall.params || '{}', + content: rawData.content, + isError: rawData.isError === true + }) + const responseText = this.toolContentToText(normalizedContent) const prepared = await this.toolOutputGuard.prepareToolOutput({ sessionId, toolCallId: toolCall.id || '', @@ -2956,6 +2973,174 @@ export class DeepChatAgentPresenter implements IAgentImplementation { }) } + private async normalizeToolResultContent(params: { + sessionId: string + toolCallId: string + toolName: string + toolArgs: string + content: MCPToolResponse['content'] + isError: boolean + }): Promise { + if (params.isError) { + return params.content + } + + const screenshotPayload = this.extractScreenshotToolPayload( + params.toolName, + params.toolArgs, + params.content + ) + if (!screenshotPayload) { + return params.content + } + + const visionModel = await this.resolveScreenshotVisionModel(params.sessionId) + if (!visionModel) { + return 'Screenshot captured, but automatic English analysis is unavailable because no vision model is configured.' + } + + try { + const messages: ChatMessage[] = [ + { + role: 'user', + content: [ + { + type: 'text', + text: this.buildScreenshotAnalysisPrompt() + }, + { + type: 'image_url', + image_url: { + url: screenshotPayload.dataUrl, + detail: 'auto' + } + } + ] + } + ] + + const modelConfig = this.configPresenter.getModelConfig( + visionModel.modelId, + visionModel.providerId + ) + const response = await this.llmProviderPresenter.generateCompletionStandalone( + visionModel.providerId, + messages, + visionModel.modelId, + modelConfig?.temperature ?? 0.2, + Math.min(modelConfig?.maxTokens ?? 900, 900) + ) + const normalized = response.trim() + if (!normalized) { + return 'Screenshot captured, but automatic English analysis returned no usable description.' + } + return normalized + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + console.warn('[DeepChatAgent] Failed to normalize screenshot tool output:', { + sessionId: params.sessionId, + toolCallId: params.toolCallId, + error: message + }) + return `Screenshot captured, but automatic English analysis failed: ${message}` + } + } + + private extractScreenshotToolPayload( + toolName: string, + toolArgs: string, + content: MCPToolResponse['content'] + ): { dataUrl: string } | null { + if (toolName !== 'cdp_send' || typeof content !== 'string') { + return null + } + + const parsedArgs = this.parseJsonRecord(toolArgs) + if (!parsedArgs || parsedArgs.method !== 'Page.captureScreenshot') { + return null + } + + const parsedContent = this.parseJsonRecord(content) + const rawData = typeof parsedContent?.data === 'string' ? parsedContent.data.trim() : '' + if (!rawData) { + return null + } + + const screenshotParams = this.normalizeJsonRecord(parsedArgs.params) + const mimeType = this.resolveScreenshotMimeType(screenshotParams?.format) + const dataUrl = rawData.startsWith('data:image/') + ? rawData + : `data:${mimeType};base64,${rawData}` + + return { dataUrl } + } + + private normalizeJsonRecord(value: unknown): Record | null { + if (typeof value === 'object' && value !== null && !Array.isArray(value)) { + return value as Record + } + + if (typeof value !== 'string' || !value.trim()) { + return null + } + + return this.parseJsonRecord(value) + } + + private parseJsonRecord(value: string): Record | null { + try { + const parsed = JSON.parse(value) as unknown + if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) { + return parsed as Record + } + } catch {} + + return null + } + + private resolveScreenshotMimeType(format: unknown): string { + if (format === 'jpeg') { + return 'image/jpeg' + } + if (format === 'webp') { + return 'image/webp' + } + return 'image/png' + } + + private async resolveScreenshotVisionModel( + sessionId: string + ): Promise<{ providerId: string; modelId: string } | null> { + const agentId = this.getSessionAgentId(sessionId) ?? 'deepchat' + + try { + const agentConfig = await this.configPresenter.resolveDeepChatAgentConfig(agentId) + const providerId = agentConfig.visionModel?.providerId?.trim() + const modelId = agentConfig.visionModel?.modelId?.trim() + if (providerId && modelId) { + return { providerId, modelId } + } + } catch (error) { + console.warn('[DeepChatAgent] Failed to resolve agent vision model:', { + sessionId, + agentId, + error + }) + } + + return null + } + + private buildScreenshotAnalysisPrompt(): string { + return [ + 'Analyze this browser screenshot and respond in English only.', + 'Describe only what is clearly visible.', + 'Include the page type or layout, the most important visible text, interactive controls, status indicators, warnings, errors, and any detail that matters for the next browser action.', + 'Do not speculate about hidden or unreadable content.', + 'Return concise plain text in a single short paragraph.' + ].join('\n') + } + private toolContentToText(content: MCPToolResponse['content']): string { if (typeof content === 'string') { return content diff --git a/src/main/presenter/deepchatAgentPresenter/types.ts b/src/main/presenter/deepchatAgentPresenter/types.ts index 785ae76e1..df53d0936 100644 --- a/src/main/presenter/deepchatAgentPresenter/types.ts +++ b/src/main/presenter/deepchatAgentPresenter/types.ts @@ -6,7 +6,7 @@ import type { } from '@shared/types/agent-interface' import type { LLMCoreStreamEvent } from '@shared/types/core/llm-events' import type { ChatMessage } from '@shared/types/core/chat-message' -import type { MCPToolDefinition } from '@shared/types/core/mcp' +import type { MCPToolDefinition, MCPToolResponse } from '@shared/types/core/mcp' import type { ModelConfig } from '@shared/presenter' import type { IToolPresenter } from '@shared/types/presenters/tool.presenter' import type { DeepChatMessageStore } from './messageStore' @@ -76,6 +76,14 @@ export interface ProcessHooks { reasoningContentLength: number toolCallCount: number }) => void + normalizeToolResult?: (tool: { + sessionId: string + toolCallId: string + toolName: string + toolArgs: string + content: MCPToolResponse['content'] + isError: boolean + }) => Promise } export interface PendingToolInteraction { diff --git a/test/main/presenter/deepchatAgentPresenter/deepchatAgentPresenter.test.ts b/test/main/presenter/deepchatAgentPresenter/deepchatAgentPresenter.test.ts index 10b55624a..d75e6e26c 100644 --- a/test/main/presenter/deepchatAgentPresenter/deepchatAgentPresenter.test.ts +++ b/test/main/presenter/deepchatAgentPresenter/deepchatAgentPresenter.test.ts @@ -175,6 +175,7 @@ function createMockLlmProviderPresenter() { getProviderInstance: vi.fn().mockReturnValue({ coreStream: vi.fn().mockReturnValue(createMockCoreStream()()) }), + generateCompletionStandalone: vi.fn().mockResolvedValue('English screenshot summary'), generateText: vi.fn().mockResolvedValue({ content: ['## Current Goal', '- Continue the session safely'].join('\n') }) @@ -223,7 +224,9 @@ function createMockConfigPresenter() { getAutoCompactionEnabled: vi.fn().mockReturnValue(true), getAutoCompactionTriggerThreshold: vi.fn().mockReturnValue(80), getAutoCompactionRetainRecentPairs: vi.fn().mockReturnValue(2), - getSetting: vi.fn().mockReturnValue(undefined) + getSetting: vi.fn().mockReturnValue(undefined), + getDefaultVisionModel: vi.fn().mockReturnValue(undefined), + resolveDeepChatAgentConfig: vi.fn().mockResolvedValue({}) } as any } @@ -2472,11 +2475,14 @@ describe('DeepChatAgentPresenter', () => { expect(updatedBlocks[1].extra.needsUserAction).toBe(false) }) - it('offloads deferred tool results before resume', async () => { + it('normalizes deferred screenshot tool results before resume', async () => { tempHome = await fs.mkdtemp(path.join(os.tmpdir(), 'deepchat-deferred-offload-')) getPathSpy = vi.spyOn(app, 'getPath').mockReturnValue(tempHome) await agent.initSession('s1', { providerId: 'openai', modelId: 'gpt-4' }) + configPresenter.resolveDeepChatAgentConfig.mockResolvedValueOnce({ + visionModel: { providerId: 'anthropic', modelId: 'claude-3-7-sonnet' } + }) makeAssistantRow({ blocks: [ { @@ -2539,10 +2545,8 @@ describe('DeepChatAgentPresenter', () => { const updatedBlocks = JSON.parse( sqlitePresenter.deepchatMessagesTable.updateContent.mock.calls[0][1] ) - expect(updatedBlocks[0].tool_call.response).toContain('[Tool output offloaded]') - expect(updatedBlocks[0].tool_call.response).toContain('tool_tc1') - expect(updatedBlocks[0].tool_call.response).toContain('.offload') - expect(updatedBlocks[0].tool_call.response).not.toContain(tempHome!) + expect(updatedBlocks[0].tool_call.response).toBe('English screenshot summary') + expect(updatedBlocks[0].tool_call.response).not.toContain('[Tool output offloaded]') expect(updatedBlocks[0].status).toBe('success') expect(processStream).toHaveBeenCalledTimes(1) }) @@ -2781,5 +2785,123 @@ describe('DeepChatAgentPresenter', () => { }) ) }) + + it('rewrites screenshot tool output with the agent vision model during deferred execution', async () => { + toolPresenter.getAllToolDefinitions.mockResolvedValueOnce([ + { + type: 'function', + function: { + name: 'cdp_send', + description: 'CDP tool', + parameters: { type: 'object', properties: {} } + }, + server: { name: 'yobrowser', icons: '', description: '' } + } + ]) + toolPresenter.callTool.mockResolvedValueOnce({ + content: '{"data":"YWJj"}', + rawData: { toolCallId: 'tc1', content: '{"data":"YWJj"}', isError: false } + }) + configPresenter.resolveDeepChatAgentConfig.mockResolvedValueOnce({ + visionModel: { providerId: 'anthropic', modelId: 'claude-3-7-sonnet' } + }) + + await agent.initSession('s1', { + agentId: 'agent-vision', + providerId: 'openai', + modelId: 'gpt-4' + }) + + const result = await (agent as any).executeDeferredToolCall('s1', { + id: 'tc1', + name: 'cdp_send', + params: '{"method":"Page.captureScreenshot","params":{"format":"jpeg"}}' + }) + + expect(llmProvider.generateCompletionStandalone).toHaveBeenCalledWith( + 'anthropic', + [ + { + role: 'user', + content: [ + expect.objectContaining({ + type: 'text' + }), + { + type: 'image_url', + image_url: { + url: 'data:image/jpeg;base64,YWJj', + detail: 'auto' + } + } + ] + } + ], + 'claude-3-7-sonnet', + expect.any(Number), + expect.any(Number) + ) + expect(result).toEqual( + expect.objectContaining({ + isError: false, + responseText: 'English screenshot summary' + }) + ) + }) + + it('uses the current session agent to resolve the vision model', async () => { + sqlitePresenter.newSessionsTable.get.mockReturnValue({ + id: 's1', + agent_id: 'persisted-agent' + }) + configPresenter.resolveDeepChatAgentConfig.mockResolvedValueOnce({ + visionModel: { providerId: 'google', modelId: 'gemini-2.5-flash' } + }) + + await agent.initSession('s1', { providerId: 'openai', modelId: 'gpt-4' }) + + const normalized = await (agent as any).normalizeToolResultContent({ + sessionId: 's1', + toolCallId: 'tc1', + toolName: 'cdp_send', + toolArgs: '{"method":"Page.captureScreenshot"}', + content: '{"data":"YWJj"}', + isError: false + }) + + expect(configPresenter.resolveDeepChatAgentConfig).toHaveBeenCalledWith('persisted-agent') + expect(llmProvider.generateCompletionStandalone).toHaveBeenCalledWith( + 'google', + expect.any(Array), + 'gemini-2.5-flash', + expect.any(Number), + expect.any(Number) + ) + expect(normalized).toBe('English screenshot summary') + }) + + it('returns a readable error when the current session agent has no vision model', async () => { + configPresenter.resolveDeepChatAgentConfig.mockResolvedValueOnce({}) + configPresenter.getDefaultVisionModel.mockReturnValueOnce({ + providerId: 'openai', + modelId: 'gpt-4o' + }) + + await agent.initSession('s1', { providerId: 'openai', modelId: 'gpt-4' }) + + const normalized = await (agent as any).normalizeToolResultContent({ + sessionId: 's1', + toolCallId: 'tc1', + toolName: 'cdp_send', + toolArgs: '{"method":"Page.captureScreenshot"}', + content: '{"data":"YWJj"}', + isError: false + }) + + expect(normalized).toContain('no vision model is configured') + expect(normalized).not.toContain('YWJj') + expect(configPresenter.getDefaultVisionModel).not.toHaveBeenCalled() + expect(llmProvider.generateCompletionStandalone).not.toHaveBeenCalled() + }) }) }) diff --git a/test/main/presenter/deepchatAgentPresenter/dispatch.test.ts b/test/main/presenter/deepchatAgentPresenter/dispatch.test.ts index 6f3e589f9..78c18ff88 100644 --- a/test/main/presenter/deepchatAgentPresenter/dispatch.test.ts +++ b/test/main/presenter/deepchatAgentPresenter/dispatch.test.ts @@ -709,6 +709,68 @@ describe('dispatch', () => { expect(state.blocks[0].status).toBe('success') }) + it('normalizes tool output before offload when a hook rewrites screenshot content', async () => { + const tools = [makeTool('cdp_send')] + const longScreenshot = JSON.stringify({ data: 'x'.repeat(7000) }) + const toolPresenter = createMockToolPresenter({ cdp_send: longScreenshot }) + const conversation: any[] = [] + const hooks = { + normalizeToolResult: vi.fn().mockResolvedValue('English screenshot summary') + } + + state.blocks.push({ + type: 'tool_call', + content: '', + status: 'pending', + timestamp: Date.now(), + tool_call: { + id: 'tc-normalized', + name: 'cdp_send', + params: '{"method":"Page.captureScreenshot"}', + response: '' + } + }) + state.completedToolCalls = [ + { + id: 'tc-normalized', + name: 'cdp_send', + arguments: '{"method":"Page.captureScreenshot"}' + } + ] + + const executed = await executeTools( + state, + conversation, + 0, + tools, + toolPresenter, + 'gpt-4', + io, + 'full_access', + new ToolOutputGuard(), + 32000, + 1024, + hooks, + 'openai' + ) + + expect(executed.terminalError).toBeUndefined() + expect(hooks.normalizeToolResult).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: 's1', + toolCallId: 'tc-normalized', + toolName: 'cdp_send', + toolArgs: '{"method":"Page.captureScreenshot"}', + content: longScreenshot, + isError: false + }) + ) + const toolMessage = conversation.find((message: any) => message.role === 'tool') + expect(toolMessage.content).toBe('English screenshot summary') + expect(toolMessage.content).not.toContain('[Tool output offloaded]') + expect(state.blocks[0].tool_call?.response).toBe('English screenshot summary') + }) + it('turns offload write failures into tool errors instead of falling back to raw content', async () => { tempHome = await fs.mkdtemp(path.join(os.tmpdir(), 'deepchat-dispatch-offload-fail-')) getPathSpy = vi.spyOn(app, 'getPath').mockReturnValue(tempHome) From 31af7b37ec800feb9eb8694f3e630e00ec3ac6c6 Mon Sep 17 00:00:00 2001 From: zerob13 Date: Fri, 27 Mar 2026 00:31:20 +0800 Subject: [PATCH 3/5] refactor(vision): unify session image analysis --- src/main/presenter/configPresenter/index.ts | 65 ++- .../configPresenter/mcpConfHelper.ts | 16 +- .../presenter/deepchatAgentPresenter/index.ts | 35 +- src/main/presenter/index.ts | 20 + .../mcpPresenter/inMemoryServers/builder.ts | 5 +- .../inMemoryServers/imageServer.ts | 479 ------------------ .../agentTools/agentToolManager.ts | 69 ++- src/main/presenter/toolPresenter/index.ts | 5 + .../presenter/toolPresenter/runtimePorts.ts | 7 + .../presenter/vision/sessionVisionResolver.ts | 60 +++ .../common/DefaultModelSettingsSection.vue | 63 +-- .../components/mcp-config/mcpServerForm.vue | 84 +-- src/renderer/src/i18n/da-DK/mcp.json | 4 - src/renderer/src/i18n/da-DK/settings.json | 4 +- src/renderer/src/i18n/en-US/mcp.json | 4 - src/renderer/src/i18n/en-US/settings.json | 4 +- src/renderer/src/i18n/fa-IR/mcp.json | 4 - src/renderer/src/i18n/fa-IR/settings.json | 4 +- src/renderer/src/i18n/fr-FR/mcp.json | 4 - src/renderer/src/i18n/fr-FR/settings.json | 4 +- src/renderer/src/i18n/he-IL/mcp.json | 4 - src/renderer/src/i18n/he-IL/settings.json | 4 +- src/renderer/src/i18n/ja-JP/mcp.json | 4 - src/renderer/src/i18n/ja-JP/settings.json | 4 +- src/renderer/src/i18n/ko-KR/mcp.json | 4 - src/renderer/src/i18n/ko-KR/settings.json | 4 +- src/renderer/src/i18n/pt-BR/mcp.json | 4 - src/renderer/src/i18n/pt-BR/settings.json | 4 +- src/renderer/src/i18n/ru-RU/mcp.json | 4 - src/renderer/src/i18n/ru-RU/settings.json | 4 +- src/renderer/src/i18n/zh-CN/mcp.json | 4 - src/renderer/src/i18n/zh-CN/settings.json | 4 +- src/renderer/src/i18n/zh-HK/mcp.json | 4 - src/renderer/src/i18n/zh-HK/settings.json | 4 +- src/renderer/src/i18n/zh-TW/mcp.json | 4 - src/renderer/src/i18n/zh-TW/settings.json | 4 +- .../types/presenters/legacy.presenters.d.ts | 2 - .../defaultModelSettings.test.ts | 23 - .../deepchatAgentPresenter.test.ts | 46 +- .../agentTools/agentToolManagerRead.test.ts | 84 ++- .../agentToolManagerSettings.test.ts | 1 + .../toolPresenter/toolPresenter.test.ts | 12 +- 42 files changed, 327 insertions(+), 845 deletions(-) delete mode 100644 src/main/presenter/mcpPresenter/inMemoryServers/imageServer.ts create mode 100644 src/main/presenter/vision/sessionVisionResolver.ts diff --git a/src/main/presenter/configPresenter/index.ts b/src/main/presenter/configPresenter/index.ts index 12a9a7fb9..c2afb1096 100644 --- a/src/main/presenter/configPresenter/index.ts +++ b/src/main/presenter/configPresenter/index.ts @@ -102,7 +102,7 @@ interface IAppSettings { enableSkills?: boolean // Skills system global toggle hooksNotifications?: HooksNotificationsSettings // Hooks & notifications settings defaultModel?: { providerId: string; modelId: string } // Default model for new conversations - defaultVisionModel?: { providerId: string; modelId: string } // Default vision model for image tools + defaultVisionModel?: { providerId: string; modelId: string } // Legacy vision model setting for migration only defaultProjectPath?: string | null acpRegistryMigrationVersion?: number unifiedAgentsMigrationVersion?: number @@ -362,6 +362,7 @@ export class ConfigPresenter implements IConfigPresenter { setAgentRepository(agentRepository: AgentRepository): void { this.agentRepository = agentRepository this.initializeUnifiedAgents() + this.migrateLegacyDefaultVisionModelToBuiltinAgent() } private getAgentRepositoryOrThrow(): AgentRepository { @@ -396,6 +397,31 @@ export class ConfigPresenter implements IConfigPresenter { this.syncRegistryAgentsToRepository() } + private migrateLegacyDefaultVisionModelToBuiltinAgent(): void { + const legacySelection = this.store.get('defaultVisionModel') as ModelSelection | undefined + if (!legacySelection) { + return + } + + const providerId = legacySelection.providerId?.trim() + const modelId = legacySelection.modelId?.trim() + const builtinVisionModel = this.getBuiltinDeepChatConfig().visionModel + + if (!builtinVisionModel?.providerId || !builtinVisionModel?.modelId) { + if (providerId && modelId) { + this.updateBuiltinDeepChatConfig({ + visionModel: { + providerId, + modelId + } + }) + } + } + + this.store.delete('defaultVisionModel') + eventBus.sendToMain(CONFIG_EVENTS.SETTING_CHANGED, 'defaultVisionModel', undefined) + } + private buildLegacyBuiltinDeepChatConfig(): DeepChatAgentConfig { const defaultModel = this.store.get('defaultModel') as ModelSelection | undefined const assistantModel = this.store.get('assistantModel') as ModelSelection | undefined @@ -760,7 +786,9 @@ export class ConfigPresenter implements IConfigPresenter { const keysToClear = getAnthropicModelSelectionKeysToClear({ defaultModel: this.getSetting('defaultModel'), assistantModel: this.getSetting('assistantModel'), - defaultVisionModel: this.getSetting('defaultVisionModel'), + defaultVisionModel: this.store.get('defaultVisionModel') as + | { providerId: string; modelId: string } + | undefined, preferredModel: this.getSetting('preferredModel') }) @@ -780,9 +808,6 @@ export class ConfigPresenter implements IConfigPresenter { if (key === 'assistantModel') { return this.getBuiltinDeepChatConfig().assistantModel as T | undefined } - if (key === 'defaultVisionModel') { - return this.getDefaultVisionModel() as T | undefined - } if (key === 'default_system_prompt') { return this.getBuiltinDeepChatConfig().systemPrompt as T | undefined } @@ -808,10 +833,6 @@ export class ConfigPresenter implements IConfigPresenter { eventBus.sendToMain(CONFIG_EVENTS.SETTING_CHANGED, key, value) return } - if (key === 'defaultVisionModel') { - this.setDefaultVisionModel(value as { providerId: string; modelId: string } | undefined) - return - } if (key === 'default_system_prompt') { this.updateBuiltinDeepChatConfig({ systemPrompt: typeof value === 'string' ? value : '' @@ -2312,32 +2333,6 @@ export class ConfigPresenter implements IConfigPresenter { eventBus.sendToMain(CONFIG_EVENTS.SETTING_CHANGED, 'defaultModel', model) } - getDefaultVisionModel(): { providerId: string; modelId: string } | undefined { - const selection = this.getBuiltinDeepChatConfig().visionModel - if (selection?.providerId && selection?.modelId) { - return { - providerId: selection.providerId, - modelId: selection.modelId - } - } - return this.store.get('defaultVisionModel') as - | { providerId: string; modelId: string } - | undefined - } - - setDefaultVisionModel(model: { providerId: string; modelId: string } | undefined): void { - this.updateBuiltinDeepChatConfig({ - visionModel: - model?.providerId && model?.modelId - ? { - providerId: model.providerId, - modelId: model.modelId - } - : null - }) - eventBus.sendToMain(CONFIG_EVENTS.SETTING_CHANGED, 'defaultVisionModel', model) - } - getDefaultProjectPath(): string | null { const path = this.getSetting('defaultProjectPath') return path?.trim() ? path.trim() : null diff --git a/src/main/presenter/configPresenter/mcpConfHelper.ts b/src/main/presenter/configPresenter/mcpConfHelper.ts index d653133e7..4fd1a2543 100644 --- a/src/main/presenter/configPresenter/mcpConfHelper.ts +++ b/src/main/presenter/configPresenter/mcpConfHelper.ts @@ -165,16 +165,6 @@ const DEFAULT_INMEMORY_SERVERS: Record> }, disable: false }, - imageServer: { - args: [], - descriptions: 'Image processing MCP service', - icons: '🖼️', - autoApprove: ['read_image_base64', 'read_multiple_images_base64'], // Auto-approve reading, require confirmation for uploads - type: 'inmemory' as MCPServerType, - command: 'image', // We need to map this command to the ImageServer class later - env: {}, - disable: false - }, ragflowKnowledge: { args: [], descriptions: 'DeepChat内置RAGFlow知识库检索服务', @@ -374,7 +364,11 @@ export class McpConfHelper { private removeDeprecatedBuiltInServers( servers: Record ): Record { - const deprecatedBuiltInServers = ['powerpack', 'deepchat-inmemory/meeting-server'] + const deprecatedBuiltInServers = [ + 'powerpack', + 'deepchat-inmemory/meeting-server', + 'imageServer' + ] let hasChanges = false const removedBuiltInServers = new Set(this.getRemovedBuiltInServers()) let removedListChanged = false diff --git a/src/main/presenter/deepchatAgentPresenter/index.ts b/src/main/presenter/deepchatAgentPresenter/index.ts index b95756c1b..7d8bfd34d 100644 --- a/src/main/presenter/deepchatAgentPresenter/index.ts +++ b/src/main/presenter/deepchatAgentPresenter/index.ts @@ -52,6 +52,7 @@ import { ToolOutputGuard } from './toolOutputGuard' import type { ProviderRequestTracePayload } from '../llmProviderPresenter/requestTrace' import type { NewSessionHooksBridge } from '../hooksNotifications/newSessionBridge' import { providerDbLoader } from '../configPresenter/providerDbLoader' +import { resolveSessionVisionTarget } from '../vision/sessionVisionResolver' type PendingInteractionEntry = { interaction: PendingToolInteraction @@ -2996,7 +2997,7 @@ export class DeepChatAgentPresenter implements IAgentImplementation { const visionModel = await this.resolveScreenshotVisionModel(params.sessionId) if (!visionModel) { - return 'Screenshot captured, but automatic English analysis is unavailable because no vision model is configured.' + return 'Screenshot captured, but automatic English analysis is unavailable because neither the current session model nor the agent vision model can analyze images.' } try { @@ -3111,24 +3112,24 @@ export class DeepChatAgentPresenter implements IAgentImplementation { private async resolveScreenshotVisionModel( sessionId: string ): Promise<{ providerId: string; modelId: string } | null> { - const agentId = this.getSessionAgentId(sessionId) ?? 'deepchat' + const state = this.runtimeState.get(sessionId) + const dbSession = this.sessionStore.get(sessionId) + const resolved = await resolveSessionVisionTarget({ + providerId: state?.providerId ?? dbSession?.provider_id, + modelId: state?.modelId ?? dbSession?.model_id, + agentId: this.getSessionAgentId(sessionId) ?? 'deepchat', + configPresenter: this.configPresenter, + logLabel: `screenshot:${sessionId}` + }) - try { - const agentConfig = await this.configPresenter.resolveDeepChatAgentConfig(agentId) - const providerId = agentConfig.visionModel?.providerId?.trim() - const modelId = agentConfig.visionModel?.modelId?.trim() - if (providerId && modelId) { - return { providerId, modelId } - } - } catch (error) { - console.warn('[DeepChatAgent] Failed to resolve agent vision model:', { - sessionId, - agentId, - error - }) + if (!resolved) { + return null } - return null + return { + providerId: resolved.providerId, + modelId: resolved.modelId + } } private buildScreenshotAnalysisPrompt(): string { @@ -3137,7 +3138,7 @@ export class DeepChatAgentPresenter implements IAgentImplementation { 'Describe only what is clearly visible.', 'Include the page type or layout, the most important visible text, interactive controls, status indicators, warnings, errors, and any detail that matters for the next browser action.', 'Do not speculate about hidden or unreadable content.', - 'Return concise plain text in a single short paragraph.' + 'Return detailed plain text in a single paragraph.' ].join('\n') } diff --git a/src/main/presenter/index.ts b/src/main/presenter/index.ts index 6d513c108..ccd66399e 100644 --- a/src/main/presenter/index.ts +++ b/src/main/presenter/index.ts @@ -259,6 +259,26 @@ export class Presenter implements IPresenter { return null }, + resolveConversationSessionInfo: async (conversationId) => { + try { + const session = await this.newAgentPresenter?.getSession(conversationId) + if (!session) { + return null + } + + return { + agentId: session.agentId, + providerId: session.providerId, + modelId: session.modelId + } + } catch (error) { + console.warn('[Presenter] Failed to resolve new session info:', { + conversationId, + error + }) + return null + } + }, getSkillPresenter: () => this.skillPresenter, getYoBrowserToolHandler: () => this.yoBrowserPresenter.toolHandler, getFilePresenter: () => ({ diff --git a/src/main/presenter/mcpPresenter/inMemoryServers/builder.ts b/src/main/presenter/mcpPresenter/inMemoryServers/builder.ts index 42170938c..62272a788 100644 --- a/src/main/presenter/mcpPresenter/inMemoryServers/builder.ts +++ b/src/main/presenter/mcpPresenter/inMemoryServers/builder.ts @@ -2,7 +2,6 @@ import { ArtifactsServer } from './artifactsServer' // FileSystemServer has been removed - filesystem capabilities are now provided via Agent tools import { BochaSearchServer } from './bochaSearchServer' import { BraveSearchServer } from './braveSearchServer' -import { ImageServer } from './imageServer' import { DifyKnowledgeServer } from './difyKnowledgeServer' import { RagflowKnowledgeServer } from './ragflowKnowledgeServer' import { FastGptKnowledgeServer } from './fastGptKnowledgeServer' @@ -15,7 +14,7 @@ import { AppleServer } from './appleServer' export function getInMemoryServer( serverName: string, - args: string[], + _args: string[], env?: Record ) { switch (serverName) { @@ -28,8 +27,6 @@ export function getInMemoryServer( return new BraveSearchServer(env) case 'deepResearch': return new DeepResearchServer(env) - case 'imageServer': - return new ImageServer(args[0] || undefined, args[1] || undefined) case 'difyKnowledge': return new DifyKnowledgeServer( env as { diff --git a/src/main/presenter/mcpPresenter/inMemoryServers/imageServer.ts b/src/main/presenter/mcpPresenter/inMemoryServers/imageServer.ts deleted file mode 100644 index 370acb6eb..000000000 --- a/src/main/presenter/mcpPresenter/inMemoryServers/imageServer.ts +++ /dev/null @@ -1,479 +0,0 @@ -import { Server } from '@modelcontextprotocol/sdk/server/index.js' -import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js' -import fs from 'fs/promises' -import path from 'path' -import { z } from 'zod' -import { zodToJsonSchema } from 'zod-to-json-schema' -import { Transport } from '@modelcontextprotocol/sdk/shared/transport.js' -import { presenter } from '@/presenter' -import { ChatMessage, ChatMessageContent } from '@shared/presenter' -// import { GenerateCompletionOptions } from '@/presenter/llmProviderPresenter' // Assuming this path and type exist - using any for now - -// --- Zod Schemas for Tool Arguments --- - -const ReadImageBase64ArgsSchema = z.object({ - path: z.string().describe('Path to the image file.') -}) - -const UploadImageArgsSchema = z.object({ - path: z.string().describe('Path to the image file to upload.') -}) - -const ReadMultipleImagesBase64ArgsSchema = z.object({ - paths: z.array(z.string()).describe('List of paths to the image files.') -}) - -const UploadMultipleImagesArgsSchema = z.object({ - paths: z.array(z.string()).describe('List of paths to the image files to upload.') -}) - -const QueryImageWithPromptArgsSchema = z.object({ - path: z.string().describe('Path to the image file to query.'), - prompt: z - .string() - .describe('The prompt to use when querying the image with the multimodal model.') -}) - -const DescribeImageArgsSchema = z.object({ - path: z.string().describe('Path to the image file to do simple describe.') -}) - -const OcrImageArgsSchema = z.object({ - path: z.string().describe('Path to the image file for OCR text extraction.') -}) - -// --- Image Server Implementation --- - -export class ImageServer { - private server: Server - private provider: string - private model: string - - constructor(provider?: string, model?: string) { - const defaultVisionModel = presenter.configPresenter.getDefaultVisionModel() - this.provider = provider || defaultVisionModel?.providerId || 'openai' - this.model = model || defaultVisionModel?.modelId || 'gpt-4o' - this.server = new Server( - { - name: 'image-processing-server', - version: '0.1.0' - }, - { - capabilities: { - tools: {} - } - } - ) - this.setupRequestHandlers() - } - - // No specific initialization needed for now, but can be added for upload service config - // public async initialize(): Promise { - // // Initialization logic, e.g., configure upload service client - // } - - private getEffectiveModel(): { provider: string; model: string } { - if (this.provider && this.model) { - return { provider: this.provider, model: this.model } - } - - const defaultVisionModel = presenter.configPresenter.getDefaultVisionModel() - if (defaultVisionModel?.providerId && defaultVisionModel?.modelId) { - return { provider: defaultVisionModel.providerId, model: defaultVisionModel.modelId } - } - - throw new Error( - 'No vision model configured. Please set a default vision model in Settings > Common > Default Model.' - ) - } - - public startServer(transport: Transport): void { - this.server.connect(transport) - } - - // --- Placeholder for Image Upload Logic --- - private async uploadImageToService(filePath: string, fileBuffer: Buffer): Promise { - // TODO: Implement actual image upload logic here - // This might involve using a library like 'axios' or a specific SDK - // for services like Imgur, AWS S3, Cloudinary, etc. - console.log(`Uploading ${filePath} (size: ${fileBuffer.length} bytes)...`) - // Replace with actual upload call - await new Promise((resolve) => setTimeout(resolve, 500)) // Simulate network delay - const fakeUrl = `https://fake-upload-service.com/uploads/${path.basename(filePath)}_${Date.now()}` - console.log(`Upload complete: ${fakeUrl}`) - return fakeUrl - } - - // --- Placeholder for Multimodal Model Interaction --- - private async queryImageWithModel( - filePath: string, - fileBuffer: Buffer, - prompt: string - ): Promise { - const { provider, model } = this.getEffectiveModel() - // TODO: Implement actual API call to a multimodal model (e.g., GPT-4o, Gemini) - console.log( - `Querying ${filePath} (size: ${fileBuffer.length} bytes) using ${provider}/${model} with prompt: "${prompt}"...` - ) - - // Construct the messages array for the multimodal model - const base64Image = fileBuffer.toString('base64') - // TODO: Dynamically determine mime type if possible, otherwise assume common type like jpeg - const dataUrl = `data:image/jpeg;base64,${base64Image}` - - const messages: ChatMessage[] = [ - { - role: 'user', - content: [ - { type: 'text', text: prompt }, // Use the provided prompt - { - type: 'image_url', - image_url: { url: dataUrl } - } - ] as ChatMessageContent[] // Type assertion might be needed depending on ChatMessageContent definition - } - ] - - const modelConfig = presenter.configPresenter.getModelConfig(model, provider) - - try { - const response = await presenter.llmproviderPresenter.generateCompletionStandalone( - provider, - messages, - model, - modelConfig?.temperature ?? 0.6, - modelConfig?.maxTokens || 1000 - ) - console.log(`Model response received: ${response}`) - return response ?? 'No response generated.' // Handle potential null/undefined response - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error) - console.error(`Error querying image: ${errorMessage}`) - // Re-throw or return an error message - throw new Error(`Failed to query image: ${errorMessage}`) - // Or return `Error generating response: ${errorMessage}`; - } - } - - private async ocrImageWithModel(filePath: string, fileBuffer: Buffer): Promise { - const { provider, model } = this.getEffectiveModel() - // TODO: Implement actual API call to an OCR service or a multimodal model capable of OCR - console.log( - `Requesting OCR for ${filePath} (size: ${fileBuffer.length} bytes) using ${provider}/${model}...` - ) - - // Construct the messages array for the multimodal model - const base64Image = fileBuffer.toString('base64') - // TODO: Dynamically determine mime type if possible - const dataUrl = `data:image/jpeg;base64,${base64Image}` - - const messages: ChatMessage[] = [ - { - role: 'user', - content: [ - { type: 'text', text: 'Perform OCR on this image and return the extracted text.' }, - { - type: 'image_url', - image_url: { url: dataUrl } - } - ] as ChatMessageContent[] // Type assertion - } - ] - - console.log(messages) - - const modelConfig = presenter.configPresenter.getModelConfig(model, provider) - - try { - const ocrText = await presenter.llmproviderPresenter.generateCompletionStandalone( - provider, - messages, - model, - modelConfig?.temperature ?? 0.6, - modelConfig?.maxTokens || 1000 - ) - console.log(`OCR text received: ${ocrText}`) - return ocrText ?? 'No text extracted.' // Handle potential null/undefined response - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error) - console.error(`Error performing OCR: ${errorMessage}`) - // Re-throw or return an error message - throw new Error(`Failed to perform OCR: ${errorMessage}`) - // Or return `Error performing OCR: ${errorMessage}`; - } - } - - // --- Request Handlers --- - - private setupRequestHandlers(): void { - // List Tools Handler - this.server.setRequestHandler(ListToolsRequestSchema, async () => { - return { - tools: [ - { - name: 'read_image_base64', - description: - 'Reads an image file from the specified path and returns its base64 encoded content.', - inputSchema: zodToJsonSchema(ReadImageBase64ArgsSchema), - annotations: { - title: 'Read Image Base64', - readOnlyHint: true - } - }, - { - name: 'upload_image', - description: - 'Uploads an image file from the specified path to a hosting service and returns the public URL.', - inputSchema: zodToJsonSchema(UploadImageArgsSchema), - annotations: { - title: 'Upload Image', - destructiveHint: false, - openWorldHint: true - } - }, - { - name: 'read_multiple_images_base64', - description: - 'Reads multiple image files from the specified paths and returns their base64 encoded content.', - inputSchema: zodToJsonSchema(ReadMultipleImagesBase64ArgsSchema), - annotations: { - title: 'Read Multiple Images Base64', - readOnlyHint: true - } - }, - { - name: 'upload_multiple_images', - description: - 'Uploads multiple image files from the specified paths to a hosting service and returns their public URLs.', - inputSchema: zodToJsonSchema(UploadMultipleImagesArgsSchema), - annotations: { - title: 'Upload Multiple Images', - destructiveHint: false, - openWorldHint: true - } - }, - { - name: 'describe_image', - description: - 'Uses a multimodal model to simply describe the image at the specified path.', - inputSchema: zodToJsonSchema(DescribeImageArgsSchema), - annotations: { - title: 'Describe Image', - readOnlyHint: true, - openWorldHint: true - } - }, - { - name: 'query_image_with_prompt', - description: - 'Uses a multimodal model to answer a query (prompt) about the image at the specified path.', - inputSchema: zodToJsonSchema(QueryImageWithPromptArgsSchema), - annotations: { - title: 'Query Image with Prompt', - readOnlyHint: true, - openWorldHint: true - } - }, - { - name: 'ocr_image', - description: - 'Performs Optical Character Recognition (OCR) on the image at the specified path and returns the extracted text.', - inputSchema: zodToJsonSchema(OcrImageArgsSchema), - annotations: { - title: 'OCR Image', - readOnlyHint: true, - openWorldHint: true - } - } - ] - } - }) - - // Call Tool Handler - this.server.setRequestHandler(CallToolRequestSchema, async (request) => { - try { - const { name, arguments: args } = request.params - - switch (name) { - case 'read_image_base64': { - const parsed = ReadImageBase64ArgsSchema.safeParse(args) - if (!parsed.success) { - throw new Error(`Invalid arguments for ${name}: ${parsed.error}`) - } - // TODO: Implement path validation if necessary (similar to FileSystemServer) - const filePath = parsed.data.path - const fileBuffer = await fs.readFile(filePath) - const base64Content = fileBuffer.toString('base64') - // Determine mime type (optional but good practice) - // const mimeType = lookup(filePath) || 'application/octet-stream'; - // const dataUri = `data:${mimeType};base64,${base64Content}`; - return { - content: [{ type: 'text', text: base64Content }] // Or return dataUri - } - } - - case 'upload_image': { - const parsed = UploadImageArgsSchema.safeParse(args) - if (!parsed.success) { - throw new Error(`Invalid arguments for ${name}: ${parsed.error}`) - } - // TODO: Implement path validation if necessary - const filePath = parsed.data.path - const fileBuffer = await fs.readFile(filePath) - const imageUrl = await this.uploadImageToService(filePath, fileBuffer) - return { - content: [{ type: 'text', text: imageUrl }] - } - } - - case 'read_multiple_images_base64': { - const parsed = ReadMultipleImagesBase64ArgsSchema.safeParse(args) - if (!parsed.success) { - throw new Error(`Invalid arguments for ${name}: ${parsed.error}`) - } - const results = await Promise.allSettled( - parsed.data.paths.map(async (filePath: string) => { - try { - // TODO: Implement path validation if necessary - const fileBuffer = await fs.readFile(filePath) - return { - path: filePath, - base64: fileBuffer.toString('base64'), - status: 'fulfilled' - } - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error) - // Ensure the structure includes path and error for rejected promises - return Promise.reject({ path: filePath, error: errorMessage }) - } - }) - ) - - // Format output: [{path: string, base64?: string, error?: string}] - const formattedResults = results.map((result) => { - if (result.status === 'fulfilled') { - return { path: result.value.path, base64: result.value.base64 } - } else { - // Access reason directly as it contains the rejected structure - return { path: result.reason.path, error: result.reason.error } - } - }) - - return { - content: [{ type: 'text', text: JSON.stringify(formattedResults, null, 2) }] - } - } - - case 'upload_multiple_images': { - const parsed = UploadMultipleImagesArgsSchema.safeParse(args) - if (!parsed.success) { - throw new Error(`Invalid arguments for ${name}: ${parsed.error}`) - } - - const results = await Promise.allSettled( - parsed.data.paths.map(async (filePath: string) => { - try { - // TODO: Implement path validation if necessary - const fileBuffer = await fs.readFile(filePath) - const url = await this.uploadImageToService(filePath, fileBuffer) - return { path: filePath, url: url, status: 'fulfilled' } - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error) - // Ensure the structure includes path and error for rejected promises - return Promise.reject({ path: filePath, error: errorMessage }) - } - }) - ) - - // Format output: [{path: string, url?: string, error?: string}] - const formattedResults = results.map((result) => { - if (result.status === 'fulfilled') { - return { path: result.value.path, url: result.value.url } - } else { - // Access reason directly as it contains the rejected structure - return { path: result.reason.path, error: result.reason.error } - } - }) - - return { - content: [{ type: 'text', text: JSON.stringify(formattedResults, null, 2) }] - } - } - - case 'describe_image': { - const parsed = DescribeImageArgsSchema.safeParse(args) - if (!parsed.success) { - throw new Error(`Invalid arguments for ${name}: ${parsed.error}`) - } - // TODO: Implement path validation if necessary - const filePath = parsed.data.path - const fileBuffer = await fs.readFile(filePath) - const description = await this.queryImageWithModel( - filePath, - fileBuffer, - 'Describe this image.' - ) - return { - content: [{ type: 'text', text: description }] - } - } - - case 'query_image_with_prompt': { - const parsed = QueryImageWithPromptArgsSchema.safeParse(args) - if (!parsed.success) { - throw new Error(`Invalid arguments for ${name}: ${parsed.error}`) - } - // TODO: Implement path validation if necessary - const filePath = parsed.data.path - const prompt = parsed.data.prompt // Get the prompt - const fileBuffer = await fs.readFile(filePath) - // Call the renamed function with the prompt - const response = await this.queryImageWithModel(filePath, fileBuffer, prompt) - return { - content: [{ type: 'text', text: response }] - } - } - - case 'ocr_image': { - const parsed = OcrImageArgsSchema.safeParse(args) - if (!parsed.success) { - throw new Error(`Invalid arguments for ${name}: ${parsed.error}`) - } - // TODO: Implement path validation if necessary - const filePath = parsed.data.path - const fileBuffer = await fs.readFile(filePath) - const ocrText = await this.ocrImageWithModel(filePath, fileBuffer) - return { - content: [{ type: 'text', text: ocrText }] - } - } - - default: - throw new Error(`Unknown tool: ${name}`) - } - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error) - // Consider logging the error server-side - console.error(`Error processing tool call: ${errorMessage}`) - // Ensure the error response structure matches expected format - return { - content: [{ type: 'text', text: `Error: ${errorMessage}` }], - isError: true // Indicate this is an error response - } - } - }) - } -} - -// --- Usage Example (similar to FileSystemServer) --- -// import { WebSocketServerTransport } from '@modelcontextprotocol/sdk/transport/node'; -// -// const imageServer = new ImageServer('your-llm-provider', 'your-multimodal-model'); -// // await imageServer.initialize(); // If initialization is added -// -// // Example using WebSocket transport -// const transport = new WebSocketServerTransport({ port: 8081 }); // Choose a different port -// imageServer.startServer(transport); -// console.log('ImageServer started on port 8081'); - -// You would need a client to connect to this server and call the tools. diff --git a/src/main/presenter/toolPresenter/agentTools/agentToolManager.ts b/src/main/presenter/toolPresenter/agentTools/agentToolManager.ts index 06583badb..6ab178bc1 100644 --- a/src/main/presenter/toolPresenter/agentTools/agentToolManager.ts +++ b/src/main/presenter/toolPresenter/agentTools/agentToolManager.ts @@ -20,6 +20,7 @@ import { } from './chatSettingsTools' import type { AgentToolRuntimePort } from '../runtimePorts' import { YO_BROWSER_TOOL_NAMES } from '../../browser/YoBrowserToolDefinitions' +import { resolveSessionVisionTarget } from '../../vision/sessionVisionResolver' // Consider moving to a shared handlers location in future refactoring import { @@ -433,7 +434,7 @@ export class AgentToolManager { function: { name: 'read', description: - "Read the contents of a file. Supports pagination via offset/limit for large files (auto-truncated at 4500 chars if not specified). When invoked from a skill context with relative paths, provide base_directory as the skill's root directory.", + "Read the contents of a file. Supports pagination via offset/limit for large files (auto-truncated at 4500 chars if not specified). For image files, returns an English description of visible content instead of raw pixels. When invoked from a skill context with relative paths, provide base_directory as the skill's root directory.", parameters: zodToJsonSchema(schemas.read) as { type: string properties: Record @@ -721,7 +722,7 @@ export class AgentToolManager { if (this.isImageMimeType(mimeType)) { return { - content: await this.readImageWithVisionFallback(validPath, mimeType) + content: await this.readImageWithVisionFallback(validPath, mimeType, conversationId) } } @@ -1063,13 +1064,17 @@ export class AgentToolManager { return lines.join('\n') } - private async readImageWithVisionFallback(filePath: string, mimeType: string): Promise { + private async readImageWithVisionFallback( + filePath: string, + mimeType: string, + conversationId?: string + ): Promise { const fileBuffer = await fs.promises.readFile(filePath) const metadata = this.buildImageMetadataBlock(filePath, mimeType, fileBuffer.length) - const defaultVisionModel = this.configPresenter.getDefaultVisionModel?.() + const visionTarget = await this.resolveVisionTargetForConversation(conversationId) - if (!defaultVisionModel?.providerId || !defaultVisionModel?.modelId) { - return `${metadata}\n\nNo defaultVisionModel configured, downgraded to metadata.` + if (!visionTarget) { + return `${metadata}\n\nImage analysis unavailable because neither the current session model nor the agent vision model can analyze images.` } try { @@ -1080,12 +1085,7 @@ export class AgentToolManager { content: [ { type: 'text', - text: [ - 'Analyze this image and return exactly two sections.', - 'Section 1 title: OCR', - 'Section 2 title: Summary', - 'Keep OCR as faithful extracted text and Summary concise.' - ].join('\n') + text: this.buildImageAnalysisPrompt() }, { type: 'image_url', @@ -1096,28 +1096,61 @@ export class AgentToolManager { ] const modelConfig = this.configPresenter.getModelConfig( - defaultVisionModel.modelId, - defaultVisionModel.providerId + visionTarget.modelId, + visionTarget.providerId ) const response = await this.getLlmProviderPresenter().generateCompletionStandalone( - defaultVisionModel.providerId, + visionTarget.providerId, messages, - defaultVisionModel.modelId, + visionTarget.modelId, modelConfig?.temperature ?? 0.2, modelConfig?.maxTokens ?? 1200 ) const normalized = (response || '').trim() if (!normalized) { - return `${metadata}\n\nOCR:\n\nSummary:\nNo result returned by vision model.` + return `${metadata}\n\nImage analysis returned no usable description.` } - return normalized.startsWith('OCR:') ? normalized : `OCR:\n\nSummary:\n${normalized}` + return normalized } catch (error) { const message = error instanceof Error ? error.message : String(error) return `${metadata}\n\nVision analysis failed, downgraded to metadata.\nerror: ${message}` } } + private async resolveVisionTargetForConversation(conversationId?: string) { + if (!conversationId) { + return null + } + + try { + const sessionInfo = await this.runtimePort.resolveConversationSessionInfo(conversationId) + return await resolveSessionVisionTarget({ + providerId: sessionInfo?.providerId, + modelId: sessionInfo?.modelId, + agentId: sessionInfo?.agentId, + configPresenter: this.configPresenter, + logLabel: `read:${conversationId}` + }) + } catch (error) { + logger.warn('[AgentToolManager] Failed to resolve vision target for conversation:', { + conversationId, + error + }) + return null + } + } + + private buildImageAnalysisPrompt(): string { + return [ + 'Analyze this image and respond in English only.', + 'Describe only what is clearly visible.', + 'Include the main subject, scene or layout, any legible text, UI elements if present, status indicators, warnings, errors, and any detail that matters for understanding the image.', + 'Do not speculate about hidden or unreadable content.', + 'Return detailed plain text in a single paragraph.' + ].join('\n') + } + private assertWritePermission( toolName: string, args: Record, diff --git a/src/main/presenter/toolPresenter/index.ts b/src/main/presenter/toolPresenter/index.ts index 71dcf250c..fc03ca324 100644 --- a/src/main/presenter/toolPresenter/index.ts +++ b/src/main/presenter/toolPresenter/index.ts @@ -377,6 +377,11 @@ export class ToolPresenter implements IToolPresenter { 'Use `background: true` when you know a command should detach immediately; otherwise a foreground `exec` may yield a running `sessionId` after `yieldMs`.' ) } + if (toolNames.has('read')) { + lines.push( + 'When `read` targets an image file, it returns an English description of the visible content and any legible text.' + ) + } if (toolNames.has('exec') && toolNames.has('read') && toolNames.has('edit')) { lines.push( 'Recommended file task flow: `exec` for discovery/search -> `read` -> `edit`/`write`.' diff --git a/src/main/presenter/toolPresenter/runtimePorts.ts b/src/main/presenter/toolPresenter/runtimePorts.ts index 899540811..806b436c2 100644 --- a/src/main/presenter/toolPresenter/runtimePorts.ts +++ b/src/main/presenter/toolPresenter/runtimePorts.ts @@ -6,8 +6,15 @@ import type { } from '@shared/presenter' import type { ISkillPresenter } from '@shared/types/skill' +export interface ConversationSessionInfo { + agentId: string + providerId: string + modelId: string +} + export interface AgentToolRuntimePort { resolveConversationWorkdir(conversationId: string): Promise + resolveConversationSessionInfo(conversationId: string): Promise getSkillPresenter(): ISkillPresenter getYoBrowserToolHandler(): IYoBrowserPresenter['toolHandler'] getFilePresenter(): Pick diff --git a/src/main/presenter/vision/sessionVisionResolver.ts b/src/main/presenter/vision/sessionVisionResolver.ts new file mode 100644 index 000000000..5bb09d847 --- /dev/null +++ b/src/main/presenter/vision/sessionVisionResolver.ts @@ -0,0 +1,60 @@ +import type { IConfigPresenter } from '@shared/presenter' + +export type SessionVisionTarget = { + providerId: string + modelId: string + source: 'session-model' | 'agent-vision-model' +} + +type SessionVisionResolverParams = { + providerId?: string | null + modelId?: string | null + agentId?: string | null + configPresenter: Pick + logLabel?: string +} + +export async function resolveSessionVisionTarget( + params: SessionVisionResolverParams +): Promise { + const sessionProviderId = params.providerId?.trim() + const sessionModelId = params.modelId?.trim() + + if ( + sessionProviderId && + sessionModelId && + params.configPresenter.getModelConfig(sessionModelId, sessionProviderId)?.vision + ) { + return { + providerId: sessionProviderId, + modelId: sessionModelId, + source: 'session-model' + } + } + + const agentId = params.agentId?.trim() + if (!agentId) { + return null + } + + try { + const agentConfig = await params.configPresenter.resolveDeepChatAgentConfig(agentId) + const providerId = agentConfig.visionModel?.providerId?.trim() + const modelId = agentConfig.visionModel?.modelId?.trim() + if (providerId && modelId) { + return { + providerId, + modelId, + source: 'agent-vision-model' + } + } + } catch (error) { + console.warn('[Vision] Failed to resolve agent vision model:', { + agentId, + context: params.logLabel ?? 'unknown', + error + }) + } + + return null +} diff --git a/src/renderer/settings/components/common/DefaultModelSettingsSection.vue b/src/renderer/settings/components/common/DefaultModelSettingsSection.vue index cef7bd378..7a2ef692e 100644 --- a/src/renderer/settings/components/common/DefaultModelSettingsSection.vue +++ b/src/renderer/settings/components/common/DefaultModelSettingsSection.vue @@ -68,42 +68,6 @@ - -
- {{ - t('settings.common.defaultModel.visionModel') - }} -
- - - - - - - - -
-
@@ -119,7 +83,6 @@ import ModelIcon from '@/components/icons/ModelIcon.vue' import { useThemeStore } from '@/stores/theme' import { useModelStore } from '@/stores/modelStore' import { usePresenter } from '@/composables/usePresenter' -import { ModelType } from '@shared/model' import type { RENDERER_MODEL_META } from '@shared/presenter' const { t } = useI18n() @@ -129,7 +92,6 @@ const configPresenter = usePresenter('configPresenter') const assistantModelSelectOpen = ref(false) const chatModelSelectOpen = ref(false) -const visionModelSelectOpen = ref(false) interface SelectedModel { providerId: string @@ -138,7 +100,6 @@ interface SelectedModel { const selectedAssistantModel = ref(null) const selectedChatModel = ref(null) -const selectedVisionModel = ref(null) let isSyncingModelDefaults = false const selectBySetting = ( @@ -164,7 +125,7 @@ const selectBySetting = ( } const persistModelSetting = async ( - key: 'assistantModel' | 'defaultModel' | 'defaultVisionModel', + key: 'assistantModel' | 'defaultModel', previous: { providerId: string; modelId: string } | undefined, current: SelectedModel | null ): Promise => { @@ -198,15 +159,6 @@ const handleChatModelSelect = async ( chatModelSelectOpen.value = false } -const handleVisionModelSelect = async ( - model: RENDERER_MODEL_META, - providerId: string -): Promise => { - selectedVisionModel.value = { providerId, model } - await configPresenter.setSetting('defaultVisionModel', { providerId, modelId: model.id }) - visionModelSelectOpen.value = false -} - const syncModelSelections = async (): Promise => { if (isSyncingModelDefaults) { return @@ -219,9 +171,6 @@ const syncModelSelections = async (): Promise => { const defaultModelSetting = (await configPresenter.getSetting('defaultModel')) as | { providerId: string; modelId: string } | undefined - const defaultVisionModelSetting = (await configPresenter.getSetting('defaultVisionModel')) as - | { providerId: string; modelId: string } - | undefined const chatSelection = selectBySetting( defaultModelSetting, @@ -233,21 +182,11 @@ const syncModelSelections = async (): Promise => { (_model, providerId) => providerId !== 'acp' ) - const visionSelection = selectBySetting( - defaultVisionModelSetting, - (model, providerId) => - providerId !== 'acp' && - Boolean(model.vision) && - (model.type === ModelType.Chat || model.type === ModelType.ImageGeneration) - ) - selectedChatModel.value = chatSelection selectedAssistantModel.value = assistantSelection - selectedVisionModel.value = visionSelection await persistModelSetting('defaultModel', defaultModelSetting, chatSelection) await persistModelSetting('assistantModel', assistantModelSetting, assistantSelection) - await persistModelSetting('defaultVisionModel', defaultVisionModelSetting, visionSelection) } catch (error) { console.error('Failed to sync model selections:', error) } finally { diff --git a/src/renderer/src/components/mcp-config/mcpServerForm.vue b/src/renderer/src/components/mcp-config/mcpServerForm.vue index 8b1ca56bb..8c5d32923 100644 --- a/src/renderer/src/components/mcp-config/mcpServerForm.vue +++ b/src/renderer/src/components/mcp-config/mcpServerForm.vue @@ -18,16 +18,12 @@ import { EmojiPicker } from '@/components/emoji-picker' import { useToast } from '@/components/use-toast' import { Icon } from '@iconify/vue' import { X } from 'lucide-vue-next' -import ModelIcon from '@/components/icons/ModelIcon.vue' -import { useModelStore } from '@/stores/modelStore' import { usePresenter } from '@/composables/usePresenter' import { nanoid } from 'nanoid' const { t } = useI18n() const { toast } = useToast() -const modelStore = useModelStore() const devicePresenter = usePresenter('devicePresenter') -const configPresenter = usePresenter('configPresenter') const props = defineProps<{ serverName?: string initialConfig?: MCPServerConfig @@ -57,14 +53,8 @@ const customHeadersFocused = ref(false) const customHeadersDisplayValue = ref('') const npmRegistry = ref(props.initialConfig?.customNpmRegistry || '') -// imageServer 展示用（只读，来源于 defaultVisionModel） -const selectedImageModelName = ref('') -const selectedImageModelProvider = ref('') - // 判断是否是inmemory类型 const isInMemoryType = computed(() => type.value === 'inmemory') -// 判断是否是imageServer -const isImageServer = computed(() => isInMemoryType.value && name.value === 'imageServer') // 判断是否是buildInFileSystem const isBuildInFileSystem = computed( () => isInMemoryType.value && name.value === 'buildInFileSystem' @@ -80,32 +70,6 @@ const formatJsonHeaders = (headers: Record): string => { .map(([key, value]) => `${key}=${value}`) .join('\n') } -const refreshImageServerDefaultModelDisplay = async (): Promise => { - if (!isImageServer.value) { - selectedImageModelName.value = '' - selectedImageModelProvider.value = '' - return - } - - const defaultVisionModel = (await configPresenter.getSetting('defaultVisionModel')) as - | { providerId: string; modelId: string } - | undefined - if (!defaultVisionModel?.providerId || !defaultVisionModel?.modelId) { - selectedImageModelName.value = '' - selectedImageModelProvider.value = '' - return - } - - selectedImageModelProvider.value = defaultVisionModel.providerId - const providerEntry = modelStore.enabledModels.find( - (entry) => entry.providerId === defaultVisionModel.providerId - ) - const resolvedModel = providerEntry?.models.find( - (model) => model.id === defaultVisionModel.modelId - ) - selectedImageModelName.value = - resolvedModel?.name || `${defaultVisionModel.providerId}/${defaultVisionModel.modelId}` -} // 获取内置服务器的本地化名称和描述 const getLocalizedName = computed(() => { @@ -144,11 +108,9 @@ const jsonConfig = ref('') const showBaseUrl = computed(() => isRemoteType.value) // 添加计算属性来控制命令相关字段的显示 const showCommandFields = computed(() => type.value === 'stdio') -// 控制参数输入框的显示 (stdio 或 非imageServer且非buildInFileSystem的inmemory) +// 控制参数输入框的显示 (stdio 或 非buildInFileSystem的inmemory) const showArgsInput = computed( - () => - showCommandFields.value || - (isInMemoryType.value && !isImageServer.value && !isBuildInFileSystem.value) + () => showCommandFields.value || (isInMemoryType.value && !isBuildInFileSystem.value) ) // 控制文件夹选择界面的显示 (仅针对 buildInFileSystem) @@ -253,11 +215,11 @@ const isNameValid = computed(() => name.value.trim().length > 0) const isCommandValid = computed(() => { // 对于SSE类型，命令不是必需的 if (isRemoteType.value) return true - // 对于STDIO 或 inmemory 类型，命令是必需的 (排除内置 server) - if (type.value === 'stdio' || (isInMemoryType.value && !isImageServer.value)) { + // 对于STDIO 或 inmemory 类型，命令是必需的 + if (type.value === 'stdio' || isInMemoryType.value) { return command.value.trim().length > 0 } - return true // 其他情况（如 imageServer）默认有效 + return true }) const isEnvValid = computed(() => { try { @@ -473,11 +435,9 @@ const handleSubmit = (): void => { } } else { // STDIO 或 inmemory 类型的服务器 - const normalizedArgs = isImageServer.value - ? [] - : isBuildInFileSystem.value - ? foldersList.value.filter((folder) => folder.trim().length > 0) - : argsRows.value.map((row) => row.value.trim()).filter((value) => value.length > 0) + const normalizedArgs = isBuildInFileSystem.value + ? foldersList.value.filter((folder) => folder.trim().length > 0) + : argsRows.value.map((row) => row.value.trim()).filter((value) => value.length > 0) serverConfig = { ...baseConfig, command: command.value.trim(), @@ -592,15 +552,6 @@ watch( { immediate: true } ) -// imageServer 仅展示默认视觉模型，不再通过 args 配置 -watch( - [() => name.value, () => type.value, () => modelStore.enabledModels], - () => { - void refreshImageServerDefaultModelDisplay() - }, - { immediate: true, deep: true } -) - // Watch for initial config changes (primarily for edit mode) watch( () => props.initialConfig, @@ -793,25 +744,6 @@ HTTP-Referer=deepchatai.cn` /> - -
- -
- - {{ - selectedImageModelName || t('settings.mcp.serverForm.imageModel') - }} -
-
-