[Feature] support and harden native multimodal file handling

packages/adapter-claude/src/utils.ts

@@ -56,41 +56,15 @@ export async function langchainMessageToClaudeMessage(
 
     const mappedMessages = await Promise.all(
         messages.map(async (rawMessage) => {
-            let content: string | ClaudeInputContentBlockParam[] | undefined =
+            const content: string | ClaudeInputContentBlockParam[] | undefined =
                 typeof rawMessage.content === 'string'
                     ? rawMessage.content
                     : await processMessageContent(plugin, rawMessage.content)
 
-            const images = rawMessage.additional_kwargs.images as
-                | string[]
-                | null
-
-            if (
-                (model?.includes('claude-3') || model?.includes('claude-4')) &&
-                images != null
-            ) {
-                const mappedImages = await Promise.all(
-                    images.map(async (image) =>
-                        processImageContent(plugin, {
-                            type: 'image_url',
-                            image_url: { url: image }
-                        } as MessageContentImageUrl)
-                    )
+            if (rawMessage.additional_kwargs.images != null) {
+                logger.warn(
+                    'Deprecated: `additional_kwargs.images` is no longer supported. Use `image_url` content parts instead.'
                 )
-
-                const nextContent: ClaudeInputContentBlockParam[] =
-                    mappedImages.filter((item) => item != null)
-
-                if (Array.isArray(content)) {
-                    nextContent.push(...content)
-                } else if ((content?.length ?? 0) > 0) {
-                    nextContent.push({
-                        type: 'text',
-                        text: content
-                    })
-                }
-
-                content = nextContent
             }
 
             const result: ClaudeMessage = {

packages/adapter-gemini/src/utils.ts

@@ -14,7 +14,6 @@ import {
     ChatCompletionResponseMessageRoleEnum,
     ChatFunctionCallingPart,
     ChatFunctionResponsePart,
-    ChatMessagePart,
     ChatPart,
     ChatResponse,
     GeminiUsageMetadata
@@ -78,9 +77,10 @@ export async function langchainMessageToGeminiMessage(
                           thoughtData
                       )
 
-            const images = message.additional_kwargs.images as string[] | null
-            if (images) {
-                processImageParts(result, images, model)
+            if (message.additional_kwargs.images != null) {
+                logger.warn(
+                    'Deprecated: `additional_kwargs.images` is no longer supported. Use `image_url` content parts instead.'
+                )
             }
 
             return result
@@ -203,39 +203,6 @@ async function processFunctionMessage(
         ]
     }
 }
-function processImageParts(
-    result: ChatCompletionResponseMessage,
-    images: string[],
-    model: string
-) {
-    if (
-        !(
-            (model.includes('vision') ||
-                model.includes('gemini') ||
-                model.includes('gemma2')) &&
-            !model.includes('gemini-1.0')
-        )
-    ) {
-        return
-    }
-
-    for (const image of images) {
-        const mineType = image.split(';')?.[0]?.split(':')?.[1] ?? 'image/jpeg'
-        const data = image.replace(/^data:image\/\w+;base64,/, '')
-
-        result.parts.push({
-            inline_data: { data, mime_type: mineType }
-        })
-    }
-
-    result.parts = result.parts.filter((uncheckedPart) => {
-        const part = partAsTypeCheck<ChatMessagePart>(
-            uncheckedPart,
-            (part) => part['text'] != null
-        )
-        return part == null || part.text.length > 0
-    })
-}
 
 async function processGeminiImageContent(
     plugin: ChatLunaPlugin,

packages/adapter-ollama/src/utils.ts

@@ -21,25 +21,24 @@ export async function langchainMessageToOllamaMessage(
 
     const mappedMessage = await Promise.all(
         messages.map(async (rawMessage) => {
-            let images: string[] = []
-
-            if (rawMessage.additional_kwargs.images != null && supportImage) {
-                images = rawMessage.additional_kwargs.images as string[]
-            } else {
-                images =
-                    typeof rawMessage.content === 'string'
-                        ? undefined
-                        : await Promise.all(
-                              rawMessage.content
-                                  .filter((part) =>
-                                      isMessageContentImageUrl(part)
-                                  )
-                                  .map((part) =>
-                                      processOllamaImageContent(plugin, part)
-                                  )
-                          )
+            if (rawMessage.additional_kwargs.images != null) {
+                logger.warn(
+                    'Deprecated: `additional_kwargs.images` is no longer supported. Use `image_url` content parts instead.'
+                )
             }
 
+            const images: string[] | undefined = supportImage
+                ? typeof rawMessage.content === 'string'
+                    ? undefined
+                    : await Promise.all(
+                          rawMessage.content
+                              .filter((part) => isMessageContentImageUrl(part))
+                              .map((part) =>
+                                  processOllamaImageContent(plugin, part)
+                              )
+                      )
+                : undefined
+
             const result = {
                 role: messageTypeToOllamaRole(rawMessage.getType()),
                 content: getMessageContent(rawMessage.content),

packages/adapter-openai-like/src/client.ts

@@ -20,10 +20,12 @@ import { OpenAIRequester } from './requester'
 import { ChatLunaPlugin } from 'koishi-plugin-chatluna/services/chat'
 import {
     getModelMaxContextSize,
+    getOpenAIFileHandlingConfig,
     isEmbeddingModel,
     isImageGenerationModel,
     isNonLLMModel,
     isRerankerModel,
+    supportAudioInput,
     supportImageInput
 } from '@chatluna/v1-shared-adapter'
 import { RunnableConfig } from '@langchain/core/runnables'
@@ -92,6 +94,9 @@ export class OpenAIClient extends PlatformModelEmbeddingsAndRerankerClient {
                         ModelCapabilities.ToolCall,
                         supportImageInput(model)
                             ? ModelCapabilities.ImageInput
+                            : null,
+                        supportAudioInput(model)
+                            ? ModelCapabilities.AudioInput
                             : null
                     ].filter(Boolean)
                 }
@@ -167,6 +172,7 @@ export class OpenAIClient extends PlatformModelEmbeddingsAndRerankerClient {
                 temperature: this._config.temperature,
                 maxRetries: this._config.maxRetries,
                 llmType: 'openai',
+                fileHandlingConfig: getOpenAIFileHandlingConfig(model),
                 isThinkModel:
                     model.includes('reasoner') ||
                     model.includes('r1') ||

packages/adapter-openai/src/client.ts

@@ -20,6 +20,8 @@ import { OpenAIRequester } from './requester'
 import { ChatLunaPlugin } from 'koishi-plugin-chatluna/services/chat'
 import {
     getModelMaxContextSize,
+    getOpenAIFileHandlingConfig,
+    supportAudioInput,
     supportImageInput
 } from '@chatluna/v1-shared-adapter'
 import { RunnableConfig } from '@langchain/core/runnables'
@@ -65,13 +67,11 @@ export class OpenAIClient extends PlatformModelAndEmbeddingsClient<ClientConfig>
                     (model) =>
                         !(
                             model.includes('instruct') ||
-                            [
-                                'whisper',
-                                'tts',
-                                'dall-e',
-                                'audio',
-                                'realtime'
-                            ].some((keyword) => model.includes(keyword))
+                            ['whisper', 'tts', 'dall-e', 'realtime'].some(
+                                (keyword) => model.includes(keyword)
+                            ) ||
+                            (model.includes('audio') &&
+                                !supportAudioInput(model))
                         )
                 )
                 .map((model) => {
@@ -84,6 +84,9 @@ export class OpenAIClient extends PlatformModelAndEmbeddingsClient<ClientConfig>
                             ModelCapabilities.ToolCall,
                             supportImageInput(model)
                                 ? ModelCapabilities.ImageInput
+                                : undefined,
+                            supportAudioInput(model)
+                                ? ModelCapabilities.AudioInput
                                 : undefined
                         ].filter(Boolean)
                     } as ModelInfo
@@ -125,6 +128,7 @@ export class OpenAIClient extends PlatformModelAndEmbeddingsClient<ClientConfig>
                 timeout: this._config.timeout,
                 temperature: this._config.temperature,
                 maxRetries: this._config.maxRetries,
+                fileHandlingConfig: getOpenAIFileHandlingConfig(model),
                 llmType: 'openai'
             })
         }

packages/adapter-qwen/src/utils.ts

@@ -5,7 +5,6 @@ import {
     ChatMessageChunk,
     FunctionMessageChunk,
     HumanMessageChunk,
-    MessageContentImageUrl,
     MessageType,
     SystemMessageChunk,
     ToolMessage,
@@ -21,11 +20,11 @@ import {
 } from './types'
 import {
     fetchImageUrl,
-    removeAdditionalProperties,
-    supportImageInput
+    removeAdditionalProperties
 } from '@chatluna/v1-shared-adapter'
 import { ChatLunaPlugin } from 'koishi-plugin-chatluna/services/chat'
 import { isZodSchemaV3 } from '@langchain/core/utils/types'
+import { logger } from '.'
 
 export function formatToolsToQWenTools(
     tools: StructuredTool[]
@@ -113,50 +112,13 @@ export async function langchainMessageToQWenMessage(
             }
         }
 
-        const images = rawMessage.additional_kwargs.images as string[] | null
-
-        if (
-            (model?.includes('qwen-vl') ||
-                model?.includes('omni') ||
-                model?.includes('qwen2.5-vl') ||
-                model?.includes('qwen2.5-omni') ||
-                model?.includes('qwen-omni') ||
-                model?.includes('qwen2-vl') ||
-                model?.includes('qvq') ||
-                supportImageInput(model)) &&
-            images != null
-        ) {
-            msg.content = [
-                {
-                    type: 'text',
-                    text: rawMessage.content as string
-                }
-            ]
-
-            const imageContents = await Promise.all(
-                images.map(async (image) => {
-                    try {
-                        const url = await fetchImageUrl(plugin, {
-                            type: 'image_url',
-                            image_url: { url: image }
-                        } as MessageContentImageUrl)
-                        return {
-                            type: 'image_url',
-                            image_url: {
-                                url,
-                                detail: 'low'
-                            }
-                        } as const
-                    } catch {
-                        return null
-                    }
-                })
+        if (rawMessage.additional_kwargs.images != null) {
+            logger.warn(
+                'Deprecated: `additional_kwargs.images` is no longer supported. Use `image_url` content parts instead.'
             )
+        }
 
-            msg.content.push(
-                ...imageContents.filter((content) => content != null)
-            )
-        } else if (Array.isArray(msg.content) && msg.content.length > 0) {
+        if (Array.isArray(msg.content) && msg.content.length > 0) {
             const mappedContent = await Promise.all(
                 msg.content.map(async (content) => {
                     if (!isMessageContentImageUrl(content)) return content

packages/service-multimodal/README.md

@@ -1,7 +1,7 @@
-## koishi-plugin-chatluna-long-memory
+## koishi-plugin-chatluna-multimodal-service
 
-## [![npm](https://img.shields.io/npm/v/koishi-plugin-chatluna-long-memory)](https://www.npmjs.com/package/koishi-plugin-chatluna-long-memory) [![npm](https://img.shields.io/npm/dm/koishi-plugin-chatluna-long-memory)](https://www.npmjs.com/package//koishi-plugin-chatluna-long-memory)
+## [![npm](https://img.shields.io/npm/v/koishi-plugin-chatluna-multimodal-service)](https://www.npmjs.com/package/koishi-plugin-chatluna-multimodal-service) [![npm](https://img.shields.io/npm/dm/koishi-plugin-chatluna-multimodal-service)](https://www.npmjs.com/package/koishi-plugin-chatluna-multimodal-service)
 
-> 提供长期记忆支持的插件
+> ChatLuna 的多模态服务插件，提供上下文图像/语音描述、GIF 处理与 `read_files` 文件读取工具。
 
-[长期记忆文档](https://chatluna.chat/ecosystem/renderer/image.html)
+[多模态插件文档](https://chatluna.chat/ecosystem/plugin/multimodal-service.html)

packages/service-multimodal/src/index.ts

@@ -83,7 +83,7 @@ export const Config: Schema<Config> = Schema.intersect([
 
 export const inject = {
     required: ['chatluna'],
-    optional: ['chatluna_storage', 'ffmpeg', 'silk']
+    optional: ['ffmpeg', 'silk']
 }
 
 export const name = 'chatluna-multimodal-service'