From 4efb6d29991920ffa56edc58879891c0c4926875 Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Thu, 18 Jun 2026 15:02:49 +0000
Subject: [PATCH 1/2] derive attachment content type from MIME

Attachments were hardcoded to the "document" AG-UI content type, so
uploaded images (and audio/video) were mislabeled and never reached the
model as the correct modality. Derive the discriminator from the file's
MIME type instead, falling back to "document".

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 frontend/src/chat/chatinput.tsx | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/frontend/src/chat/chatinput.tsx b/frontend/src/chat/chatinput.tsx
index 399bd6e..b684cbf 100644
--- a/frontend/src/chat/chatinput.tsx
+++ b/frontend/src/chat/chatinput.tsx
@@ -106,8 +106,18 @@ function ChatInput(): ReactNode {
       // well-factored. Could be better...
       const fics: readonly api.FileInputContent[] = await Promise.all(
         inputFiles.map(async ({ file }) => {
+          // Derive the content discriminator from the file's MIME type so
+          // images, audio, and video are labeled as the correct modality,
+          // falling back to "document" for everything else.
+          const type = file.type.startsWith("image/")
+            ? "image"
+            : file.type.startsWith("audio/")
+              ? "audio"
+              : file.type.startsWith("video/")
+                ? "video"
+                : "document";
           return {
-            type: "document",
+            type,
             source: {
               type: "data",
               mimeType: file.type,

From d7b25e002dfdf832ebcc588aa8711b51c64483ba Mon Sep 17 00:00:00 2001
From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com>
Date: Thu, 18 Jun 2026 14:11:00 -0500
Subject: [PATCH 2/2] Drive file upload UI from agent multimodal capabilities
 (#151)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 frontend/src/chat/chatinput.tsx | 79 ++++++++++++++++++++++++---------
 1 file changed, 59 insertions(+), 20 deletions(-)

diff --git a/frontend/src/chat/chatinput.tsx b/frontend/src/chat/chatinput.tsx
index 901ad47..b86041b 100644
--- a/frontend/src/chat/chatinput.tsx
+++ b/frontend/src/chat/chatinput.tsx
@@ -7,7 +7,7 @@ import { ArrowUp, Hammer, Paperclip, X } from 'lucide-react';
 
 import type { FormEvent, KeyboardEvent, MouseEvent, ReactNode } from 'react';
 
-import { useCallback, useRef, useState } from 'react';
+import { useCallback, useMemo, useRef, useState } from 'react';
 
 import type * as api from '@/api';
 
@@ -27,6 +27,10 @@ import { cn } from '@/lib/utils';
 
 import { useOnSubmit } from './hooks';
 
+// The `accept` value used for agents that accept generic document uploads,
+// and for agents that don't advertise any multimodal input capabilities.
+const DOCUMENT_ACCEPT = '.txt,.csv,.md';
+
 /**
  * A react component that renders the chat input box.
  */
@@ -48,6 +52,37 @@ export function ChatInput(): ReactNode {
   const agent = agents.find((a) => a.id === agentId);
   const hasTools = (agent?.capabilities?.tools?.items?.length ?? 0) > 0;
 
+  // Resolve the file-upload behavior from the agent's advertised AG-UI
+  // multimodal input capabilities: build the picker's `accept` value and
+  // decide whether to offer uploads at all.
+  const { uploadsAllowed, accept } = useMemo(() => {
+    const input = agent?.capabilities?.multimodal?.input;
+
+    // If the agent didn't advertise any input modalities, preserve the
+    // historical behavior: allow document uploads.
+    if (input === undefined) {
+      return { uploadsAllowed: true, accept: DOCUMENT_ACCEPT };
+    }
+
+    // Otherwise build the `accept` value from the enabled modalities.
+    const accepts: string[] = [];
+    if (input.image) {
+      accepts.push('image/*');
+    }
+    if (input.audio) {
+      accepts.push('audio/*');
+    }
+    if (input.video) {
+      accepts.push('video/*');
+    }
+    if (input.file) {
+      accepts.push(DOCUMENT_ACCEPT);
+    }
+
+    // Uploads are allowed only if at least one modality is enabled.
+    return { uploadsAllowed: accepts.length > 0, accept: accepts.join(',') };
+  }, [agent]);
+
   // Check file-related permissions.
   const canAttachFiles = useHasPermissions(['files:read', 'files:write']);
 
@@ -305,25 +340,29 @@ export function ChatInput(): ReactNode {
           className="outline-none resize-none field-sizing-content w-full"
         />
         <div className="flex flex-row gap-2">
-          <input
-            ref={inputRef}
-            onChange={handleInputChange}
-            className="hidden"
-            type="file"
-            multiple
-            accept=".txt,.csv,.md"
-          />
-          {filePermissionTooltip ? (
-            <Tooltip>
-              <TooltipTrigger asChild>
-                <span>{attachButton}</span>
-              </TooltipTrigger>
-              <TooltipContent side="top">
-                {filePermissionTooltip}
-              </TooltipContent>
-            </Tooltip>
-          ) : (
-            attachButton
+          {uploadsAllowed && (
+            <>
+              <input
+                ref={inputRef}
+                onChange={handleInputChange}
+                className="hidden"
+                type="file"
+                multiple
+                accept={accept}
+              />
+              {filePermissionTooltip ? (
+                <Tooltip>
+                  <TooltipTrigger asChild>
+                    <span>{attachButton}</span>
+                  </TooltipTrigger>
+                  <TooltipContent side="top">
+                    {filePermissionTooltip}
+                  </TooltipContent>
+                </Tooltip>
+              ) : (
+                attachButton
+              )}
+            </>
           )}
           {toolsToggle}
           <div className="grow flex flex-row flex-wrap gap-2 items-center">