diff --git a/docs-site/src/content/docs/guides/combos.md b/docs-site/src/content/docs/guides/combos.md index 434cdcbb46..6bc5f3b6ee 100644 --- a/docs-site/src/content/docs/guides/combos.md +++ b/docs-site/src/content/docs/guides/combos.md @@ -211,6 +211,14 @@ default and leaves the target's own behavior unchanged. Supported values are `lo `high`, `xhigh`, `max`, and `ultra`; omit the field or set it to `null` to leave effort entirely to the caller and target. +## Image / multimodal capability + +By default a combo publishes the **intersection** of its targets' input modalities (image is +enabled only when every target advertises it). Set `imageInput: "disabled"` to force text-only +even when every target supports images — the catalog drops `image` from `inputModalities`, and +image-bearing requests are rejected with HTTP 400 before any target is called. `"auto"` (or +omitting the field) keeps the automatic intersection. + ## Encrypted v2 sub-agent tasks There is one important limitation for Codex v2 sub-agents ([issue #92](https://github.com/lidge-jun/opencodex/issues/92)). @@ -304,6 +312,7 @@ Combos are stored in the top-level `combos` object, keyed by combo id: | `strategy` | No | `"failover"` | `"failover"` or `"round-robin"`. | | `stickyLimit` | No | `1` | Integer from 1 to 100 successful requests per round-robin selection. | | `defaultEffort` | No | `null` | `low`, `medium`, `high`, `xhigh`, `max`, or `ultra`; applied only when the caller omits effort and the target advertises support. | +| `imageInput` | No | `"auto"` | `"auto"` or `"disabled"`. `"auto"` publishes image support only when every target supports images; `"disabled"` forces text-only (drops image from published modalities and rejects image-bearing requests before dispatch). | | `alias` | No | none | Optional trimmed public model id; use the alias rules above. An empty value is stored as no alias. | | `nativeAlias` | No | `false` | Explicitly permit a currently supported bare native `alias` to take routing and catalog precedence. Never inferred from the alias. | | `displayName` | No | none | Bounded display-only catalog label. Required and non-empty when `nativeAlias` is true. | diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 18ca8a3690..fd18aed5ad 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -70,6 +70,7 @@ namespace, and cannot use reserved bare native families such as `gpt-*`, `o1-*`, | `strategy?` | `"failover" \| "round-robin"` | `"failover"` | Selection strategy. Target order is failover priority; weights shape smooth weighted round-robin. | | `stickyLimit?` | `number` | `1` | Successful requests retained in one round-robin batch. Range 1–100. | | `defaultEffort?` | `"low" \| "medium" \| "high" \| "xhigh" \| "max" \| "ultra" \| null` | unset | Applied only when the caller omits effort and the selected target advertises the requested rung. | +| `imageInput?` | `"auto" \| "disabled"` | `"auto"` | `"auto"` publishes image only when every target supports images; `"disabled"` forces text-only (drops image from published modalities and rejects image-bearing requests before dispatch). | | `alias?` | `string` | — | Optional public model id in place of the canonical picker slug. | | `nativeAlias?` | `boolean` | `false` | Let a currently supported bare native id take precedence only for that unqualified id. Bare `gpt-5.6-*` ids use Codex Pool/Direct credentials. Account-qualified routes remain distinct. Provider-qualified routes such as `openai-apikey/gpt-5.6-*` use their configured API-key route and never fall through to the native alias. | | `displayName?` | `string` | — | Display-only catalog label, required and non-empty for a native alias. | diff --git a/docs-site/src/content/docs/zh-cn/guides/combos.md b/docs-site/src/content/docs/zh-cn/guides/combos.md index 244d3d71ba..fe89fc04c7 100644 --- a/docs-site/src/content/docs/zh-cn/guides/combos.md +++ b/docs-site/src/content/docs/zh-cn/guides/combos.md @@ -158,6 +158,10 @@ combo 失败分为 **跳转** 失败和 **终止** 失败。 当目标能力未知，或者不包含配置的 effort 时，opencodex 会省略默认值，并保持目标自身行为不变。支持的值是 `low`、`medium`、`high`、`xhigh`、`max` 和 `ultra`；省略该字段或将其设为 `null`，就会把 effort 完全交给调用方和目标。 +## 图片 / 多模态能力 + +默认情况下，combo 会发布其目标 **input modalities 的交集**（只有当每个目标都声明支持图片时，图片才会启用）。设置 `imageInput: "disabled"` 可在目标均支持图片时仍强制仅文本——目录会从 `inputModalities` 中去掉 `image`，带图请求会在分发前以 HTTP 400 拒绝。`"auto"`（或省略该字段）保持自动交集。 + ## 加密的 v2 子代理任务 对于 Codex v2 子代理，有一个重要限制（[issue #92](https://github.com/lidge-jun/opencodex/issues/92)）。原生父进程只能把新启动 worker 的任务，以为原生 ChatGPT 后端生成的密文形式发送出去。外部 provider 无法读取那段负载。 @@ -241,6 +245,7 @@ combo 会存储在顶层的 `combos` 对象中，并以 combo id 作为键： | `strategy` | 否 | `"failover"` | `"failover"` 或 `"round-robin"`。 | | `stickyLimit` | 否 | `1` | 每次轮询选择可连续处理的成功请求数，范围为 1 到 100。 | | `defaultEffort` | 否 | `null` | `low`、`medium`、`high`、`xhigh`、`max` 或 `ultra`；仅当调用方省略 effort 且目标声明支持时才会应用。 | +| `imageInput` | 否 | `"auto"` | `"auto"` 或 `"disabled"`。`"auto"` 仅在每个目标都支持图片时发布图片能力；`"disabled"` 强制仅文本（从对外能力中去掉图片，并在分发前拒绝带图请求）。 | | `alias` | 否 | 无 | 可选的、已修剪的公开模型 id；使用上面的别名规则。空值会以“无别名”形式存储。 | | `nativeAlias` | 否 | `false` | 显式允许当前受支持的裸原生 alias 接管路由和 catalog 优先级；绝不会根据 alias 自动推断。 | | `displayName` | 否 | 无 | 仅用于 catalog 展示的有界标签；`nativeAlias` 为 true 时必须非空。 | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/routing.md b/docs-site/src/content/docs/zh-cn/reference/configuration/routing.md index 166a2ce17b..11a3ad97ce 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/routing.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/routing.md @@ -64,6 +64,7 @@ Codex Auth 页面将此 picker 行为作为选择加入项。关闭它会隐藏 | `strategy?` | `"failover" \| "round-robin"` | `"failover"` | 选择策略。目标顺序表示故障切换优先级；权重会影响平滑加权轮询。 | | `stickyLimit?` | `number` | `1` | 在单个轮询批次中保留的成功请求数。范围 1–100。 | | `defaultEffort?` | `"low" \| "medium" \| "high" \| "xhigh" \| "max" \| "ultra" \| null` | unset | 仅在调用方省略 effort 且所选目标声明了请求的档位时应用。 | +| `imageInput?` | `"auto" \| "disabled"` | `"auto"` | `"auto"` 仅在每个目标都支持图片时发布图片能力；`"disabled"` 强制仅文本（从对外能力中去掉图片，并在分发前拒绝带图请求）。 | | `alias?` | `string` | — | 可选的公开 model id，用于替代规范化的选择器 slug。 | | `nativeAlias?` | `boolean` | `false` | 仅让当前受支持的裸原生 id 对该不带限定前缀的 id 优先；带账号或提供方限定的 OpenAI 路由仍是独立路由。 | | `displayName?` | `string` | — | 仅用于 catalog 展示的标签；native alias 必须提供非空值。 | diff --git a/gui/src/combo-capabilities.ts b/gui/src/combo-capabilities.ts new file mode 100644 index 0000000000..32f91f66b9 --- /dev/null +++ b/gui/src/combo-capabilities.ts @@ -0,0 +1,14 @@ +import type { ComboTarget } from "./combo-workspace-data"; +import type { ModelOption } from "./components/combo-workspace-types"; + +/** Whether every selected target advertises image input (incomplete rows fail closed). */ +export function comboImagesSupported(targets: ComboTarget[], models: ModelOption[]): boolean { + if (targets.length === 0) return false; + return targets.every((target) => { + const provider = target.provider.trim(); + const modelId = target.model.trim(); + if (!provider || !modelId) return false; + const model = models.find((row) => row.provider === provider && row.id === modelId); + return !!model?.inputModalities?.includes("image"); + }); +} diff --git a/gui/src/combo-workspace-data.ts b/gui/src/combo-workspace-data.ts index 56ce088238..4f5e96e0b7 100644 --- a/gui/src/combo-workspace-data.ts +++ b/gui/src/combo-workspace-data.ts @@ -60,6 +60,11 @@ export function newComboTarget(partial: Partial = {}): ComboTarget }; } + +function normalizeImageInput(value: unknown): "auto" | "disabled" { + return value === "disabled" ? "disabled" : "auto"; +} + export interface ComboItem { id: string; /** Wire id shown to clients, e.g. combo/free */ @@ -73,6 +78,7 @@ export interface ComboItem { strategy: ComboStrategy; stickyLimit: number; defaultEffort: ComboEffort | null; + imageInput?: "auto" | "disabled"; targets: ComboTarget[]; } @@ -177,6 +183,7 @@ export function parseComboList(payload: unknown): ComboItem[] { strategy: normalizeStrategy(r.strategy), stickyLimit: normalizeStickyLimit(r.stickyLimit), defaultEffort: normalizeDefaultEffort(r.defaultEffort), + imageInput: normalizeImageInput(r.imageInput), targets, }); } @@ -236,6 +243,7 @@ export function draftEquals(a: ComboItem, b: ComboItem): boolean { || a.strategy !== b.strategy || a.stickyLimit !== b.stickyLimit || a.defaultEffort !== b.defaultEffort + || (a.imageInput ?? "auto") !== (b.imageInput ?? "auto") ) return false; if (a.targets.length !== b.targets.length) return false; return a.targets.every((t, i) => { @@ -252,6 +260,7 @@ export function toPutBody(item: ComboItem, options: { renameFrom?: string } = {} strategy: ComboStrategy; stickyLimit?: number; defaultEffort: ComboEffort | null; + imageInput?: "disabled"; alias?: string; nativeAlias?: true; displayName?: string; @@ -266,6 +275,7 @@ export function toPutBody(item: ComboItem, options: { renameFrom?: string } = {} : { provider: target.provider.trim(), model: target.model.trim() }), strategy: item.strategy, defaultEffort: item.defaultEffort, + ...(item.imageInput === "disabled" ? { imageInput: "disabled" as const } : {}), ...(item.strategy === "round-robin" ? { stickyLimit: item.stickyLimit } : {}), ...(item.alias && item.alias.trim() ? { alias: item.alias.trim() } : {}), ...(item.nativeAlias ? { nativeAlias: true } : {}), @@ -372,6 +382,7 @@ export function emptyDraft(id = ""): ComboItem { strategy: "failover", stickyLimit: 1, defaultEffort: null, + imageInput: "auto", targets: [newComboTarget()], }; } diff --git a/gui/src/components/combo-workspace-add-modal.tsx b/gui/src/components/combo-workspace-add-modal.tsx index cff828c5c8..4d334a759f 100644 --- a/gui/src/components/combo-workspace-add-modal.tsx +++ b/gui/src/components/combo-workspace-add-modal.tsx @@ -10,7 +10,7 @@ import { IconX } from "../icons"; import { useT } from "../i18n/shared"; import { Notice } from "../ui"; import type { ModelOption, ProviderOption } from "./combo-workspace-types"; -import { EffortSelect, StrategySeg, TargetEditor } from "./combo-workspace-controls"; +import { ComboCapabilities, EffortSelect, StrategySeg, TargetEditor } from "./combo-workspace-controls"; import { clampedNumberInput } from "./combo-workspace-utils"; export function AddComboModal({ @@ -204,6 +204,13 @@ export function AddComboModal({ {draft.strategy === "failover" ? t("cws.targets.failoverHint") : t("cws.targets.roundRobinHint")}

+ setDraft((d) => ({ ...d, ...patch }))} + />
diff --git a/gui/src/components/combo-workspace-controls.tsx b/gui/src/components/combo-workspace-controls.tsx index 8a7277fd48..0a7ecaa65d 100644 --- a/gui/src/components/combo-workspace-controls.tsx +++ b/gui/src/components/combo-workspace-controls.tsx @@ -1,8 +1,10 @@ import { useState } from "react"; import type { ComboEffort, ComboStrategy, ComboTarget } from "../combo-workspace-data"; +import { comboImagesSupported } from "../combo-capabilities"; import { COMBO_EFFORTS, newComboTarget } from "../combo-workspace-data"; import { IconArrowDown, IconArrowUp, IconGrip, IconPlus, IconTrash } from "../icons"; import { useT } from "../i18n/shared"; +import { Switch } from "../ui"; import { formatProviderDisplayName } from "../provider-icons"; import type { ModelOption, ProviderOption } from "./combo-workspace-types"; import { clampedNumberInput, enabledProviders, modelsForProvider } from "./combo-workspace-utils"; @@ -83,6 +85,49 @@ export function EffortSelect({ ); } + +export function ComboCapabilities({ + targets, + models, + imageInput, + disabled, + onChange, +}: { + targets: ComboTarget[]; + models: ModelOption[]; + imageInput: "auto" | "disabled"; + disabled?: boolean; + onChange: (patch: { imageInput?: "auto" | "disabled" }) => void; +}) { + const t = useT(); + const imagesSupported = comboImagesSupported(targets, models); + // Default: checked (auto) when supported; force off when any target lacks image. + const effectiveOn = imagesSupported && imageInput !== "disabled"; + + return ( +
+ {t("cws.capabilities")} +
+
+ {t("cws.capability.imageInput")} +

+ {imagesSupported ? t("cws.capability.imageInputHint") : t("cws.capability.imageInputUnavailable")} +

+
+ { + if (!imagesSupported) return; + onChange({ imageInput: imageInput === "auto" ? "disabled" : "auto" }); + }} + disabled={disabled || !imagesSupported} + label={t("cws.capability.imageInput")} + /> +
+
+ ); +} + export function TargetEditor({ targets, strategy, diff --git a/gui/src/components/combo-workspace-detail-panel.tsx b/gui/src/components/combo-workspace-detail-panel.tsx index 1c6f9e0593..16b266c513 100644 --- a/gui/src/components/combo-workspace-detail-panel.tsx +++ b/gui/src/components/combo-workspace-detail-panel.tsx @@ -12,7 +12,7 @@ import { IconChevron, IconTrash } from "../icons"; import { useT } from "../i18n/shared"; import { Notice } from "../ui"; import type { ModelOption, ProviderOption } from "./combo-workspace-types"; -import { EffortSelect, StrategySeg, TargetEditor } from "./combo-workspace-controls"; +import { ComboCapabilities, EffortSelect, StrategySeg, TargetEditor } from "./combo-workspace-controls"; import { clampedNumberInput } from "./combo-workspace-utils"; type DetailTab = "config" | "about"; @@ -81,7 +81,7 @@ export function DetailPanel({ const [msg, setMsg] = useState<{ ok: boolean; text: string } | null>(null); const [copied, setCopied] = useState(false); const dirty = !draftEquals(draft, baseline); - const baselineSyncKey = `${baseline.id}:${baseline.alias ?? ""}:${baseline.nativeAlias}:${baseline.displayName ?? ""}:${baseline.strategy}:${baseline.stickyLimit}:${baseline.defaultEffort}:${baseline.targets.map((t) => `${t.provider}/${t.model}:${t.weight ?? 1}`).join(",")}`; + const baselineSyncKey = `${baseline.id}:${baseline.alias ?? ""}:${baseline.nativeAlias}:${baseline.displayName ?? ""}:${baseline.strategy}:${baseline.stickyLimit}:${baseline.defaultEffort}:${baseline.imageInput ?? "auto"}:${baseline.targets.map((t) => `${t.provider}/${t.model}:${t.weight ?? 1}`).join(",")}`; const effortMap = useMemo(() => { const map = new Map(); for (const model of models) { @@ -354,6 +354,13 @@ export function DetailPanel({ {draft.strategy === "failover" ? t("cws.targets.failoverHint") : t("cws.targets.roundRobinHint")}

+ updateDraft((d) => ({ ...d, ...patch }))} + /> )} diff --git a/gui/src/components/combo-workspace-types.ts b/gui/src/components/combo-workspace-types.ts index 39870e90f2..c4dc1eac6c 100644 --- a/gui/src/components/combo-workspace-types.ts +++ b/gui/src/components/combo-workspace-types.ts @@ -13,6 +13,7 @@ export type ModelOption = { id: string; namespaced?: string; reasoningEfforts?: string[]; + inputModalities?: string[]; }; export interface ComboWorkspaceProps { diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index a53882bef9..0408e61082 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -1770,6 +1770,10 @@ export const de: Record = { "cws.field.defaultEffort": "Standard-Reasoning", "cws.field.defaultEffortNone": "Keine (Ziel-Standard)", "cws.field.defaultEffortHint": "Nur verwendet, wenn der Client keinen Reasoning-Aufwand sendet. Optionen sind die Schnittmenge der beworbenen Aufwände der gewählten Ziele.", + "cws.capability.imageInputUnavailable": "Erst verfügbar, wenn jedes gewählte Ziel Bildeingabe unterstützt.", + "cws.capability.imageInputHint": "Standardmäßig aktiv, wenn jedes Ziel Bilder unterstützt. Ausschalten für nur Text.", + "cws.capability.imageInput": "Bild / multimodal", + "cws.capabilities": "Fähigkeiten", "cws.field.defaultEffortUnsupported": "Dieser Aufwand liegt nicht in der gemeinsamen Leiter der Ziele — er wird zur Anfragezeit ignoriert oder angepasst.", "cws.field.defaultEffortUnsupportedOption": "nicht in der Schnittmenge", "cws.targets": "Ziele", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index 6d4347316f..1933e29d4b 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -1804,6 +1804,10 @@ export const en = { "cws.field.defaultEffort": "Default reasoning", "cws.field.defaultEffortNone": "None (target default)", "cws.field.defaultEffortHint": "Used only when the client omits reasoning effort. Options are the intersection of the selected targets' advertised efforts; targets without catalog effort metadata offer none.", + "cws.capability.imageInputUnavailable": "Unavailable until every selected target supports image input.", + "cws.capability.imageInputHint": "On by default when every target supports images. Turn off to accept text only.", + "cws.capability.imageInput": "Image / multimodal", + "cws.capabilities": "Capabilities", "cws.field.defaultEffortUnsupported": "This effort is not in the targets' common ladder — it will be ignored or snapped at request time.", "cws.field.defaultEffortUnsupportedOption": "not in intersection", "cws.targets": "Targets", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index 3770c07224..fce4f5196f 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -1838,6 +1838,10 @@ export const ja: Record = { "cws.field.defaultEffort": "デフォルトの推論", "cws.field.defaultEffortNone": "なし(ターゲットのデフォルト)", "cws.field.defaultEffortHint": "クライアントが推論負荷を省略した場合のみ使用されます。選択肢は選択ターゲットが広告する負荷の交差です。", + "cws.capability.imageInputUnavailable": "選択した全ターゲットが画像入力に対応すると有効になります。", + "cws.capability.imageInputHint": "全ターゲットが画像対応なら既定でオン。オフにするとテキストのみ。", + "cws.capability.imageInput": "画像 / マルチモーダル", + "cws.capabilities": "能力", "cws.field.defaultEffortUnsupported": "この負荷はターゲット共通の階段にありません — リクエスト時に無視またはスナップされます。", "cws.field.defaultEffortUnsupportedOption": "交差に含まれない", "cws.targets": "ターゲット", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index 1021288915..bfd58fffba 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -1797,6 +1797,10 @@ export const ko: Record = { "cws.field.defaultEffort": "기본 추론 수준", "cws.field.defaultEffortNone": "없음 (대상 기본값)", "cws.field.defaultEffortHint": "클라이언트가 추론 수준을 생략한 경우에만 사용합니다. 옵션은 선택한 대상이 광고하는 수준의 교집합입니다.", + "cws.capability.imageInputUnavailable": "선택한 모든 대상이 이미지 입력을 지원해야 사용할 수 있습니다.", + "cws.capability.imageInputHint": "모든 대상이 이미지를 지원하면 기본으로 켜집니다. 끄면 텍스트만 허용합니다.", + "cws.capability.imageInput": "이미지 / 멀티모달", + "cws.capabilities": "기능", "cws.field.defaultEffortUnsupported": "이 수준은 대상의 공통 사다리에 없습니다 — 요청 시 무시되거나 스냅됩니다.", "cws.field.defaultEffortUnsupportedOption": "교집합에 없음", "cws.targets": "대상", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index 2f9fc958d1..ceffb75922 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -1880,6 +1880,10 @@ export const ru: Record = { "cws.field.defaultEffort": "Рассуждения по умолчанию", "cws.field.defaultEffortNone": "Нет (по умолчанию для цели)", "cws.field.defaultEffortHint": "Используется, только если клиент не указал уровень рассуждений. Варианты — пересечение заявленных уровней выбранных целей.", + "cws.capability.imageInputUnavailable": "Доступно, когда все выбранные цели поддерживают ввод изображений.", + "cws.capability.imageInputHint": "Включено по умолчанию, если все цели поддерживают изображения. Выключите, чтобы принимать только текст.", + "cws.capability.imageInput": "Изображения / мультимодальность", + "cws.capabilities": "Возможности", "cws.field.defaultEffortUnsupported": "Этот уровень не входит в общую лестницу целей — при запросе он будет проигнорирован или снижен.", "cws.field.defaultEffortUnsupportedOption": "нет в пересечении", "cws.targets": "Цели", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index 2ae8b6ac2c..b9869d0c8c 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -1790,6 +1790,10 @@ export const zh: Record = { "cws.field.defaultEffort": "默认推理级别", "cws.field.defaultEffortNone": "无（使用目标默认）", "cws.field.defaultEffortHint": "仅在客户端未指定推理级别时使用。选项为所选目标已公布努力级别的交集。", + "cws.capability.imageInputUnavailable": "所有已选目标均支持图片输入后才可用。", + "cws.capability.imageInputHint": "所有目标均支持图片时默认开启；关闭后仅接受文本。", + "cws.capability.imageInput": "图片 / 多模态", + "cws.capabilities": "能力", "cws.field.defaultEffortUnsupported": "该级别不在目标的公共阶梯中 — 请求时会被忽略或就近映射。", "cws.field.defaultEffortUnsupportedOption": "不在交集中", "cws.targets": "目标", diff --git a/gui/src/pages/Combos.tsx b/gui/src/pages/Combos.tsx index 7b4fc86d67..691ba1146c 100644 --- a/gui/src/pages/Combos.tsx +++ b/gui/src/pages/Combos.tsx @@ -21,7 +21,7 @@ type ProviderOption = { adapter?: string; baseUrl?: string; }; -type ModelOption = { provider: string; id: string; namespaced?: string; reasoningEfforts?: string[] }; +type ModelOption = { provider: string; id: string; namespaced?: string; reasoningEfforts?: string[]; inputModalities?: string[] }; type ProviderDto = { adapter: string; baseUrl: string; @@ -148,6 +148,7 @@ export default function Combos({ namespaced?: unknown; disabled?: unknown; reasoningEfforts?: unknown; + inputModalities?: unknown; }; if (typeof model.provider !== "string" || typeof model.id !== "string") continue; const provider = model.provider.trim(); @@ -161,11 +162,18 @@ export default function Combos({ const reasoningEfforts = Array.isArray(model.reasoningEfforts) ? model.reasoningEfforts.filter((effort): effort is string => typeof effort === "string") : undefined; + const inputModalities = Array.isArray(model.inputModalities) + ? model.inputModalities + .filter((modality): modality is string => typeof modality === "string") + .map((modality) => modality.trim()) + .filter(Boolean) + : undefined; models.push({ provider, id, namespaced: typeof model.namespaced === "string" ? model.namespaced : undefined, ...(reasoningEfforts ? { reasoningEfforts } : {}), + ...(inputModalities && inputModalities.length > 0 ? { inputModalities } : {}), }); } diff --git a/gui/src/styles-combos-workspace.css b/gui/src/styles-combos-workspace.css index 58279c128f..837f362d2f 100644 --- a/gui/src/styles-combos-workspace.css +++ b/gui/src/styles-combos-workspace.css @@ -322,6 +322,34 @@ overflow-wrap: anywhere; } +.cwi-capabilities { + display: flex; + flex-direction: column; + gap: 10px; + padding: 12px; + border: 1px solid var(--border-soft); + border-radius: var(--radius); + background: var(--raised); +} + +.cwi-capability-row { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} + +.cwi-capability-label { + font-size: 13px; + font-weight: 500; +} + +.cwi-capability-hint { + margin: 3px 0 0; + font-size: 12px; +} + + .cwi-target-list { display: flex; flex-direction: column; @@ -489,6 +517,10 @@ border-bottom: 1px solid var(--border); } + .cwi-capability-row { + align-items: flex-start; + } + .cwi-target-row, .cwi-target-row--failover { grid-template-columns: 28px auto 1fr auto; diff --git a/src/codex/catalog/aggregation.ts b/src/codex/catalog/aggregation.ts index 1031f073a1..7fb2c04892 100644 --- a/src/codex/catalog/aggregation.ts +++ b/src/codex/catalog/aggregation.ts @@ -129,9 +129,13 @@ export function deriveComboCatalogModel( ): CatalogModel | null { if (comboCatalogOmissionReason(combo, members) !== null) return null; - const inputModalities = intersectStrings( + const derivedInputModalities = intersectStrings( members.map(member => member.inputModalities ?? ["text"]), ); + const inputModalities = combo.imageInput === "disabled" + ? derivedInputModalities.filter(modality => modality !== "image") + : derivedInputModalities; + if (inputModalities.length === 0) return null; // Unknown ladders (`undefined`) are wildcards for catalog derivation — same // boundary as the GUI picker. An explicit empty ladder still constrains. const advertisedLadders = members diff --git a/src/combos/index.ts b/src/combos/index.ts index 6041427a92..571eb540d5 100644 --- a/src/combos/index.ts +++ b/src/combos/index.ts @@ -39,6 +39,7 @@ export { } from "./failover"; export { comboIdFromRawBody, + comboRequestHasImageInput, concreteComboRequestBody, resetComboEffortWarningStateForTests, } from "./request"; diff --git a/src/combos/request.ts b/src/combos/request.ts index 7727b7689b..2b198aae7a 100644 --- a/src/combos/request.ts +++ b/src/combos/request.ts @@ -23,6 +23,36 @@ export function comboIdFromRawBody(body: unknown, config: OcxConfig): string | n return resolveComboId(config, model); } +/** + * Detect image-bearing Responses *input* only. + * + * Must not walk the full request body: tool JSON schemas, metadata, or extension + * payloads can legally contain `{ "type": "input_image" }` without any image + * being dispatched. After previous_response_id expansion, scan the materialised + * `input` tree (message content and function_call_output.output). + */ +export function comboRequestHasImageInput(body: unknown): boolean { + if (!body || typeof body !== "object" || Array.isArray(body)) return false; + return responsesInputHasImage((body as { input?: unknown }).input); +} + +function responsesInputHasImage(input: unknown): boolean { + if (typeof input === "string" || input == null) return false; + if (!Array.isArray(input)) return false; + return input.some(responsesInputNodeHasImage); +} + +function responsesInputNodeHasImage(value: unknown): boolean { + if (!value || typeof value !== "object") return false; + if (Array.isArray(value)) return value.some(responsesInputNodeHasImage); + const record = value as Record; + if (record.type === "input_image") return true; + // Message content parts and nested function_call_output content/output arrays. + if (record.content !== undefined && responsesInputNodeHasImage(record.content)) return true; + if (record.output !== undefined && responsesInputNodeHasImage(record.output)) return true; + return false; +} + export function concreteComboRequestBody( body: unknown, target: Pick, diff --git a/src/combos/types.ts b/src/combos/types.ts index c82c861a02..d1ac034096 100644 --- a/src/combos/types.ts +++ b/src/combos/types.ts @@ -37,6 +37,8 @@ export interface NormalizedComboConfig { strategy: OcxComboStrategy; stickyLimit: number; defaultEffort: OcxComboDefaultEffort | null; + /** Disable image input; `auto` preserves the intersection derived from all targets. */ + imageInput: "auto" | "disabled"; /** Trimmed public alias, or null when the combo keeps the default `combo/` slug. */ alias: string | null; /** Explicit native-family alias opt-in. */ @@ -220,6 +222,9 @@ export function comboConfigIssues( message: "defaultEffort must be one of: low, medium, high, xhigh, max, ultra", }); } + if (body.imageInput !== undefined && body.imageInput !== "auto" && body.imageInput !== "disabled") { + issues.push({ path: ["imageInput"], message: 'imageInput must be "auto" or "disabled"' }); + } if (body.alias !== undefined) { if (typeof body.alias !== "string") { @@ -339,6 +344,7 @@ export function normalizeComboConfig(raw: OcxComboConfig): NormalizedComboConfig strategy: raw.strategy ?? "failover", stickyLimit: raw.stickyLimit ?? 1, defaultEffort: raw.defaultEffort ?? null, + imageInput: raw.imageInput === "disabled" ? "disabled" : "auto", alias: alias || null, nativeAlias: raw.nativeAlias === true, displayName: displayName || null, diff --git a/src/server/management/combo-routes.ts b/src/server/management/combo-routes.ts index 9c197d0d74..d347bef449 100644 --- a/src/server/management/combo-routes.ts +++ b/src/server/management/combo-routes.ts @@ -65,6 +65,16 @@ import type { MetricUnavailableReason, TokPerSecondResult, CostEstimateReason, C import type { ManagementContext } from "./context"; import { readManagementJsonBody, rethrowManagementBodyTooLarge } from "./body"; + +/** Management wire shape: omit default imageInput "auto" (persist/response sparse). */ +function sparseComboConfig(combo: T): Omit & { imageInput?: "disabled" } { + const { imageInput, ...rest } = combo; + return { + ...rest, + ...(imageInput === "disabled" ? { imageInput: "disabled" as const } : {}), + }; +} + export async function handleComboRoutes(ctx: ManagementContext): Promise { const { req, url, config, deps, convergeCodexCatalog, syncClaudeAgentDefsBestEffort } = ctx; @@ -75,7 +85,7 @@ export async function handleComboRoutes(ctx: ManagementContext): Promise 0 && body === rawBody; + if (combo.imageInput === "disabled" && unresolvedPrevious) { + return formatErrorResponse( + 400, + "previous_response_not_found", + "Continuation state is unavailable or corrupt; resend the full conversation without previous_response_id.", + ); + } + if (combo.imageInput === "disabled" && comboRequestHasImageInput(body)) { + return formatErrorResponse(400, "invalid_request_error", `Combo "${comboId}" does not accept image input`); + } + // Expansion already materialised prior input. Drop the id so the child + // handleResponses path does not expand again and double-prepend history. + if (body !== rawBody && body && typeof body === "object" && !Array.isArray(body)) { + delete (body as Record).previous_response_id; + } const adoptFailedChildLog = (childLog: RequestLogContext): void => { // Attempts remain the complete physical history; the logical row mirrors the most recent // failed target so an exhausted combo still has useful top-level reasoning diagnostics. @@ -1083,7 +1119,7 @@ export async function handleComboResponses( }; const unreadableEncryptedAgentTask = hasUnreadableEncryptedAgentTask( - (rawBody as { input?: unknown } | undefined)?.input, + (body as { input?: unknown } | undefined)?.input, ); const canDecryptUnreadableAgentTask = (target: (typeof combo.targets)[number]): boolean => { const provider = config.providers[target.provider]; @@ -1125,7 +1161,7 @@ export async function handleComboResponses( }; const targetRoute = routeConcreteModel(config, `${pick.target.provider}/${pick.target.model}`); const childBody = concreteComboRequestBody( - rawBody, + body, pick.target, comboDefaultEffort(config, comboId), supportedLadderFor({ provider: targetRoute.provider, modelId: targetRoute.modelId }), diff --git a/src/types.ts b/src/types.ts index fb45843a4d..b90a2d93e7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -923,6 +923,12 @@ export interface OcxComboConfig { stickyLimit?: number; /** Used when the client omits reasoning.effort. null/omitted leaves the target default unchanged. */ defaultEffort?: OcxComboDefaultEffort | null; + /** + * Disable image input even when every target supports it. + * Omitted / `"auto"` keeps automatic capability derivation (default: enabled when + * the target intersection includes image). + */ + imageInput?: "auto" | "disabled"; /** * Optional public model name replacing the default `combo/` slug. Bare names * without "/" are allowed (e.g. "deepseek-v4-flash") so the combo can answer to a diff --git a/tests/codex-catalog.test.ts b/tests/codex-catalog.test.ts index d654fd8960..4b7b0ecc86 100644 --- a/tests/codex-catalog.test.ts +++ b/tests/codex-catalog.test.ts @@ -66,6 +66,7 @@ function normalizedCombo( strategy: "failover", stickyLimit: 1, defaultEffort: "medium", + imageInput: "auto", alias: null, nativeAlias: false, displayName: null, @@ -154,6 +155,18 @@ describe("live model provenance (#448 custom-model misclassification)", () => { }); describe("combo catalog capability intersection", () => { + + test("imageInput disabled strips image even when every member supports it", () => { + const visionMembers = [ + { provider: "a", id: "m1", contextWindow: 128_000, maxInputTokens: 100_000, inputModalities: ["text", "image"], reasoningEfforts: ["low"] }, + { provider: "b", id: "m2", contextWindow: 128_000, maxInputTokens: 100_000, inputModalities: ["text", "image"], reasoningEfforts: ["low"] }, + ]; + expect(deriveComboCatalogModel("text-only", normalizedCombo({ imageInput: "disabled" }), visionMembers)) + .toEqual(expect.objectContaining({ inputModalities: ["text"] })); + expect(deriveComboCatalogModel("vision", normalizedCombo({ imageInput: "auto" }), visionMembers)) + .toEqual(expect.objectContaining({ inputModalities: expect.arrayContaining(["text", "image"]) })); + }); + const memberA = { provider: "a", id: "m1", diff --git a/tests/combo-management-api.test.ts b/tests/combo-management-api.test.ts index 68d439a2a6..75c19f8916 100644 --- a/tests/combo-management-api.test.ts +++ b/tests/combo-management-api.test.ts @@ -268,6 +268,31 @@ describe("combo management API", () => { const listed = await responseJson(await comboApi(config, "GET", "/api/combos")); expect((listed.combos as Array<{ id: string }>).map(row => row.id)).toEqual(["alpha", "zeta"]); expect(listComboIds(config)).toEqual(["alpha", "zeta"]); + // Default imageInput is not written to disk — only explicit "disabled" is. + expect(config.combos?.zeta).not.toHaveProperty("imageInput"); + }); + }); + + test("PUT persists explicit imageInput disabled", async () => { + await withTempHome(async () => { + const config = baseConfig({ combos: undefined }); + saveConfig(config); + const response = await comboApi(config, "PUT", "/api/combos", { + id: "limited", + combo: { + targets: [{ provider: "a", model: "m1" }], + imageInput: "disabled", + }, + }); + expect(response?.status).toBe(200); + expect(await responseJson(response)).toMatchObject({ + combo: { imageInput: "disabled" }, + }); + expect(config.combos?.limited).toMatchObject({ imageInput: "disabled" }); + const listed = await responseJson(await comboApi(config, "GET", "/api/combos")); + expect(listed.combos).toEqual([expect.objectContaining({ + id: "limited", imageInput: "disabled", + })]); }); }); diff --git a/tests/combo-workspace-data.test.ts b/tests/combo-workspace-data.test.ts index c67fd4cac3..d621375162 100644 --- a/tests/combo-workspace-data.test.ts +++ b/tests/combo-workspace-data.test.ts @@ -15,6 +15,7 @@ import { updateComboAliasDraft, validateComboDraft, } from "../gui/src/combo-workspace-data"; +import { comboImagesSupported } from "../gui/src/combo-capabilities"; const configuredProviders = { a: {}, @@ -95,6 +96,7 @@ describe("combo-workspace-data", () => { strategy: "failover", stickyLimit: 1, defaultEffort: null, + imageInput: "auto", targets: [{ provider: "a", model: "m1", weight: 1, clientKey: expect.stringMatching(/^ct-\d+$/) }], }, { @@ -106,6 +108,7 @@ describe("combo-workspace-data", () => { strategy: "round-robin", stickyLimit: 4, defaultEffort: "high", + imageInput: "auto", targets: [ { provider: "a", model: "m1", weight: 3, clientKey: expect.stringMatching(/^ct-\d+$/) }, { provider: "b", model: "m2", weight: 1, clientKey: expect.stringMatching(/^ct-\d+$/) }, @@ -530,3 +533,70 @@ describe("combo-workspace-data", () => { )).toBe(false); }); }); + + +describe("comboImagesSupported", () => { + test("returns false with no targets or incomplete targets", () => { + expect(comboImagesSupported([], [])).toBe(false); + expect(comboImagesSupported([{ provider: "", model: "" }], [])).toBe(false); + expect(comboImagesSupported( + [{ provider: "a", model: "vision" }, { provider: "", model: "" }], + [{ provider: "a", id: "vision", inputModalities: ["text", "image"] }], + )).toBe(false); + }); + + test("returns true only when every complete target advertises image", () => { + const models = [ + { provider: "a", id: "m1", inputModalities: ["text", "image"] }, + { provider: "b", id: "m2", inputModalities: ["text", "image"] }, + ]; + expect(comboImagesSupported( + [{ provider: "a", model: "m1" }, { provider: "b", model: "m2" }], + models, + )).toBe(true); + }); + + test("returns false when any target is missing from the catalog or lacks image", () => { + const models = [ + { provider: "a", id: "m1", inputModalities: ["text", "image"] }, + { provider: "b", id: "m2", inputModalities: ["text"] }, + ]; + expect(comboImagesSupported( + [{ provider: "a", model: "m1" }, { provider: "b", model: "m2" }], + models, + )).toBe(false); + expect(comboImagesSupported( + [{ provider: "a", model: "m1" }, { provider: "b", model: "ghost" }], + models, + )).toBe(false); + }); +}); + +describe("combo imageInput draft persistence", () => { + test("parseComboList preserves explicit disabled", () => { + const items = parseComboList({ + combos: [{ + id: "limited", + strategy: "failover", + imageInput: "disabled", + targets: [{ provider: "a", model: "m1" }], + }], + }); + expect(items[0]?.imageInput).toBe("disabled"); + }); + + test("draftEquals distinguishes disabled from auto", () => { + const base = emptyDraft("x"); + const disabled = { ...base, imageInput: "disabled" as const }; + expect(draftEquals(base, { ...base, imageInput: "auto" })).toBe(true); + expect(draftEquals(base, disabled)).toBe(false); + }); + + test("toPutBody emits imageInput only when disabled", () => { + const auto = emptyDraft("x"); + auto.targets = [{ provider: "a", model: "m1" }]; + expect(toPutBody(auto).combo).not.toHaveProperty("imageInput"); + const disabled = { ...auto, imageInput: "disabled" as const }; + expect(toPutBody(disabled).combo.imageInput).toBe("disabled"); + }); +}); diff --git a/tests/combos.test.ts b/tests/combos.test.ts index e46f10a8d3..b20f65efc7 100644 --- a/tests/combos.test.ts +++ b/tests/combos.test.ts @@ -16,6 +16,7 @@ import { comboIdFromRawBody, comboModelId, comboPublicModelId, + comboRequestHasImageInput, concreteComboRequestBody, coolComboTarget, getCombo, @@ -214,6 +215,48 @@ describe("combo request cloning", () => { expect(comboIdFromRawBody(null, config)).toBeNull(); }); + test("comboRequestHasImageInput scans Responses input only, not tools or metadata", () => { + expect(comboRequestHasImageInput({ + model: "combo/free", + input: [{ role: "user", content: [{ type: "input_image", image_url: "data:image/png;base64,aGVsbG8=" }] }], + })).toBe(true); + expect(comboRequestHasImageInput({ + model: "combo/free", + input: [{ type: "input_image", image_url: "https://example.test/i.png" }], + })).toBe(true); + expect(comboRequestHasImageInput({ + model: "combo/free", + input: [{ + type: "function_call_output", + call_id: "call_1", + output: [{ type: "input_image", image_url: "https://example.test/tool.png" }], + }], + })).toBe(true); + // Tool schemas / metadata may legally mention the same type string without + // carrying image content for the model. + expect(comboRequestHasImageInput({ + model: "combo/free", + input: [{ role: "user", content: "text only" }], + tools: [{ + type: "function", + name: "describe", + parameters: { + type: "object", + properties: { + kind: { type: "string", enum: ["input_image", "input_text"] }, + example: { type: "input_image" }, + }, + }, + }], + metadata: { note: { type: "input_image" } }, + })).toBe(false); + expect(comboRequestHasImageInput({ + model: "combo/free", + input: "plain text", + tools: [{ type: "function", function: { name: "x", parameters: { type: "input_image" } } }], + })).toBe(false); + }); + test("clones the untouched body and injects an omitted combo default", () => { const raw = { model: "combo/free", input: [{ role: "user", content: "hi" }] }; const concrete = concreteComboRequestBody(raw, target, "high", ["low", "high"]); @@ -589,6 +632,7 @@ describe("combo validation and normalization", () => { strategy: "failover", stickyLimit: 1, defaultEffort: "high", + imageInput: "auto", alias: null, nativeAlias: false, displayName: null, diff --git a/tests/server-combo-failover-e2e.test.ts b/tests/server-combo-failover-e2e.test.ts index 311d672507..36b304deaa 100644 --- a/tests/server-combo-failover-e2e.test.ts +++ b/tests/server-combo-failover-e2e.test.ts @@ -1304,6 +1304,105 @@ describe("server combo failover 030 activation matrix", () => { expect(bHits).toBe(2); }); + test("disabled image input rejects the request before any combo target is called", async () => { + let hits = 0; + const a = serve(() => { + hits += 1; + return chatSuccess("unexpected", "m1"); + }); + const config = comboConfig({ a: provider("openai-chat", baseUrl(a), "key-a") }, undefined, { + imageInput: "disabled", + }); + const response = await post(config, { + input: [{ role: "user", content: [{ type: "input_image", image_url: "data:image/png;base64,aGVsbG8=" }] }], + }); + expect(response.status).toBe(400); + expect(await response.text()).toContain("does not accept image input"); + expect(hits).toBe(0); + }); + + test("disabled image input ignores tool schemas that only mention input_image", async () => { + let hits = 0; + const bodies: Array> = []; + const a = serve(async request => { + hits += 1; + bodies.push(await request.json() as Record); + return chatSuccess("text only", "m1"); + }); + const config = comboConfig({ a: provider("openai-chat", baseUrl(a), "key-a") }, undefined, { + imageInput: "disabled", + }); + const response = await post(config, { + input: [{ role: "user", content: "describe without images" }], + tools: [{ + type: "function", + name: "classify", + parameters: { + type: "object", + properties: { + part: { type: "string", enum: ["input_image", "input_text"] }, + example: { type: "input_image" }, + }, + }, + }], + metadata: { sample: { type: "input_image" } }, + }); + expect(response.status).toBe(200); + expect(hits).toBe(1); + // openai-chat upstream receives the bare model id after concrete routing. + expect(bodies[0]?.model).toBe("m1"); + }); + + test("disabled image input rejects unavailable previous_response_id before dispatch", async () => { + let hits = 0; + const a = serve(() => { + hits += 1; + return chatSuccess("unexpected", "m1"); + }); + const config = comboConfig({ a: provider("openai-chat", baseUrl(a), "key-a") }, undefined, { + imageInput: "disabled", + }); + const response = await post(config, { + previous_response_id: "resp_missing_local_state", + input: [{ role: "user", content: "continue" }], + }); + expect(response.status).toBe(400); + expect(await response.text()).toContain("Continuation state is unavailable"); + expect(hits).toBe(0); + }); + + test("disabled image input expands text-only previous_response_id and dispatches once", async () => { + const { rememberResponseState } = await import("../src/responses/state"); + rememberResponseState( + { model: "combo/free", input: [{ role: "user", content: "earlier text" }] }, + { + id: "resp_combo_text_prev", + status: "completed", + output: [{ type: "message", role: "assistant", content: "ack" }], + }, + ); + const bodies: Array> = []; + const a = serve(async request => { + bodies.push(await request.json() as Record); + return chatSuccess("continued", "m1"); + }); + const config = comboConfig({ a: provider("openai-chat", baseUrl(a), "key-a") }, undefined, { + imageInput: "disabled", + }); + const response = await post(config, { + previous_response_id: "resp_combo_text_prev", + input: [{ role: "user", content: "next turn" }], + }); + expect(response.status).toBe(200); + expect(bodies).toHaveLength(1); + const child = bodies[0]!; + // Parent already expanded; child must not keep previous_response_id (would double-prepend). + expect(child.previous_response_id).toBeUndefined(); + const inputText = JSON.stringify(child.input ?? child.messages ?? child); + expect(inputText).toContain("earlier text"); + expect(inputText).toContain("next turn"); + }); + test("fresh child reparsing recomputes vision and effort per target", async () => { const bodies: Array<{ provider: string; body: Record }> = []; const a = serve(async request => {