diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cff29e0..d91f3d1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ title: Changelog description: Release notes for claude-code-proxy. --- +## Unreleased + +- Grok handles images in user messages and tool results without failing requests. + Images are omitted by default, with opt-in vision through + `CCP_GROK_TOOL_IMAGE`. Traffic captures redact image payloads. + ## v0.1.25 (2026-07-24) - Kimi users can select Kimi K3 with the `kimi-k3` or `k3` model name, including diff --git a/docs/src/content/docs/providers/grok.md b/docs/src/content/docs/providers/grok.md index 342e8f51..d634db8b 100644 --- a/docs/src/content/docs/providers/grok.md +++ b/docs/src/content/docs/providers/grok.md @@ -44,12 +44,28 @@ Claude Code hosted search tools map to Grok-native tools: ## Multimodal support -Grok support in claude-code-proxy focuses on text, reasoning, function tools, and hosted search. Treat image and other media behavior as unsupported unless the current source and tests explicitly cover it. +`CCP_GROK_TOOL_IMAGE` controls image blocks in user messages and tool results: + +- `omit`, the default, replaces each image with an `[image omitted: ...]` + placeholder. The model does not receive the pixels. +- `reattach` keeps the placeholder in each tool result and sends accepted images + in a following user message. +- `inline` sends accepted tool-result images alongside text as `input_image` + parts. Text-only outputs retain their string shape. +- `reject` returns the image validation error used by older versions. + +Vision modes accept base64 PNG, JPEG, and GIF images with a minimum side of 8 +pixels, a minimum area of 512 square pixels, and a decoded size up to 5 MB. At +most the last four accepted images in a request are sent. Images that fail a +gate, WebP images, and remote URL sources degrade to placeholders with a reason. + +Traffic captures redact Anthropic image data and upstream image data URLs. ## Configuration - `CCP_GROK_BASE_URL` or `grok.baseUrl` changes the API base URL. - `CCP_GROK_CLIENT_VERSION` or `grok.clientVersion` changes the client version header. +- `CCP_GROK_TOOL_IMAGE` selects `omit`, `reattach`, `inline`, or `reject`. See [Configuration](/reference/configuration/) for defaults. diff --git a/docs/src/content/docs/reference/configuration.md b/docs/src/content/docs/reference/configuration.md index dde63afc..66cdb8cb 100644 --- a/docs/src/content/docs/reference/configuration.md +++ b/docs/src/content/docs/reference/configuration.md @@ -98,6 +98,7 @@ All keys are optional. An unreadable file, malformed JSON, or incompatible field | --- | --- | --- | --- | | `CCP_GROK_BASE_URL` | `grok.baseUrl` | `https://cli-chat-proxy.grok.com/v1` | Changes the Responses API base URL. | | `CCP_GROK_CLIENT_VERSION` | `grok.clientVersion` | `0.2.93` | Changes the Grok client version header. | +| `CCP_GROK_TOOL_IMAGE` | none | `omit` | Selects `omit`, `reattach`, `inline`, or `reject` image handling. | ## Cursor Agent diff --git a/src/config.rs b/src/config.rs index f345d9ae..595d75fe 100644 --- a/src/config.rs +++ b/src/config.rs @@ -316,6 +316,62 @@ pub fn grok_client_version() -> String { "0.2.93".to_string() } +// --------------------------------------------------------------------------- +// Grok tool-image policy (CCP_GROK_TOOL_IMAGE) +// --------------------------------------------------------------------------- + +/// How the Grok translator treats Anthropic `image` blocks (tool results and +/// top-level user messages). `omit` is the safe default: degrade to the L1 +/// placeholder string. `reattach` keeps the placeholder in the tool output and +/// additionally appends a user message carrying the images as `input_image` +/// data URLs. `inline` sends the tool output itself as an array of +/// `input_text` + `input_image` parts (string-only outputs still serialize as +/// plain strings). `reject` restores the pre-L1 hard error. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GrokToolImageMode { + Omit, + Reattach, + Inline, + Reject, +} + +pub fn parse_grok_tool_image_mode(raw: Option<&str>) -> GrokToolImageMode { + match raw.map(str::trim) { + Some("reattach") => GrokToolImageMode::Reattach, + Some("inline") => GrokToolImageMode::Inline, + Some("reject") => GrokToolImageMode::Reject, + // Any unknown/empty value degrades to the safe default. + _ => GrokToolImageMode::Omit, + } +} + +pub fn grok_tool_image_mode() -> GrokToolImageMode { + parse_grok_tool_image_mode(std::env::var("CCP_GROK_TOOL_IMAGE").ok().as_deref()) +} + +/// Warn once at startup when an unknown mode was requested. Called from the +/// Grok provider constructor rather than per request. +pub fn warn_grok_tool_image_mode_once(log: &crate::logging::Logger) { + match std::env::var("CCP_GROK_TOOL_IMAGE") + .ok() + .as_deref() + .map(str::trim) + { + Some(other) if !matches!(other, "" | "omit" | "reattach" | "inline" | "reject") => { + let mut fields = serde_json::Map::new(); + fields.insert( + "value".to_string(), + serde_json::Value::String(other.to_string()), + ); + log.warn( + "unrecognized CCP_GROK_TOOL_IMAGE value; falling back to omit", + Some(fields), + ); + } + _ => {} + } +} + pub fn is_verbose() -> bool { log_verbose() } diff --git a/src/providers/grok/count_tokens.rs b/src/providers/grok/count_tokens.rs index b456e6e2..9f586497 100644 --- a/src/providers/grok/count_tokens.rs +++ b/src/providers/grok/count_tokens.rs @@ -1,4 +1,6 @@ -use super::translate::request::{GrokContentPart, GrokInputItem, GrokResponsesRequest}; +use super::translate::request::{ + GrokContentPart, GrokInputItem, GrokResponsesRequest, GrokToolOutput, +}; const MESSAGE_OVERHEAD_TOKENS: u64 = 4; const TOOL_OVERHEAD_TOKENS: u64 = 4; @@ -37,21 +39,33 @@ pub fn count_tokens(request: &GrokResponsesRequest) -> u64 { fn count_input_item(item: &GrokInputItem) -> u64 { match item { - GrokInputItem::Message { content, .. } => content - .iter() - .map(|part| match part { - GrokContentPart::InputText { text } | GrokContentPart::OutputText { text } => { - approx_token_count(text) - } - }) - .sum(), + GrokInputItem::Message { content, .. } => count_parts(content), GrokInputItem::FunctionCall { name, arguments, .. } => approx_token_count(name) + approx_token_count(arguments), - GrokInputItem::FunctionCallOutput { output, .. } => approx_token_count(output), + GrokInputItem::FunctionCallOutput { output, .. } => match output { + GrokToolOutput::Text(text) => approx_token_count(text), + GrokToolOutput::Parts(parts) => count_parts(parts), + }, } } +fn count_parts(parts: &[GrokContentPart]) -> u64 { + parts + .iter() + .map(|part| match part { + GrokContentPart::InputText { text } | GrokContentPart::OutputText { text } => { + approx_token_count(text) + } + // Images count toward tokens; charge a flat per-image estimate + // instead of counting base64 payload characters. + GrokContentPart::InputImage { .. } => IMAGE_TOKEN_ESTIMATE, + }) + .sum() +} + +const IMAGE_TOKEN_ESTIMATE: u64 = 1_024; + fn approx_token_count(text: &str) -> u64 { if text.is_empty() { return 0; diff --git a/src/providers/grok/mod.rs b/src/providers/grok/mod.rs index fa6d2692..94eb93c7 100644 --- a/src/providers/grok/mod.rs +++ b/src/providers/grok/mod.rs @@ -38,6 +38,7 @@ pub struct GrokProvider { } impl GrokProvider { pub fn new() -> Self { + crate::config::warn_grok_tool_image_mode_once(&crate::logging::create_logger("grok")); Self { client: Arc::new( client::GrokClient::new( diff --git a/src/providers/grok/translate/request.rs b/src/providers/grok/translate/request.rs index 82a026f2..15c4ac2e 100644 --- a/src/providers/grok/translate/request.rs +++ b/src/providers/grok/translate/request.rs @@ -4,6 +4,8 @@ use serde::Serialize; use serde_json::Value; use crate::anthropic::schema::{Message, MessagesRequest}; +use crate::config::GrokToolImageMode; +use crate::providers::translate_shared::{ImageSource, image_source_to_url}; #[derive(Debug, Clone, Serialize)] pub struct GrokResponsesRequest { @@ -36,7 +38,20 @@ pub enum GrokInputItem { arguments: String, }, #[serde(rename = "function_call_output")] - FunctionCallOutput { call_id: String, output: String }, + FunctionCallOutput { + call_id: String, + output: GrokToolOutput, + }, +} + +/// Tool output payload: a plain string, or (in `inline` image mode) an array +/// of `input_text` + `input_image` parts. Untagged so string outputs serialize +/// byte-identically to the pre-inline shape. +#[derive(Debug, Clone, Serialize)] +#[serde(untagged)] +pub enum GrokToolOutput { + Text(String), + Parts(Vec), } #[derive(Debug, Clone, Serialize)] @@ -46,6 +61,8 @@ pub enum GrokContentPart { InputText { text: String }, #[serde(rename = "output_text")] OutputText { text: String }, + #[serde(rename = "input_image")] + InputImage { image_url: String }, } #[derive(Debug, Clone, Serialize)] @@ -108,6 +125,14 @@ pub enum GrokToolChoice { pub fn translate_request( req: &MessagesRequest, model: String, +) -> anyhow::Result { + translate_request_with_mode(req, model, crate::config::grok_tool_image_mode()) +} + +pub fn translate_request_with_mode( + req: &MessagesRequest, + model: String, + image_mode: GrokToolImageMode, ) -> anyhow::Result { reject_unknown_top_level(req)?; let mut instructions = parse_system(req.extra.get("system"))?; @@ -148,8 +173,9 @@ pub fn translate_request( }; let mut call_ids = HashSet::new(); let mut input = Vec::new(); + let mut budget = ReattachBudget::new(&req.messages, image_mode); for message in &req.messages { - parse_message(message, &mut input, &mut call_ids)?; + parse_message(message, &mut input, &mut call_ids, image_mode, &mut budget)?; } Ok(GrokResponsesRequest { model, @@ -163,6 +189,86 @@ pub fn translate_request( }) } +/// Pre-scans the request to decide which gate-passing images survive the +/// request-wide "keep only the last few" cap, so the main walk can mark +/// cap-dropped images with a reason instead of silently dropping pixels. +struct ReattachBudget { + /// Per-image-URL count of gate-passing occurrences that must be dropped + /// (the oldest ones) before survivors begin, so the *last* few images win. + drops: std::collections::HashMap, +} + +impl ReattachBudget { + fn new(messages: &[Message], image_mode: GrokToolImageMode) -> Self { + let mut drops: std::collections::HashMap = std::collections::HashMap::new(); + if matches!( + image_mode, + GrokToolImageMode::Reattach | GrokToolImageMode::Inline + ) { + let passing: Vec = messages + .iter() + .flat_map(candidate_image_blocks) + .filter_map(|block| { + let source = parse_image_source(block)?; + gate_image(&source) + .ok() + .map(|()| image_source_to_url(&source)) + }) + .collect(); + // Drop the oldest occurrences beyond the cap, keeping the last few. + for url in passing + .iter() + .take(passing.len().saturating_sub(MAX_REATTACHED_IMAGES)) + { + *drops.entry(url.clone()).or_insert(0) += 1; + } + } + Self { drops } + } + + /// Record a gate-passing image; `true` when it survives the request cap. + /// Oldest occurrences are dropped first, in conversation order. + fn admit(&mut self, image_url: &str) -> bool { + let Some(drops) = self.drops.get_mut(image_url) else { + return true; + }; + *drops -= 1; + if *drops == 0 { + self.drops.remove(image_url); + } + false + } +} + +/// Every image block in one message: top-level user images and tool_result +/// image children, in conversation order. +fn candidate_image_blocks(message: &Message) -> Vec<&serde_json::Map> { + let mut blocks = Vec::new(); + if let Value::Array(items) = &message.content { + for item in items { + let Some(object) = item.as_object() else { + continue; + }; + match object.get("type").and_then(Value::as_str) { + Some("image") => blocks.push(object), + Some("tool_result") => { + if let Some(Value::Array(parts)) = object.get("content") { + for part in parts { + if let Some(part) = part.as_object() + && part.get("type").and_then(Value::as_str) == Some("image") + { + blocks.push(part); + } + } + } + } + _ => {} + } + } + } + blocks +} + fn append_guidance(instructions: &mut Option, guidance: &str) { *instructions = Some(match instructions.take() { Some(existing) if !existing.is_empty() => format!("{existing}\n\n{guidance}"), @@ -400,6 +506,8 @@ fn parse_message( message: &Message, out: &mut Vec, calls: &mut HashSet, + image_mode: GrokToolImageMode, + budget: &mut ReattachBudget, ) -> anyhow::Result<()> { if !["system", "user", "assistant"].contains(&message.role.as_str()) { anyhow::bail!("unsupported message role"); @@ -446,6 +554,46 @@ fn parse_message( } ("assistant", "web_search_tool_result" | "x_search_tool_result") | ("user", "web_search_tool_result" | "x_search_tool_result") => {} + ("user", "image") => { + if object + .keys() + .any(|key| !["type", "source", "cache_control"].contains(&key.as_str())) + || !valid_cache_control(object.get("cache_control")) + { + anyhow::bail!("unsupported image block field"); + } + match image_mode { + GrokToolImageMode::Reject => { + anyhow::bail!("unsupported content block: image"); + } + GrokToolImageMode::Omit => { + let placeholder = image_placeholder(object) + .ok_or_else(|| anyhow::anyhow!("image source is invalid"))?; + content.push(GrokContentPart::InputText { text: placeholder }); + } + GrokToolImageMode::Reattach | GrokToolImageMode::Inline => { + let source = parse_image_source(object) + .ok_or_else(|| anyhow::anyhow!("image source is invalid"))?; + match gate_image(&source) { + Ok(()) => { + let image_url = image_source_to_url(&source); + if budget.admit(&image_url) { + content.push(GrokContentPart::InputImage { image_url }); + } else { + content.push(GrokContentPart::InputText { + text: omit_with_reason(&source, &cap_reason()), + }); + } + } + Err(reason) => { + content.push(GrokContentPart::InputText { + text: omit_with_reason(&source, &reason), + }); + } + } + } + } + } ("assistant", "tool_use") => { if object.keys().any(|key| { !["type", "id", "name", "input", "cache_control"].contains(&key.as_str()) @@ -507,10 +655,12 @@ fn parse_message( let value = object .get("content") .ok_or_else(|| anyhow::anyhow!("tool result content is required"))?; + let mut tool_result_images: Vec = Vec::new(); let output = match value { - Value::String(text) => text.clone(), + Value::String(text) => GrokToolOutput::Text(text.clone()), Value::Array(parts) => { let mut texts = Vec::new(); + let mut inline_parts: Vec = Vec::new(); for part in parts { let part = part.as_object().ok_or_else(|| { anyhow::anyhow!("tool result child must be an object") @@ -528,6 +678,84 @@ fn parse_message( } continue; } + if part.get("type").and_then(Value::as_str) == Some("image") { + if part.keys().any(|key| { + !["type", "source", "cache_control"].contains(&key.as_str()) + }) || !valid_cache_control(part.get("cache_control")) + { + anyhow::bail!("unsupported image child"); + } + match image_mode { + GrokToolImageMode::Reject => { + anyhow::bail!("tool result supports text children only"); + } + GrokToolImageMode::Omit => { + let placeholder = + image_placeholder(part).ok_or_else(|| { + anyhow::anyhow!( + "tool result image source is invalid" + ) + })?; + texts.push(placeholder); + } + GrokToolImageMode::Reattach => { + let source = parse_image_source(part).ok_or_else(|| { + anyhow::anyhow!("tool result image source is invalid") + })?; + match gate_image(&source) { + Ok(()) => { + let image_url = image_source_to_url(&source); + if budget.admit(&image_url) { + texts.push(image_placeholder(part).ok_or_else( + || { + anyhow::anyhow!( + "tool result image source is invalid" + ) + }, + )?); + tool_result_images.push(image_url); + } else { + texts.push(omit_with_reason( + &source, + &cap_reason(), + )); + } + } + Err(reason) => { + texts.push(omit_with_reason(&source, &reason)); + } + } + } + GrokToolImageMode::Inline => { + let source = parse_image_source(part).ok_or_else(|| { + anyhow::anyhow!("tool result image source is invalid") + })?; + match gate_image(&source) { + Ok(()) => { + let image_url = image_source_to_url(&source); + if budget.admit(&image_url) { + inline_parts.push( + GrokContentPart::InputImage { image_url }, + ); + } else { + inline_parts.push(GrokContentPart::InputText { + text: omit_with_reason( + &source, + &cap_reason(), + ), + }); + } + } + Err(reason) => { + inline_parts.push(GrokContentPart::InputText { + text: omit_with_reason(&source, &reason), + }); + } + } + } + } + continue; + } if part.get("type").and_then(Value::as_str) != Some("text") || part.keys().any(|key| { !["type", "text", "cache_control"].contains(&key.as_str()) @@ -536,13 +764,41 @@ fn parse_message( { anyhow::bail!("tool result supports text children only"); } - texts.push( - part.get("text").and_then(Value::as_str).ok_or_else(|| { - anyhow::anyhow!("tool result text is invalid") - })?, - ); + let text = part + .get("text") + .and_then(Value::as_str) + .ok_or_else(|| anyhow::anyhow!("tool result text is invalid"))? + .to_string(); + if image_mode == GrokToolImageMode::Inline { + inline_parts.push(GrokContentPart::InputText { text }); + } else { + texts.push(text); + } + } + if image_mode == GrokToolImageMode::Inline { + // Text-only results keep the plain-string shape so + // inline mode is invisible unless an image is present. + let mut joined: Option = Some(String::new()); + for part in &inline_parts { + match part { + GrokContentPart::InputText { text } => { + if let Some(acc) = joined.as_mut() { + if !acc.is_empty() { + acc.push('\n'); + } + acc.push_str(text); + } + } + _ => joined = None, + } + } + match joined { + Some(text) => GrokToolOutput::Text(text), + None => GrokToolOutput::Parts(inline_parts), + } + } else { + GrokToolOutput::Text(texts.join("\n")) } - texts.join("") } _ => anyhow::bail!("tool result supports text only"), }; @@ -550,6 +806,15 @@ fn parse_message( call_id: id.into(), output, }); + if image_mode == GrokToolImageMode::Reattach && !tool_result_images.is_empty() { + out.push(GrokInputItem::Message { + role: "user".into(), + content: tool_result_images + .into_iter() + .map(|image_url| GrokContentPart::InputImage { image_url }) + .collect(), + }); + } } _ => anyhow::bail!("unsupported content block: {typ}"), } @@ -558,6 +823,255 @@ fn parse_message( Ok(()) } +fn image_placeholder(object: &serde_json::Map) -> Option { + let source = object.get("source")?.as_object()?; + match source.get("type").and_then(Value::as_str) { + Some("base64") + if source + .get("media_type") + .and_then(Value::as_str) + .is_some_and(|media_type| !media_type.is_empty()) + && source.get("data").and_then(Value::as_str).is_some() => + { + let media_type = source.get("media_type").and_then(Value::as_str)?; + Some(format!("[image omitted: {media_type}]")) + } + Some("url") + if source + .get("url") + .and_then(Value::as_str) + .is_some_and(|url| !url.is_empty()) => + { + Some("[image omitted: url]".into()) + } + _ => None, + } +} + +// --------------------------------------------------------------------------- +// L2a reattach gates (limits verified against cli-chat-proxy.grok.com on +// 2026-07-20: 1x1 and 8x8 rejected, 32x32 accepted; production-scale +// screenshots well above the minimums; both `reattach` and `inline` wire +// shapes returned 200 with correct visual answers) +// --------------------------------------------------------------------------- + +/// Upstream rejects images whose smallest side is under this many pixels. +const MIN_IMAGE_SIDE_PX: u32 = 8; +/// Upstream rejects images whose area is under this many square pixels. +const MIN_IMAGE_AREA_PX: u64 = 512; +/// Decoded RGB(A) payload cap; larger images are degraded to the omit marker. +const MAX_IMAGE_DECODED_BYTES: u64 = 5 * 1024 * 1024; +/// Only the last few images across the whole request are attached. +const MAX_REATTACHED_IMAGES: usize = 4; + +/// Parse an Anthropic image block into a shared `ImageSource`. Returns `None` +/// for structurally invalid blocks (missing fields, unknown source type). +fn parse_image_source(object: &serde_json::Map) -> Option { + let source = object.get("source")?.as_object()?; + let source_type = source.get("type").and_then(Value::as_str)?; + match source_type { + "base64" => { + let media_type = source + .get("media_type") + .and_then(Value::as_str) + .filter(|media_type| !media_type.is_empty())?; + let data = source.get("data").and_then(Value::as_str)?; + Some(ImageSource { + media_type: media_type.to_string(), + data: data.to_string(), + source_type: source_type.to_string(), + }) + } + "url" => { + let url = source + .get("url") + .and_then(Value::as_str) + .filter(|url| !url.is_empty())?; + Some(ImageSource { + media_type: "image/*".to_string(), + data: url.to_string(), + source_type: source_type.to_string(), + }) + } + _ => None, + } +} + +/// Gate a candidate image against the upstream-verified limits. Returns the +/// omit reason on failure so the caller can degrade just this one image. +fn gate_image(source: &ImageSource) -> Result<(), String> { + if source.source_type == "url" { + // The proxy cannot gate a remote image without fetching it (dimensions + // and decoded size are unknown), and an unverifiable image can 400 the + // whole turn upstream — so URL sources never reattach. + return Err("url source cannot be gated".to_string()); + } + use base64::Engine; + let bytes = base64::engine::general_purpose::STANDARD + .decode(source.data.as_bytes()) + .map_err(|_| "undecodable base64".to_string())?; + let raster = image_raster(&bytes, &source.media_type) + .ok_or_else(|| format!("unreadable dimensions for {}", source.media_type))?; + let width = raster.width; + let height = raster.height; + let min_side = width.min(height); + if min_side < MIN_IMAGE_SIDE_PX { + return Err(format!( + "{width}x{height} below minimum side {MIN_IMAGE_SIDE_PX}px" + )); + } + let area = width as u64 * height as u64; + if area < MIN_IMAGE_AREA_PX { + return Err(format!( + "{width}x{height} below minimum area {MIN_IMAGE_AREA_PX}px" + )); + } + let decoded = area.saturating_mul(raster.bytes_per_pixel); + if decoded > MAX_IMAGE_DECODED_BYTES { + return Err(format!( + "{width}x{height} too large (decoded ~{}MB > {}MB cap)", + decoded / (1024 * 1024), + MAX_IMAGE_DECODED_BYTES / (1024 * 1024) + )); + } + Ok(()) +} + +/// Render the L1 omit marker with a gate-failure reason appended. +fn omit_with_reason(source: &ImageSource, reason: &str) -> String { + let base = if source.source_type == "url" { + "[image omitted: url]".to_string() + } else { + format!("[image omitted: {}]", source.media_type) + }; + format!("{base} ({reason})") +} + +/// Reason attached to images that passed every per-image gate but lost the +/// request-wide "keep only the last few" cap. +fn cap_reason() -> String { + format!("only the last {MAX_REATTACHED_IMAGES} images are attached per request") +} + +/// Extract raster dimensions and a conservative decoded byte width from the +/// encoded image header. PNG accounting reserves alpha when the format may +/// carry transparency. GIF accounting reserves an RGBA output pixel. +#[derive(Clone, Copy)] +struct ImageRaster { + width: u32, + height: u32, + bytes_per_pixel: u64, +} + +fn image_raster(bytes: &[u8], media_type: &str) -> Option { + match media_type { + "image/png" => png_raster(bytes), + "image/jpeg" => jpeg_raster(bytes), + "image/gif" => gif_raster(bytes), + _ => sniff_raster(bytes), + } +} + +fn sniff_raster(bytes: &[u8]) -> Option { + png_raster(bytes) + .or_else(|| jpeg_raster(bytes)) + .or_else(|| gif_raster(bytes)) +} + +fn png_raster(bytes: &[u8]) -> Option { + // Signature (8) + IHDR length (4) + "IHDR" (4) + IHDR fields. + if bytes.len() < 26 + || &bytes[..8] != b"\x89PNG\r\n\x1a\n" + || u32::from_be_bytes(bytes[8..12].try_into().ok()?) != 13 + || &bytes[12..16] != b"IHDR" + { + return None; + } + let width = u32::from_be_bytes(bytes[16..20].try_into().ok()?); + let height = u32::from_be_bytes(bytes[20..24].try_into().ok()?); + let bit_depth = bytes[24]; + let color_type = bytes[25]; + let bytes_per_channel = match bit_depth { + 1 | 2 | 4 | 8 => 1, + 16 => 2, + _ => return None, + }; + let channels = match color_type { + // Grayscale and RGB may carry transparency in a tRNS chunk. + 0 | 4 => 2, + 2 | 3 | 6 => 4, + _ => return None, + }; + Some(ImageRaster { + width, + height, + bytes_per_pixel: bytes_per_channel * channels, + }) +} + +fn gif_raster(bytes: &[u8]) -> Option { + // "GIF87a"/"GIF89a" (6) + width (2 LE) + height (2 LE). + if bytes.len() < 10 || !matches!(&bytes[..6], b"GIF87a" | b"GIF89a") { + return None; + } + let width = u16::from_le_bytes(bytes[6..8].try_into().ok()?) as u32; + let height = u16::from_le_bytes(bytes[8..10].try_into().ok()?) as u32; + Some(ImageRaster { + width, + height, + bytes_per_pixel: 4, + }) +} + +fn jpeg_raster(bytes: &[u8]) -> Option { + // Walk SOI-delimited segments to the first SOF0..SOF15 frame header. + if bytes.len() < 4 || bytes[0] != 0xff || bytes[1] != 0xd8 { + return None; + } + let mut cursor = 2usize; + while cursor + 4 <= bytes.len() { + if bytes[cursor] != 0xff { + return None; + } + let marker = bytes[cursor + 1]; + // Standalone markers without a length field. + if marker == 0xd8 || marker == 0x01 || (0xd0..=0xd7).contains(&marker) { + cursor += 2; + continue; + } + let segment_len = + u16::from_be_bytes(bytes[cursor + 2..cursor + 4].try_into().ok()?) as usize; + if segment_len < 2 || cursor + 2 + segment_len > bytes.len() { + return None; + } + let is_sof = matches!( + marker, + 0xc0..=0xc3 | 0xc5..=0xc7 | 0xc9..=0xcb | 0xcd..=0xcf + ); + if is_sof { + // Segment: length (2), precision (1), dimensions (4), components (1). + if segment_len < 8 { + return None; + } + let precision = bytes[cursor + 4]; + let height = u16::from_be_bytes(bytes[cursor + 5..cursor + 7].try_into().ok()?) as u32; + let width = u16::from_be_bytes(bytes[cursor + 7..cursor + 9].try_into().ok()?) as u32; + let components = bytes[cursor + 9] as usize; + if components == 0 || segment_len < 8 + 3 * components { + return None; + } + let bytes_per_channel = u64::from(precision).div_ceil(8); + return Some(ImageRaster { + width, + height, + bytes_per_pixel: bytes_per_channel.saturating_mul(components as u64), + }); + } + cursor += 2 + segment_len; + } + None +} + fn valid_cache_control(value: Option<&Value>) -> bool { let Some(value) = value else { return true }; let Some(object) = value.as_object() else { @@ -897,6 +1411,16 @@ mod tests { .unwrap() } + fn translated_with_mode( + request: &MessagesRequest, + image_mode: crate::config::GrokToolImageMode, + ) -> Value { + serde_json::to_value( + translate_request_with_mode(request, "grok-4.5".into(), image_mode).unwrap(), + ) + .unwrap() + } + #[test] fn grok_translation_rejects_unknown_tool_block_fields() { let mut request = request_with_blocks(serde_json::json!([ @@ -911,12 +1435,86 @@ mod tests { assert!(translate_request(&request, "grok-4.5".into()).is_err()); } + #[test] + fn grok_translation_omits_image_only_tool_result_children() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":"aGVsbG8="}} + ]} + ])); + let translated = translated_with_mode(&request, crate::config::GrokToolImageMode::Omit); + assert_eq!(translated["input"][1]["type"], "function_call_output"); + assert_eq!( + translated["input"][1]["output"], + "[image omitted: image/png]" + ); + } + + #[test] + fn grok_translation_omits_url_image_tool_result_children() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"url","url":"https://example.invalid/a.png"}} + ]} + ])); + let translated = translated_with_mode(&request, crate::config::GrokToolImageMode::Omit); + assert_eq!(translated["input"][1]["output"], "[image omitted: url]"); + } + + #[test] + fn grok_translation_joins_text_and_image_tool_result_children() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"text","text":"caption"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":"aGVsbG8="}}, + {"type":"image","source":{"type":"url","url":"https://example.invalid/a.png"}} + ]} + ])); + let translated = translated_with_mode(&request, crate::config::GrokToolImageMode::Omit); + assert_eq!( + translated["input"][1]["output"], + "caption\n[image omitted: image/png]\n[image omitted: url]" + ); + } + + #[test] + fn grok_translation_joins_multiple_text_tool_result_children_with_newlines() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"text","text":"first"}, + {"type":"text","text":"second"} + ]} + ])); + let translated = + serde_json::to_value(translate_request(&request, "grok-4.5".into()).unwrap()).unwrap(); + assert_eq!(translated["input"][1]["output"], "first\nsecond"); + } + + #[test] + fn grok_translation_omits_top_level_user_image_blocks() { + let request: MessagesRequest = serde_json::from_value(serde_json::json!({ + "model":"grok-4.5", + "messages":[{"role":"user","content":[ + {"type":"text","text":"what is this?"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":"aGVsbG8="}} + ]}] + })) + .unwrap(); + let translated = translated_with_mode(&request, crate::config::GrokToolImageMode::Omit); + assert_eq!( + translated["input"][0]["content"], + serde_json::json!([ + {"type":"input_text","text":"what is this?"}, + {"type":"input_text","text":"[image omitted: image/png]"} + ]) + ); + } + #[test] fn grok_translation_rejects_malformed_tool_result_children() { for child in [ serde_json::json!("text"), serde_json::json!({"text":"ok"}), - serde_json::json!({"type":"image","text":"ok"}), serde_json::json!({"type":"text","text":1}), serde_json::json!({"type":"text","text":"ok","unknown":true}), ] { @@ -999,4 +1597,701 @@ mod tests { assert_eq!(translated["input"][1]["output"], "result"); assert!(!translated.to_string().contains("cache_control")); } + + // --------------------------------------------------------------------- + // L2a: CCP_GROK_TOOL_IMAGE flag + reattach vision tests + // --------------------------------------------------------------------- + + // 32x32 solid red PNG (valid dimensions, passes all gates). + const PNG_32_RED: &str = "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAAKElEQVR4nO3NsQ0AAAzCMP5/un0CNkuZ41wybXsHAAAAAAAAAAAAxR4yw/wuPL6QkAAAAABJRU5ErkJggg=="; + // 1x1 red PNG (min-side + area gate failure). + const PNG_1_RED: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC"; + // 8x8 red PNG (area gate failure: 64 < 512). + const PNG_8_RED: &str = "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAEklEQVR4nGP4z8CAFWEXHbQSACj/P8Fu7N9hAAAAAElFTkSuQmCC"; + + #[test] + fn grok_tool_image_mode_parses_flag_values() { + use crate::config::{GrokToolImageMode, parse_grok_tool_image_mode}; + assert_eq!(parse_grok_tool_image_mode(None), GrokToolImageMode::Omit); + assert_eq!( + parse_grok_tool_image_mode(Some("omit")), + GrokToolImageMode::Omit + ); + assert_eq!( + parse_grok_tool_image_mode(Some("reattach")), + GrokToolImageMode::Reattach + ); + assert_eq!( + parse_grok_tool_image_mode(Some("reject")), + GrokToolImageMode::Reject + ); + assert_eq!( + parse_grok_tool_image_mode(Some("inline")), + GrokToolImageMode::Inline + ); + // Unknown values are the safe default. + assert_eq!( + parse_grok_tool_image_mode(Some("bogus")), + GrokToolImageMode::Omit + ); + assert_eq!( + parse_grok_tool_image_mode(Some("")), + GrokToolImageMode::Omit + ); + } + + #[test] + fn reattach_tool_result_image_emits_following_user_message_with_input_image() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"text","text":"screenshot"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reattach, + ) + .unwrap(), + ) + .unwrap(); + // function_call_output keeps the L1 omit marker (text stays verbatim). + assert_eq!(translated["input"][1]["type"], "function_call_output"); + assert_eq!( + translated["input"][1]["output"], + "screenshot\n[image omitted: image/png]" + ); + // A user message follows carrying the image as an input_image data URL. + assert_eq!(translated["input"][2]["type"], "message"); + assert_eq!(translated["input"][2]["role"], "user"); + let parts = translated["input"][2]["content"].as_array().unwrap(); + assert_eq!(parts.len(), 1); + assert_eq!(parts[0]["type"], "input_image"); + let url = parts[0]["image_url"].as_str().unwrap(); + assert!( + url.starts_with("data:image/png;base64,"), + "unexpected image_url prefix: {}", + &url[..url.len().min(40)] + ); + assert!(url.contains(PNG_32_RED)); + } + + #[test] + fn reattach_top_level_user_image_becomes_input_image_part() { + let request: MessagesRequest = serde_json::from_value(serde_json::json!({ + "model":"grok-4.5", + "messages":[{"role":"user","content":[ + {"type":"text","text":"what color is this?"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]}] + })) + .unwrap(); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reattach, + ) + .unwrap(), + ) + .unwrap(); + let parts = translated["input"][0]["content"].as_array().unwrap(); + assert_eq!(parts[0]["type"], "input_text"); + assert_eq!(parts[0]["text"], "what color is this?"); + assert_eq!(parts[1]["type"], "input_image"); + assert!( + parts[1]["image_url"] + .as_str() + .unwrap() + .starts_with("data:image/png;base64,") + ); + } + + #[test] + fn reattach_degrades_tiny_image_to_omit_with_reason_never_error() { + for (data, dims) in [(PNG_1_RED, "1x1"), (PNG_8_RED, "8x8")] { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":data}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reattach, + ) + .unwrap(), + ) + .unwrap(); + let output = translated["input"][1]["output"].as_str().unwrap(); + assert!( + output.starts_with("[image omitted: image/png"), + "unexpected output: {output}" + ); + assert!(output.contains(dims), "reason must cite dims: {output}"); + // No reattached message: gated-out images never produce input_image. + assert_eq!(translated["input"].as_array().unwrap().len(), 2); + } + } + + #[test] + fn reattach_accepts_rgb_image_under_decoded_size_cap() { + // The 8-bit RGB raster is 2.1MB. A format-independent 8-byte estimate + // would incorrectly reject this common screenshot size. + let image = solid_rgb_png_base64(1000, 700); + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":image}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reattach, + ) + .unwrap(), + ) + .unwrap(); + assert_eq!( + translated["input"][1]["output"], + "[image omitted: image/png]" + ); + assert_eq!(translated["input"][2]["content"][0]["type"], "input_image"); + } + + #[test] + fn reattach_degrades_oversized_image_to_omit_with_reason() { + // 6000x6000 solid PNG: decoded raw size is at least 108MB. Deflate of a + // solid scanline is tiny, so the base64 fixture stays small. + let big = solid_rgb_png_base64(6000, 6000); + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":big}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reattach, + ) + .unwrap(), + ) + .unwrap(); + let output = translated["input"][1]["output"].as_str().unwrap(); + assert!( + output.starts_with("[image omitted: image/png"), + "unexpected output: {output}" + ); + assert!( + output.contains("too large"), + "reason must cite size: {output}" + ); + assert_eq!(translated["input"].as_array().unwrap().len(), 2); + } + + #[test] + fn reattach_keeps_only_last_four_images_per_request() { + let mut result_parts = Vec::new(); + for _ in 0..6 { + result_parts.push(serde_json::json!({"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}})); + } + let request: MessagesRequest = serde_json::from_value(serde_json::json!({ + "model":"grok-4.5", + "messages":[ + {"role":"assistant","content":[{"type":"tool_use","id":"call_1","name":"lookup","input":{}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"call_1","content":result_parts}]} + ] + })) + .unwrap(); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reattach, + ) + .unwrap(), + ) + .unwrap(); + let input = translated["input"].as_array().unwrap(); + // function_call + function_call_output + one reattached user message. + assert_eq!(input.len(), 3); + let parts = input[2]["content"].as_array().unwrap(); + assert_eq!(parts.len(), 4, "only the last 4 images survive"); + assert!(parts.iter().all(|part| part["type"] == "input_image")); + // The two cap-dropped images degrade with a reason in the tool output. + let output = input[1]["output"].as_str().unwrap(); + let cap_drops = output.matches("only the last 4 images").count(); + assert_eq!( + cap_drops, 2, + "cap-dropped images must carry a reason: {output}" + ); + let attached = output.matches("[image omitted: image/png]\n").count() + + if output.ends_with("[image omitted: image/png]") { + 1 + } else { + 0 + }; + assert_eq!(attached, 4, "surviving images keep the plain L1 marker"); + } + + #[test] + fn reattach_cap_applies_across_tool_results_in_one_request() { + // Two tool results with 3 images each: 6 gate-passing images, so only + // the last 4 across the request reattach (the whole second result's 3 + // plus the first result's last 1). + let image = || serde_json::json!({"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}}); + let request: MessagesRequest = serde_json::from_value(serde_json::json!({ + "model":"grok-4.5", + "messages":[ + {"role":"assistant","content":[ + {"type":"tool_use","id":"call_1","name":"lookup","input":{}}, + {"type":"tool_use","id":"call_2","name":"lookup","input":{}} + ]}, + {"role":"user","content":[ + {"type":"tool_result","tool_use_id":"call_1","content":[image(),image(),image()]}, + {"type":"tool_result","tool_use_id":"call_2","content":[image(),image(),image()]} + ]} + ] + })) + .unwrap(); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reattach, + ) + .unwrap(), + ) + .unwrap(); + let input = translated["input"].as_array().unwrap(); + let total_attached: usize = input + .iter() + .map(|item| { + item["content"] + .as_array() + .map(|parts| { + parts + .iter() + .filter(|part| part["type"] == "input_image") + .count() + }) + .unwrap_or(0) + }) + .sum(); + assert_eq!( + total_attached, 4, + "cap is request-wide, not per tool result" + ); + } + + #[test] + fn reattach_degrades_url_images_with_reason_instead_of_reattaching() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"url","url":"https://example.invalid/a.png"}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reattach, + ) + .unwrap(), + ) + .unwrap(); + let output = translated["input"][1]["output"].as_str().unwrap(); + assert!( + output.starts_with("[image omitted: url]"), + "unexpected output: {output}" + ); + assert!( + output.contains("cannot be gated"), + "reason must explain the skip: {output}" + ); + // No input_image part anywhere. + assert!(!translated.to_string().contains("input_image")); + } + + #[test] + fn reject_mode_restores_old_bail_on_tool_result_images() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]} + ])); + let result = translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reject, + ); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("tool result supports text children only") + ); + } + + #[test] + fn reject_mode_restores_old_bail_on_top_level_user_images() { + let request: MessagesRequest = serde_json::from_value(serde_json::json!({ + "model":"grok-4.5", + "messages":[{"role":"user","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]}] + })) + .unwrap(); + let result = translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reject, + ); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("unsupported content block: image") + ); + } + + /// Build a solid 8-bit RGB PNG of the given size, returned as base64. + fn solid_rgb_png_base64(width: u32, height: u32) -> String { + use base64::Engine; + use std::io::Write as _; + let mut png = Vec::new(); + png.extend_from_slice(&[0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a]); + let mut ihdr = Vec::new(); + ihdr.extend_from_slice(&width.to_be_bytes()); + ihdr.extend_from_slice(&height.to_be_bytes()); + ihdr.extend_from_slice(&[8, 2, 0, 0, 0]); // 8-bit RGB + write_chunk(&mut png, b"IHDR", &ihdr); + // One scanline: filter byte + width * 3 zero bytes, repeated. + let mut raw = Vec::with_capacity((width as usize * 3 + 1) * height as usize); + for _ in 0..height { + raw.push(0u8); + raw.extend(std::iter::repeat_n(0u8, width as usize * 3)); + } + let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(&raw).unwrap(); + let idat = encoder.finish().unwrap(); + write_chunk(&mut png, b"IDAT", &idat); + write_chunk(&mut png, b"IEND", &[]); + base64::engine::general_purpose::STANDARD.encode(png) + } + + fn write_chunk(out: &mut Vec, kind: &[u8; 4], data: &[u8]) { + out.extend_from_slice(&(data.len() as u32).to_be_bytes()); + out.extend_from_slice(kind); + out.extend_from_slice(data); + let mut crc_data = Vec::with_capacity(4 + data.len()); + crc_data.extend_from_slice(kind); + crc_data.extend_from_slice(data); + out.extend_from_slice(&crc32(&crc_data).to_be_bytes()); + } + + fn crc32(data: &[u8]) -> u32 { + let mut crc: u32 = 0xffff_ffff; + for byte in data { + crc ^= *byte as u32; + for _ in 0..8 { + let mask = (crc & 1).wrapping_neg(); + crc = (crc >> 1) ^ (0xedb8_8320 & mask); + } + } + !crc + } + + // --------------------------------------------------------------------- + // L2b: CCP_GROK_TOOL_IMAGE=inline — tool output as a content-part array + // --------------------------------------------------------------------- + + #[test] + fn inline_tool_result_image_emits_array_output_with_input_image_part() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"text","text":"screenshot"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(translated["input"][1]["type"], "function_call_output"); + // output is an untagged array of content parts, not a string. + let parts = translated["input"][1]["output"].as_array().unwrap(); + assert_eq!(parts.len(), 2); + assert_eq!(parts[0]["type"], "input_text"); + assert_eq!(parts[0]["text"], "screenshot"); + assert_eq!(parts[1]["type"], "input_image"); + let url = parts[1]["image_url"].as_str().unwrap(); + assert!( + url.starts_with("data:image/png;base64,"), + "unexpected image_url prefix: {}", + &url[..url.len().min(40)] + ); + assert!(url.contains(PNG_32_RED)); + // No reattached user message: the image rides inside the tool output. + assert_eq!(translated["input"].as_array().unwrap().len(), 2); + } + + #[test] + fn inline_image_only_tool_result_emits_array_with_single_image_part() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + let parts = translated["input"][1]["output"].as_array().unwrap(); + assert_eq!(parts.len(), 1); + assert_eq!(parts[0]["type"], "input_image"); + } + + #[test] + fn inline_text_only_tool_result_serializes_byte_identically_to_omit() { + // String-only outputs (and text-only array results) must keep the + // plain-string shape in every mode — inline included. + let shapes = [ + serde_json::json!({"type":"tool_result","tool_use_id":"call_1","content":"plain string"}), + serde_json::json!({"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"text","text":"first"}, + {"type":"text","text":"second"} + ]}), + ]; + for shape in shapes { + let request = request_with_blocks(serde_json::json!([shape])); + let omit = serde_json::to_string( + &translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Omit, + ) + .unwrap(), + ) + .unwrap(); + let inline = serde_json::to_string( + &translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(omit, inline, "text-only outputs must be byte-identical"); + // And the output field really is a bare JSON string, not an array. + let value: Value = serde_json::from_str(&inline).unwrap(); + assert!(value["input"][1]["output"].is_string()); + } + } + + #[test] + fn inline_string_output_matches_pre_inline_serialization_exactly() { + // Regression: the whole upstream-bound request body for a string tool + // result must serialize exactly as before the GrokToolOutput widening. + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":"result"} + ])); + let body = serde_json::to_string( + &translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + assert!( + body.contains(r#""output":"result""#), + "output must serialize as a bare string: {body}" + ); + assert!(!body.contains(r#""output":["#)); + } + + #[test] + fn inline_all_images_gated_out_collapse_to_string_with_reasons() { + for (data, dims) in [(PNG_1_RED, "1x1"), (PNG_8_RED, "8x8")] { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"text","text":"shot"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":data}} + ]} + ])); + // Gate failures degrade per-image, never a 400 for the whole turn. + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + // All parts ended up as text → collapses back to a plain string. + let output = translated["input"][1]["output"].as_str().unwrap(); + assert!( + output.contains("[image omitted: image/png"), + "unexpected output: {output}" + ); + assert!(output.contains(dims), "reason must cite dims: {output}"); + assert!(output.starts_with("shot\n")); + } + } + + #[test] + fn inline_mixed_passing_and_failing_images_keep_array_shape() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_1_RED}}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + let parts = translated["input"][1]["output"].as_array().unwrap(); + assert_eq!(parts.len(), 2); + // The gated-out image becomes an in-band omit fragment inside the array. + assert_eq!(parts[0]["type"], "input_text"); + assert!( + parts[0]["text"] + .as_str() + .unwrap() + .starts_with("[image omitted: image/png") + ); + assert!(parts[0]["text"].as_str().unwrap().contains("1x1")); + assert_eq!(parts[1]["type"], "input_image"); + } + + #[test] + fn inline_degrades_url_images_to_omit_fragment_inside_array() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"url","url":"https://example.invalid/a.png"}}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + let parts = translated["input"][1]["output"].as_array().unwrap(); + assert_eq!(parts.len(), 2); + assert_eq!(parts[0]["type"], "input_text"); + assert!( + parts[0]["text"] + .as_str() + .unwrap() + .contains("cannot be gated") + ); + assert_eq!(parts[1]["type"], "input_image"); + } + + #[test] + fn inline_keeps_only_last_four_images_per_request() { + let mut result_parts = Vec::new(); + for _ in 0..6 { + result_parts.push(serde_json::json!({"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}})); + } + let request: MessagesRequest = serde_json::from_value(serde_json::json!({ + "model":"grok-4.5", + "messages":[ + {"role":"assistant","content":[{"type":"tool_use","id":"call_1","name":"lookup","input":{}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"call_1","content":result_parts}]} + ] + })) + .unwrap(); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + let input = translated["input"].as_array().unwrap(); + // function_call + function_call_output only — no reattached message. + assert_eq!(input.len(), 2); + let parts = input[1]["output"].as_array().unwrap(); + assert_eq!(parts.len(), 6); + let images = parts + .iter() + .filter(|part| part["type"] == "input_image") + .count(); + assert_eq!(images, 4, "only the last 4 images survive"); + let cap_drops = parts + .iter() + .filter(|part| { + part["type"] == "input_text" + && part["text"] + .as_str() + .is_some_and(|text| text.contains("only the last 4 images")) + }) + .count(); + assert_eq!(cap_drops, 2, "cap-dropped images carry a reason"); + } + + #[test] + fn inline_top_level_user_image_becomes_input_image_part() { + let request: MessagesRequest = serde_json::from_value(serde_json::json!({ + "model":"grok-4.5", + "messages":[{"role":"user","content":[ + {"type":"text","text":"what color is this?"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]}] + })) + .unwrap(); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + let parts = translated["input"][0]["content"].as_array().unwrap(); + assert_eq!(parts[0]["type"], "input_text"); + assert_eq!(parts[1]["type"], "input_image"); + assert!( + parts[1]["image_url"] + .as_str() + .unwrap() + .starts_with("data:image/png;base64,") + ); + } } diff --git a/src/traffic.rs b/src/traffic.rs index 780d2b81..36d60efa 100644 --- a/src/traffic.rs +++ b/src/traffic.rs @@ -357,7 +357,15 @@ fn redact_traffic_with_depth(value: &Value, depth: u16) -> Value { let mut out = Map::new(); for (key, value) in map { let normalized = key.to_lowercase(); - if REDACT_KEYS.contains(&normalized.as_str()) + if normalized == "image_url" { + // Grok/OpenAI `input_image` parts carry data URLs; never + // persist the payload in traffic captures. + out.insert(key.clone(), redact_traffic_value(value)); + } else if normalized == "data" && looks_like_image_source(map) { + // Anthropic image blocks carry raw base64 under + // `source.data`; redact it for the same reason. + out.insert(key.clone(), redact_traffic_value(value)); + } else if REDACT_KEYS.contains(&normalized.as_str()) || matches!( normalized.as_str(), "token" @@ -390,14 +398,43 @@ fn redact_traffic_with_depth(value: &Value, depth: u16) -> Value { .map(|value| redact_traffic_with_depth(value, depth + 1)) .collect(), ), + Value::String(text) if is_data_url(text) => { + Value::String(format!("[redacted data-url len={}]", text.len())) + } _ => value.clone(), } } +fn is_data_url(text: &str) -> bool { + let lower = text.to_ascii_lowercase(); + lower.starts_with("data:image/") && lower.contains(";base64,") +} + +/// An object shaped like an Anthropic image source: `type: "base64"` and +/// either an image `media_type` or none at all. Used to redact only image +/// payloads, not arbitrary `data` keys. Missing `media_type` is treated as +/// an image: the capture is written before translation, so a malformed block +/// that the translator would reject must still not persist its payload. +fn looks_like_image_source(map: &Map) -> bool { + if !map + .get("type") + .and_then(Value::as_str) + .is_some_and(|source_type| source_type.eq_ignore_ascii_case("base64")) + { + return false; + } + match map.get("media_type").and_then(Value::as_str) { + Some(media_type) => media_type.to_ascii_lowercase().starts_with("image/"), + None => true, + } +} + fn redact_traffic_value(value: &Value) -> Value { match value { Value::String(s) => Value::String(format!("[redacted len={}]", s.len())), - Value::Object(_) | Value::Array(_) => Value::String("[redacted]".to_string()), + // Structured values (e.g. Kimi's `{url: ...}` image_url) keep their + // shape so captures stay parseable; only the payload leaves. + Value::Object(_) | Value::Array(_) => redact_traffic(value), _ => Value::String("[redacted]".to_string()), } } @@ -418,3 +455,183 @@ fn set_mode(path: &Path, mode: u32) { fn _provider_alias(_provider: &str) -> Option { None } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn redact_traffic_strips_input_image_data_urls() { + let value = serde_json::json!({ + "input": [ + {"type": "message", "role": "user", "content": [ + {"type": "input_text", "text": "what color?"}, + {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC"} + ]} + ] + }); + let redacted = redact_traffic(&value); + let rendered = redacted.to_string(); + assert!( + !rendered.contains("iVBORw0KGgo"), + "base64 payload leaked: {rendered}" + ); + assert!( + !rendered.contains("data:image/png;base64,iVBOR"), + "data URL payload leaked: {rendered}" + ); + // The image part is still structurally visible (redacted marker). + assert!(rendered.contains("input_image")); + } + + #[test] + fn redact_traffic_strips_inline_base64_image_values() { + let value = serde_json::json!({ + "output": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC" + }); + let redacted = redact_traffic(&value); + let rendered = redacted.to_string(); + assert!(!rendered.contains("iVBORw0KGgo")); + } + + #[test] + fn redact_traffic_strips_anthropic_image_source_data() { + // The incoming Anthropic request body carries raw base64 under + // `source.data` (no data: URL wrapper) — it must not persist either. + let value = serde_json::json!({ + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "image", "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC" + }} + ] + }] + }); + let redacted = redact_traffic(&value); + let rendered = redacted.to_string(); + assert!( + !rendered.contains("iVBORw0KGgo"), + "source.data base64 leaked: {rendered}" + ); + assert!(rendered.contains("redacted")); + // Structure is preserved. + assert!(rendered.contains("image/png")); + } + + #[test] + fn redact_traffic_keeps_unrelated_data_keys() { + let value = serde_json::json!({"data": "some-non-image-payload", "count": 3}); + let redacted = redact_traffic(&value); + assert_eq!(redacted["data"], "some-non-image-payload"); + } + + #[test] + fn redact_traffic_strips_data_urls_in_array_shaped_tool_output() { + // L2b inline mode: function_call_output.output is an array of content + // parts — the image_url inside must be redacted just like message parts. + let value = serde_json::json!({ + "input": [ + {"type": "function_call_output", "call_id": "call_1", "output": [ + {"type": "input_text", "text": "screenshot"}, + {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC"} + ]} + ] + }); + let redacted = redact_traffic(&value); + let rendered = redacted.to_string(); + assert!( + !rendered.contains("iVBORw0KGgo"), + "base64 payload leaked: {rendered}" + ); + assert!( + !rendered.contains("data:image/png;base64,iVBOR"), + "data URL payload leaked: {rendered}" + ); + // Structure and text survive. + assert!(rendered.contains("input_image")); + assert!(rendered.contains("screenshot")); + assert!(rendered.contains("redacted")); + } + + #[test] + fn redact_traffic_strips_uppercase_media_type_image_source() { + // Captures are written before translation validates anything, so a + // non-standard casing must still not persist its payload. + let value = serde_json::json!({ + "source": {"type": "base64", "media_type": "IMAGE/PNG", "data": "QUJDREVGRw=="} + }); + let redacted = redact_traffic(&value); + let rendered = redacted.to_string(); + assert!( + !rendered.contains("QUJDREVGRw"), + "payload leaked: {rendered}" + ); + assert!( + redacted["source"]["data"] + .as_str() + .unwrap() + .contains("redacted") + ); + } + + #[test] + fn redact_traffic_strips_uppercase_image_source_type() { + let value = serde_json::json!({ + "source": {"type": "BASE64", "media_type": "image/png", "data": "QUJDREVGRw=="} + }); + let redacted = redact_traffic(&value); + let rendered = redacted.to_string(); + assert!( + !rendered.contains("QUJDREVGRw"), + "payload leaked: {rendered}" + ); + } + + #[test] + fn redact_traffic_strips_image_source_without_media_type() { + // A malformed block the translator would reject must still be redacted + // in the pre-translation capture. + let value = serde_json::json!({ + "source": {"type": "base64", "data": "QUJDREVGRw=="} + }); + let redacted = redact_traffic(&value); + let rendered = redacted.to_string(); + assert!( + !rendered.contains("QUJDREVGRw"), + "payload leaked: {rendered}" + ); + } + + #[test] + fn redact_traffic_strips_uppercase_data_url_scheme() { + let value = serde_json::json!({ + "image_url": "DATA:IMAGE/PNG;base64,iVBORw0KGgoAAAANSUhEUg" + }); + let redacted = redact_traffic(&value); + let rendered = redacted.to_string(); + assert!( + !rendered.contains("iVBORw0KGgo"), + "data URL payload leaked: {rendered}" + ); + } + + #[test] + fn redact_traffic_keeps_structured_image_url_shape() { + // Kimi-style object image_url: redact the payload but keep the object + // structure so captures stay parseable. + let value = serde_json::json!({ + "image_url": {"url": "DATA:IMAGE/PNG;BASE64,iVBORw0KGgoAAAANSUhEUg"} + }); + let redacted = redact_traffic(&value); + assert!(redacted["image_url"].is_object(), "shape lost: {redacted}"); + let rendered = redacted.to_string(); + assert!( + !rendered.contains("iVBORw0KGgo"), + "nested payload leaked: {rendered}" + ); + } +}