From 652966fadab00b5ecec04939d7bdcbecc6c7ac1e Mon Sep 17 00:00:00 2001 From: Desicool Date: Mon, 20 Jul 2026 12:04:04 +0800 Subject: [PATCH 1/5] grok: degrade image blocks to omit placeholders instead of 400 Mirror the Codex translator: image children in tool_result become "[image omitted: ]" (base64) / "[image omitted: url]" fragments joined with text children via "\n", and top-level user image blocks degrade to the same placeholder text rather than erroring with "unsupported content block: image". Multi-part text tool_results are newline-joined to match Codex. Malformed image blocks (bad source, unknown fields) still fail strict validation. --- CHANGELOG.md | 10 ++ src/providers/grok/translate/request.rs | 139 +++++++++++++++++++++++- 2 files changed, 144 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cff29e0..475b7d68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,16 @@ description: Release notes for claude-code-proxy. reasoning-token usage. `CCP_COMPACT_EFFORT` can choose a different cap or disable the behavior. ([#67](https://github.com/raine/claude-code-proxy/pull/67)) +## Unreleased + +- Grok no longer fails requests whose `tool_result` contains image blocks (for + example Claude Code `Read` of a PNG): image children degrade to + `[image omitted: ]` / `[image omitted: url]` placeholders joined + with text children, matching the Codex translator. Top-level user-message + image blocks degrade to the same placeholder instead of erroring. +- Grok multi-part text `tool_result` content is now joined with `\n` between + parts (previously concatenated without a separator), mirroring Codex. + ## v0.1.21 (2026-07-15) - The monitor shows session token activity trends at common terminal widths, diff --git a/src/providers/grok/translate/request.rs b/src/providers/grok/translate/request.rs index 82a026f2..7b9f0587 100644 --- a/src/providers/grok/translate/request.rs +++ b/src/providers/grok/translate/request.rs @@ -446,6 +446,18 @@ fn parse_message( } ("assistant", "web_search_tool_result" | "x_search_tool_result") | ("user", "web_search_tool_result" | "x_search_tool_result") => {} + ("user", "image") => { + if object + .keys() + .any(|key| !["type", "source", "cache_control"].contains(&key.as_str())) + || !valid_cache_control(object.get("cache_control")) + { + anyhow::bail!("unsupported image block field"); + } + let placeholder = image_placeholder(object) + .ok_or_else(|| anyhow::anyhow!("image source is invalid"))?; + content.push(GrokContentPart::InputText { text: placeholder }); + } ("assistant", "tool_use") => { if object.keys().any(|key| { !["type", "id", "name", "input", "cache_control"].contains(&key.as_str()) @@ -528,6 +540,19 @@ fn parse_message( } continue; } + if part.get("type").and_then(Value::as_str) == Some("image") { + if part.keys().any(|key| { + !["type", "source", "cache_control"].contains(&key.as_str()) + }) || !valid_cache_control(part.get("cache_control")) + { + anyhow::bail!("unsupported image child"); + } + let placeholder = image_placeholder(part).ok_or_else(|| { + anyhow::anyhow!("tool result image source is invalid") + })?; + texts.push(placeholder); + continue; + } if part.get("type").and_then(Value::as_str) != Some("text") || part.keys().any(|key| { !["type", "text", "cache_control"].contains(&key.as_str()) @@ -537,12 +562,13 @@ fn parse_message( anyhow::bail!("tool result supports text children only"); } texts.push( - part.get("text").and_then(Value::as_str).ok_or_else(|| { - anyhow::anyhow!("tool result text is invalid") - })?, + part.get("text") + .and_then(Value::as_str) + .ok_or_else(|| anyhow::anyhow!("tool result text is invalid"))? + .to_string(), ); } - texts.join("") + texts.join("\n") } _ => anyhow::bail!("tool result supports text only"), }; @@ -558,6 +584,31 @@ fn parse_message( Ok(()) } +fn image_placeholder(object: &serde_json::Map) -> Option { + let source = object.get("source")?.as_object()?; + match source.get("type").and_then(Value::as_str) { + Some("base64") + if source + .get("media_type") + .and_then(Value::as_str) + .is_some_and(|media_type| !media_type.is_empty()) + && source.get("data").and_then(Value::as_str).is_some() => + { + let media_type = source.get("media_type").and_then(Value::as_str)?; + Some(format!("[image omitted: {media_type}]")) + } + Some("url") + if source + .get("url") + .and_then(Value::as_str) + .is_some_and(|url| !url.is_empty()) => + { + Some("[image omitted: url]".into()) + } + _ => None, + } +} + fn valid_cache_control(value: Option<&Value>) -> bool { let Some(value) = value else { return true }; let Some(object) = value.as_object() else { @@ -911,12 +962,90 @@ mod tests { assert!(translate_request(&request, "grok-4.5".into()).is_err()); } + #[test] + fn grok_translation_omits_image_only_tool_result_children() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":"aGVsbG8="}} + ]} + ])); + let translated = + serde_json::to_value(translate_request(&request, "grok-4.5".into()).unwrap()).unwrap(); + assert_eq!(translated["input"][1]["type"], "function_call_output"); + assert_eq!( + translated["input"][1]["output"], + "[image omitted: image/png]" + ); + } + + #[test] + fn grok_translation_omits_url_image_tool_result_children() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"url","url":"https://example.invalid/a.png"}} + ]} + ])); + let translated = + serde_json::to_value(translate_request(&request, "grok-4.5".into()).unwrap()).unwrap(); + assert_eq!(translated["input"][1]["output"], "[image omitted: url]"); + } + + #[test] + fn grok_translation_joins_text_and_image_tool_result_children() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"text","text":"caption"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":"aGVsbG8="}}, + {"type":"image","source":{"type":"url","url":"https://example.invalid/a.png"}} + ]} + ])); + let translated = + serde_json::to_value(translate_request(&request, "grok-4.5".into()).unwrap()).unwrap(); + assert_eq!( + translated["input"][1]["output"], + "caption\n[image omitted: image/png]\n[image omitted: url]" + ); + } + + #[test] + fn grok_translation_joins_multiple_text_tool_result_children_with_newlines() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"text","text":"first"}, + {"type":"text","text":"second"} + ]} + ])); + let translated = + serde_json::to_value(translate_request(&request, "grok-4.5".into()).unwrap()).unwrap(); + assert_eq!(translated["input"][1]["output"], "first\nsecond"); + } + + #[test] + fn grok_translation_omits_top_level_user_image_blocks() { + let request: MessagesRequest = serde_json::from_value(serde_json::json!({ + "model":"grok-4.5", + "messages":[{"role":"user","content":[ + {"type":"text","text":"what is this?"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":"aGVsbG8="}} + ]}] + })) + .unwrap(); + let translated = + serde_json::to_value(translate_request(&request, "grok-4.5".into()).unwrap()).unwrap(); + assert_eq!( + translated["input"][0]["content"], + serde_json::json!([ + {"type":"input_text","text":"what is this?"}, + {"type":"input_text","text":"[image omitted: image/png]"} + ]) + ); + } + #[test] fn grok_translation_rejects_malformed_tool_result_children() { for child in [ serde_json::json!("text"), serde_json::json!({"text":"ok"}), - serde_json::json!({"type":"image","text":"ok"}), serde_json::json!({"type":"text","text":1}), serde_json::json!({"type":"text","text":"ok","unknown":true}), ] { From 25f1a034dd5061a356bcdd71c9efa429d3cb4e90 Mon Sep 17 00:00:00 2001 From: Desicool Date: Mon, 20 Jul 2026 13:11:57 +0800 Subject: [PATCH 2/5] grok: opt-in vision via CCP_GROK_TOOL_IMAGE=reattach (L2a) Reattach mode keeps the image omission marker in each tool result and sends gate-passing images in a following user message as input_image data URLs. Minimum dimensions, decoded-size limits, and the request-wide last-four cap degrade individual failures to reasoned omission markers. URL sources remain omitted because the proxy cannot validate them without fetching. Traffic captures redact image data URLs and Anthropic source.data payloads. Token estimates charge a flat amount for each attached image. Omit remains the default, and reject retains strict image errors. --- CHANGELOG.md | 11 + src/config.rs | 53 ++ src/providers/grok/count_tokens.rs | 5 + src/providers/grok/mod.rs | 1 + src/providers/grok/translate/request.rs | 750 +++++++++++++++++++++++- src/traffic.rs | 101 +++- 6 files changed, 912 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 475b7d68..37a12a3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,17 @@ description: Release notes for claude-code-proxy. ## Unreleased +- Grok gains opt-in real vision via `CCP_GROK_TOOL_IMAGE=reattach`: image blocks + that pass the upstream gates (min side 8px, min area 512px, decoded size ≤5MB, + last 4 per request) are reattached as a following user message carrying + `input_image` data URLs, in addition to the omit marker left in the tool + output. Images that fail a gate — or lose the per-request cap — degrade to + the omit marker with a reason; never a request-level 400. URL-source images + cannot be gated and always degrade in this mode. Top-level user-message + images become real `input_image` parts. `CCP_GROK_TOOL_IMAGE=reject` restores + the pre-L1 hard error; `inline` is reserved and currently warns and falls + back to the default `omit`. Data URLs and Anthropic image `source.data` + payloads are redacted from traffic captures. - Grok no longer fails requests whose `tool_result` contains image blocks (for example Claude Code `Read` of a PNG): image children degrade to `[image omitted: ]` / `[image omitted: url]` placeholders joined diff --git a/src/config.rs b/src/config.rs index f345d9ae..1ea92c77 100644 --- a/src/config.rs +++ b/src/config.rs @@ -316,6 +316,59 @@ pub fn grok_client_version() -> String { "0.2.93".to_string() } +// --------------------------------------------------------------------------- +// Grok tool-image policy (CCP_GROK_TOOL_IMAGE) +// --------------------------------------------------------------------------- + +/// How the Grok translator treats Anthropic `image` blocks (tool results and +/// top-level user messages). `omit` is the safe default: degrade to the L1 +/// placeholder string. `reattach` keeps the placeholder in the tool output and +/// additionally appends a user message carrying the images as `input_image` +/// data URLs. `reject` restores the pre-L1 hard error. `inline` is reserved +/// for ticket 04 and currently warns and falls back to `omit`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GrokToolImageMode { + Omit, + Reattach, + Reject, +} + +pub fn parse_grok_tool_image_mode(raw: Option<&str>) -> GrokToolImageMode { + match raw.map(str::trim) { + Some("reattach") => GrokToolImageMode::Reattach, + Some("reject") => GrokToolImageMode::Reject, + // `inline` (ticket 04) and any unknown/empty value degrade to omit. + _ => GrokToolImageMode::Omit, + } +} + +pub fn grok_tool_image_mode() -> GrokToolImageMode { + parse_grok_tool_image_mode(std::env::var("CCP_GROK_TOOL_IMAGE").ok().as_deref()) +} + +/// Warn once at startup when a reserved/unknown mode was requested. Called from +/// the Grok provider constructor rather than per request. +pub fn warn_grok_tool_image_mode_once(log: &crate::logging::Logger) { + match std::env::var("CCP_GROK_TOOL_IMAGE").ok().as_deref() { + Some("inline") => log.warn( + "CCP_GROK_TOOL_IMAGE=inline is not implemented yet; falling back to omit", + None, + ), + Some(other) if !matches!(other, "omit" | "reattach" | "reject") => { + let mut fields = serde_json::Map::new(); + fields.insert( + "value".to_string(), + serde_json::Value::String(other.to_string()), + ); + log.warn( + "unrecognized CCP_GROK_TOOL_IMAGE value; falling back to omit", + Some(fields), + ); + } + _ => {} + } +} + pub fn is_verbose() -> bool { log_verbose() } diff --git a/src/providers/grok/count_tokens.rs b/src/providers/grok/count_tokens.rs index b456e6e2..c78a418d 100644 --- a/src/providers/grok/count_tokens.rs +++ b/src/providers/grok/count_tokens.rs @@ -43,6 +43,9 @@ fn count_input_item(item: &GrokInputItem) -> u64 { GrokContentPart::InputText { text } | GrokContentPart::OutputText { text } => { approx_token_count(text) } + // Images count toward tokens; charge a flat per-image estimate + // instead of counting base64 payload characters. + GrokContentPart::InputImage { .. } => IMAGE_TOKEN_ESTIMATE, }) .sum(), GrokInputItem::FunctionCall { @@ -52,6 +55,8 @@ fn count_input_item(item: &GrokInputItem) -> u64 { } } +const IMAGE_TOKEN_ESTIMATE: u64 = 1_024; + fn approx_token_count(text: &str) -> u64 { if text.is_empty() { return 0; diff --git a/src/providers/grok/mod.rs b/src/providers/grok/mod.rs index fa6d2692..94eb93c7 100644 --- a/src/providers/grok/mod.rs +++ b/src/providers/grok/mod.rs @@ -38,6 +38,7 @@ pub struct GrokProvider { } impl GrokProvider { pub fn new() -> Self { + crate::config::warn_grok_tool_image_mode_once(&crate::logging::create_logger("grok")); Self { client: Arc::new( client::GrokClient::new( diff --git a/src/providers/grok/translate/request.rs b/src/providers/grok/translate/request.rs index 7b9f0587..9dacf817 100644 --- a/src/providers/grok/translate/request.rs +++ b/src/providers/grok/translate/request.rs @@ -4,6 +4,8 @@ use serde::Serialize; use serde_json::Value; use crate::anthropic::schema::{Message, MessagesRequest}; +use crate::config::GrokToolImageMode; +use crate::providers::translate_shared::{ImageSource, image_source_to_url}; #[derive(Debug, Clone, Serialize)] pub struct GrokResponsesRequest { @@ -46,6 +48,8 @@ pub enum GrokContentPart { InputText { text: String }, #[serde(rename = "output_text")] OutputText { text: String }, + #[serde(rename = "input_image")] + InputImage { image_url: String }, } #[derive(Debug, Clone, Serialize)] @@ -108,6 +112,14 @@ pub enum GrokToolChoice { pub fn translate_request( req: &MessagesRequest, model: String, +) -> anyhow::Result { + translate_request_with_mode(req, model, crate::config::grok_tool_image_mode()) +} + +pub fn translate_request_with_mode( + req: &MessagesRequest, + model: String, + image_mode: GrokToolImageMode, ) -> anyhow::Result { reject_unknown_top_level(req)?; let mut instructions = parse_system(req.extra.get("system"))?; @@ -148,8 +160,9 @@ pub fn translate_request( }; let mut call_ids = HashSet::new(); let mut input = Vec::new(); + let mut budget = ReattachBudget::new(&req.messages, image_mode); for message in &req.messages { - parse_message(message, &mut input, &mut call_ids)?; + parse_message(message, &mut input, &mut call_ids, image_mode, &mut budget)?; } Ok(GrokResponsesRequest { model, @@ -163,6 +176,83 @@ pub fn translate_request( }) } +/// Pre-scans the request to decide which gate-passing images survive the +/// request-wide "keep only the last few" cap, so the main walk can mark +/// cap-dropped images with a reason instead of silently dropping pixels. +struct ReattachBudget { + /// Per-image-URL count of gate-passing occurrences that must be dropped + /// (the oldest ones) before survivors begin, so the *last* few images win. + drops: std::collections::HashMap, +} + +impl ReattachBudget { + fn new(messages: &[Message], image_mode: GrokToolImageMode) -> Self { + let mut drops: std::collections::HashMap = std::collections::HashMap::new(); + if image_mode == GrokToolImageMode::Reattach { + let passing: Vec = messages + .iter() + .flat_map(candidate_image_blocks) + .filter_map(|block| { + let source = parse_image_source(block)?; + gate_image(&source) + .ok() + .map(|()| image_source_to_url(&source)) + }) + .collect(); + // Drop the oldest occurrences beyond the cap, keeping the last few. + for url in passing + .iter() + .take(passing.len().saturating_sub(MAX_REATTACHED_IMAGES)) + { + *drops.entry(url.clone()).or_insert(0) += 1; + } + } + Self { drops } + } + + /// Record a gate-passing image; `true` when it survives the request cap. + /// Oldest occurrences are dropped first, in conversation order. + fn admit(&mut self, image_url: &str) -> bool { + let Some(drops) = self.drops.get_mut(image_url) else { + return true; + }; + *drops -= 1; + if *drops == 0 { + self.drops.remove(image_url); + } + false + } +} + +/// Every image block in one message: top-level user images and tool_result +/// image children, in conversation order. +fn candidate_image_blocks(message: &Message) -> Vec<&serde_json::Map> { + let mut blocks = Vec::new(); + if let Value::Array(items) = &message.content { + for item in items { + let Some(object) = item.as_object() else { + continue; + }; + match object.get("type").and_then(Value::as_str) { + Some("image") => blocks.push(object), + Some("tool_result") => { + if let Some(Value::Array(parts)) = object.get("content") { + for part in parts { + if let Some(part) = part.as_object() + && part.get("type").and_then(Value::as_str) == Some("image") + { + blocks.push(part); + } + } + } + } + _ => {} + } + } + } + blocks +} + fn append_guidance(instructions: &mut Option, guidance: &str) { *instructions = Some(match instructions.take() { Some(existing) if !existing.is_empty() => format!("{existing}\n\n{guidance}"), @@ -400,6 +490,8 @@ fn parse_message( message: &Message, out: &mut Vec, calls: &mut HashSet, + image_mode: GrokToolImageMode, + budget: &mut ReattachBudget, ) -> anyhow::Result<()> { if !["system", "user", "assistant"].contains(&message.role.as_str()) { anyhow::bail!("unsupported message role"); @@ -454,9 +546,37 @@ fn parse_message( { anyhow::bail!("unsupported image block field"); } - let placeholder = image_placeholder(object) - .ok_or_else(|| anyhow::anyhow!("image source is invalid"))?; - content.push(GrokContentPart::InputText { text: placeholder }); + match image_mode { + GrokToolImageMode::Reject => { + anyhow::bail!("unsupported content block: image"); + } + GrokToolImageMode::Omit => { + let placeholder = image_placeholder(object) + .ok_or_else(|| anyhow::anyhow!("image source is invalid"))?; + content.push(GrokContentPart::InputText { text: placeholder }); + } + GrokToolImageMode::Reattach => { + let source = parse_image_source(object) + .ok_or_else(|| anyhow::anyhow!("image source is invalid"))?; + match gate_image(&source) { + Ok(()) => { + let image_url = image_source_to_url(&source); + if budget.admit(&image_url) { + content.push(GrokContentPart::InputImage { image_url }); + } else { + content.push(GrokContentPart::InputText { + text: omit_with_reason(&source, &cap_reason()), + }); + } + } + Err(reason) => { + content.push(GrokContentPart::InputText { + text: omit_with_reason(&source, &reason), + }); + } + } + } + } } ("assistant", "tool_use") => { if object.keys().any(|key| { @@ -519,6 +639,7 @@ fn parse_message( let value = object .get("content") .ok_or_else(|| anyhow::anyhow!("tool result content is required"))?; + let mut tool_result_images: Vec = Vec::new(); let output = match value { Value::String(text) => text.clone(), Value::Array(parts) => { @@ -547,10 +668,48 @@ fn parse_message( { anyhow::bail!("unsupported image child"); } - let placeholder = image_placeholder(part).ok_or_else(|| { - anyhow::anyhow!("tool result image source is invalid") - })?; - texts.push(placeholder); + match image_mode { + GrokToolImageMode::Reject => { + anyhow::bail!("tool result supports text children only"); + } + GrokToolImageMode::Omit => { + let placeholder = + image_placeholder(part).ok_or_else(|| { + anyhow::anyhow!( + "tool result image source is invalid" + ) + })?; + texts.push(placeholder); + } + GrokToolImageMode::Reattach => { + let source = parse_image_source(part).ok_or_else(|| { + anyhow::anyhow!("tool result image source is invalid") + })?; + match gate_image(&source) { + Ok(()) => { + let image_url = image_source_to_url(&source); + if budget.admit(&image_url) { + texts.push(image_placeholder(part).ok_or_else( + || { + anyhow::anyhow!( + "tool result image source is invalid" + ) + }, + )?); + tool_result_images.push(image_url); + } else { + texts.push(omit_with_reason( + &source, + &cap_reason(), + )); + } + } + Err(reason) => { + texts.push(omit_with_reason(&source, &reason)); + } + } + } + } continue; } if part.get("type").and_then(Value::as_str) != Some("text") @@ -576,6 +735,15 @@ fn parse_message( call_id: id.into(), output, }); + if image_mode == GrokToolImageMode::Reattach && !tool_result_images.is_empty() { + out.push(GrokInputItem::Message { + role: "user".into(), + content: tool_result_images + .into_iter() + .map(|image_url| GrokContentPart::InputImage { image_url }) + .collect(), + }); + } } _ => anyhow::bail!("unsupported content block: {typ}"), } @@ -609,6 +777,184 @@ fn image_placeholder(object: &serde_json::Map) -> Option } } +// --------------------------------------------------------------------------- +// L2a reattach gates (upstream-verified limits; see grok-tool-image/handoff.md) +// --------------------------------------------------------------------------- + +/// Upstream rejects images whose smallest side is under this many pixels. +const MIN_IMAGE_SIDE_PX: u32 = 8; +/// Upstream rejects images whose area is under this many square pixels. +const MIN_IMAGE_AREA_PX: u64 = 512; +/// Decoded RGB(A) payload cap; larger images are degraded to the omit marker. +const MAX_IMAGE_DECODED_BYTES: u64 = 5 * 1024 * 1024; +/// Only the last few images per tool result are reattached. +const MAX_REATTACHED_IMAGES: usize = 4; + +/// Parse an Anthropic image block into a shared `ImageSource`. Returns `None` +/// for structurally invalid blocks (missing fields, unknown source type). +fn parse_image_source(object: &serde_json::Map) -> Option { + let source = object.get("source")?.as_object()?; + let source_type = source.get("type").and_then(Value::as_str)?; + match source_type { + "base64" => { + let media_type = source + .get("media_type") + .and_then(Value::as_str) + .filter(|media_type| !media_type.is_empty())?; + let data = source.get("data").and_then(Value::as_str)?; + Some(ImageSource { + media_type: media_type.to_string(), + data: data.to_string(), + source_type: source_type.to_string(), + }) + } + "url" => { + let url = source + .get("url") + .and_then(Value::as_str) + .filter(|url| !url.is_empty())?; + Some(ImageSource { + media_type: "image/*".to_string(), + data: url.to_string(), + source_type: source_type.to_string(), + }) + } + _ => None, + } +} + +/// Gate a candidate image against the upstream-verified limits. Returns the +/// omit reason on failure so the caller can degrade just this one image. +fn gate_image(source: &ImageSource) -> Result<(), String> { + if source.source_type == "url" { + // The proxy cannot gate a remote image without fetching it (dimensions + // and decoded size are unknown), and an unverifiable image can 400 the + // whole turn upstream — so URL sources never reattach. + return Err("url source cannot be gated".to_string()); + } + use base64::Engine; + let bytes = base64::engine::general_purpose::STANDARD + .decode(source.data.as_bytes()) + .map_err(|_| "undecodable base64".to_string())?; + let (width, height) = image_dimensions(&bytes, &source.media_type) + .ok_or_else(|| format!("unreadable dimensions for {}", source.media_type))?; + let min_side = width.min(height); + if min_side < MIN_IMAGE_SIDE_PX { + return Err(format!( + "{width}x{height} below minimum side {MIN_IMAGE_SIDE_PX}px" + )); + } + let area = width as u64 * height as u64; + if area < MIN_IMAGE_AREA_PX { + return Err(format!( + "{width}x{height} below minimum area {MIN_IMAGE_AREA_PX}px" + )); + } + // Decoded raster is at least 3 bytes per pixel (RGB); RGBA is 4. Use 3 as + // a conservative lower bound so borderline images stay under the cap. + let decoded = area.saturating_mul(3); + if decoded > MAX_IMAGE_DECODED_BYTES { + return Err(format!( + "{width}x{height} too large (decoded ~{}MB > {}MB cap)", + decoded / (1024 * 1024), + MAX_IMAGE_DECODED_BYTES / (1024 * 1024) + )); + } + Ok(()) +} + +/// Render the L1 omit marker with a gate-failure reason appended. +fn omit_with_reason(source: &ImageSource, reason: &str) -> String { + let base = if source.source_type == "url" { + "[image omitted: url]".to_string() + } else { + format!("[image omitted: {}]", source.media_type) + }; + format!("{base} ({reason})") +} + +/// Reason attached to images that passed every per-image gate but lost the +/// request-wide "keep only the last few" cap. +fn cap_reason() -> String { + format!("only the last {MAX_REATTACHED_IMAGES} images are attached per request") +} + +/// Extract pixel dimensions from the encoded image header. Supports PNG, JPEG +/// and GIF; returns `None` for unknown/corrupt formats. +fn image_dimensions(bytes: &[u8], media_type: &str) -> Option<(u32, u32)> { + match media_type { + "image/png" => png_dimensions(bytes), + "image/jpeg" => jpeg_dimensions(bytes), + "image/gif" => gif_dimensions(bytes), + _ => sniff_dimensions(bytes), + } +} + +fn sniff_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + png_dimensions(bytes) + .or_else(|| jpeg_dimensions(bytes)) + .or_else(|| gif_dimensions(bytes)) +} + +fn png_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + // Signature (8) + IHDR length (4) + "IHDR" (4) + width (4) + height (4). + if bytes.len() < 24 || &bytes[..8] != b"\x89PNG\r\n\x1a\n" || &bytes[12..16] != b"IHDR" { + return None; + } + let width = u32::from_be_bytes(bytes[16..20].try_into().ok()?); + let height = u32::from_be_bytes(bytes[20..24].try_into().ok()?); + Some((width, height)) +} + +fn gif_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + // "GIF87a"/"GIF89a" (6) + width (2 LE) + height (2 LE). + if bytes.len() < 10 || !matches!(&bytes[..6], b"GIF87a" | b"GIF89a") { + return None; + } + let width = u16::from_le_bytes(bytes[6..8].try_into().ok()?) as u32; + let height = u16::from_le_bytes(bytes[8..10].try_into().ok()?) as u32; + Some((width, height)) +} + +fn jpeg_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + // Walk SOI-delimited segments to the first SOF0..SOF15 frame header. + if bytes.len() < 4 || bytes[0] != 0xff || bytes[1] != 0xd8 { + return None; + } + let mut cursor = 2usize; + while cursor + 4 <= bytes.len() { + if bytes[cursor] != 0xff { + return None; + } + let marker = bytes[cursor + 1]; + // Standalone markers without a length field. + if marker == 0xd8 || marker == 0x01 || (0xd0..=0xd7).contains(&marker) { + cursor += 2; + continue; + } + let segment_len = + u16::from_be_bytes(bytes[cursor + 2..cursor + 4].try_into().ok()?) as usize; + if segment_len < 2 || cursor + 2 + segment_len > bytes.len() { + return None; + } + let is_sof = matches!( + marker, + 0xc0..=0xc3 | 0xc5..=0xc7 | 0xc9..=0xcb | 0xcd..=0xcf + ); + if is_sof { + // Segment: length (2) + precision (1) + height (2) + width (2). + if segment_len < 7 { + return None; + } + let height = u16::from_be_bytes(bytes[cursor + 5..cursor + 7].try_into().ok()?) as u32; + let width = u16::from_be_bytes(bytes[cursor + 7..cursor + 9].try_into().ok()?) as u32; + return Some((width, height)); + } + cursor += 2 + segment_len; + } + None +} + fn valid_cache_control(value: Option<&Value>) -> bool { let Some(value) = value else { return true }; let Some(object) = value.as_object() else { @@ -1128,4 +1474,392 @@ mod tests { assert_eq!(translated["input"][1]["output"], "result"); assert!(!translated.to_string().contains("cache_control")); } + + // --------------------------------------------------------------------- + // L2a: CCP_GROK_TOOL_IMAGE flag + reattach vision tests + // --------------------------------------------------------------------- + + // 32x32 solid red PNG (valid dimensions, passes all gates). + const PNG_32_RED: &str = "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAAKElEQVR4nO3NsQ0AAAzCMP5/un0CNkuZ41wybXsHAAAAAAAAAAAAxR4yw/wuPL6QkAAAAABJRU5ErkJggg=="; + // 1x1 red PNG (min-side + area gate failure). + const PNG_1_RED: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC"; + // 8x8 red PNG (area gate failure: 64 < 512). + const PNG_8_RED: &str = "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAEklEQVR4nGP4z8CAFWEXHbQSACj/P8Fu7N9hAAAAAElFTkSuQmCC"; + + #[test] + fn grok_tool_image_mode_parses_flag_values() { + use crate::config::{GrokToolImageMode, parse_grok_tool_image_mode}; + assert_eq!(parse_grok_tool_image_mode(None), GrokToolImageMode::Omit); + assert_eq!( + parse_grok_tool_image_mode(Some("omit")), + GrokToolImageMode::Omit + ); + assert_eq!( + parse_grok_tool_image_mode(Some("reattach")), + GrokToolImageMode::Reattach + ); + assert_eq!( + parse_grok_tool_image_mode(Some("reject")), + GrokToolImageMode::Reject + ); + // inline is not implemented yet (ticket 04): warn + fall back to omit. + assert_eq!( + parse_grok_tool_image_mode(Some("inline")), + GrokToolImageMode::Omit + ); + // Unknown values are the safe default. + assert_eq!( + parse_grok_tool_image_mode(Some("bogus")), + GrokToolImageMode::Omit + ); + assert_eq!( + parse_grok_tool_image_mode(Some("")), + GrokToolImageMode::Omit + ); + } + + #[test] + fn reattach_tool_result_image_emits_following_user_message_with_input_image() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"text","text":"screenshot"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reattach, + ) + .unwrap(), + ) + .unwrap(); + // function_call_output keeps the L1 omit marker (text stays verbatim). + assert_eq!(translated["input"][1]["type"], "function_call_output"); + assert_eq!( + translated["input"][1]["output"], + "screenshot\n[image omitted: image/png]" + ); + // A user message follows carrying the image as an input_image data URL. + assert_eq!(translated["input"][2]["type"], "message"); + assert_eq!(translated["input"][2]["role"], "user"); + let parts = translated["input"][2]["content"].as_array().unwrap(); + assert_eq!(parts.len(), 1); + assert_eq!(parts[0]["type"], "input_image"); + let url = parts[0]["image_url"].as_str().unwrap(); + assert!( + url.starts_with("data:image/png;base64,"), + "unexpected image_url prefix: {}", + &url[..url.len().min(40)] + ); + assert!(url.contains(PNG_32_RED)); + } + + #[test] + fn reattach_top_level_user_image_becomes_input_image_part() { + let request: MessagesRequest = serde_json::from_value(serde_json::json!({ + "model":"grok-4.5", + "messages":[{"role":"user","content":[ + {"type":"text","text":"what color is this?"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]}] + })) + .unwrap(); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reattach, + ) + .unwrap(), + ) + .unwrap(); + let parts = translated["input"][0]["content"].as_array().unwrap(); + assert_eq!(parts[0]["type"], "input_text"); + assert_eq!(parts[0]["text"], "what color is this?"); + assert_eq!(parts[1]["type"], "input_image"); + assert!( + parts[1]["image_url"] + .as_str() + .unwrap() + .starts_with("data:image/png;base64,") + ); + } + + #[test] + fn reattach_degrades_tiny_image_to_omit_with_reason_never_error() { + for (data, dims) in [(PNG_1_RED, "1x1"), (PNG_8_RED, "8x8")] { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":data}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reattach, + ) + .unwrap(), + ) + .unwrap(); + let output = translated["input"][1]["output"].as_str().unwrap(); + assert!( + output.starts_with("[image omitted: image/png"), + "unexpected output: {output}" + ); + assert!(output.contains(dims), "reason must cite dims: {output}"); + // No reattached message: gated-out images never produce input_image. + assert_eq!(translated["input"].as_array().unwrap().len(), 2); + } + } + + #[test] + fn reattach_degrades_oversized_image_to_omit_with_reason() { + // 6000x6000 solid PNG: decoded raw size ~108MB > 5MB cap. Deflate of a + // solid scanline is tiny, so the base64 fixture stays small. + let big = oversized_png_base64(6000, 6000); + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":big}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reattach, + ) + .unwrap(), + ) + .unwrap(); + let output = translated["input"][1]["output"].as_str().unwrap(); + assert!( + output.starts_with("[image omitted: image/png"), + "unexpected output: {output}" + ); + assert!( + output.contains("too large"), + "reason must cite size: {output}" + ); + assert_eq!(translated["input"].as_array().unwrap().len(), 2); + } + + #[test] + fn reattach_keeps_only_last_four_images_per_request() { + let mut result_parts = Vec::new(); + for _ in 0..6 { + result_parts.push(serde_json::json!({"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}})); + } + let request: MessagesRequest = serde_json::from_value(serde_json::json!({ + "model":"grok-4.5", + "messages":[ + {"role":"assistant","content":[{"type":"tool_use","id":"call_1","name":"lookup","input":{}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"call_1","content":result_parts}]} + ] + })) + .unwrap(); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reattach, + ) + .unwrap(), + ) + .unwrap(); + let input = translated["input"].as_array().unwrap(); + // function_call + function_call_output + one reattached user message. + assert_eq!(input.len(), 3); + let parts = input[2]["content"].as_array().unwrap(); + assert_eq!(parts.len(), 4, "only the last 4 images survive"); + assert!(parts.iter().all(|part| part["type"] == "input_image")); + // The two cap-dropped images degrade with a reason in the tool output. + let output = input[1]["output"].as_str().unwrap(); + let cap_drops = output.matches("only the last 4 images").count(); + assert_eq!( + cap_drops, 2, + "cap-dropped images must carry a reason: {output}" + ); + let attached = output.matches("[image omitted: image/png]\n").count() + + if output.ends_with("[image omitted: image/png]") { + 1 + } else { + 0 + }; + assert_eq!(attached, 4, "surviving images keep the plain L1 marker"); + } + + #[test] + fn reattach_cap_applies_across_tool_results_in_one_request() { + // Two tool results with 3 images each: 6 gate-passing images, so only + // the last 4 across the request reattach (the whole second result's 3 + // plus the first result's last 1). + let image = || serde_json::json!({"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}}); + let request: MessagesRequest = serde_json::from_value(serde_json::json!({ + "model":"grok-4.5", + "messages":[ + {"role":"assistant","content":[ + {"type":"tool_use","id":"call_1","name":"lookup","input":{}}, + {"type":"tool_use","id":"call_2","name":"lookup","input":{}} + ]}, + {"role":"user","content":[ + {"type":"tool_result","tool_use_id":"call_1","content":[image(),image(),image()]}, + {"type":"tool_result","tool_use_id":"call_2","content":[image(),image(),image()]} + ]} + ] + })) + .unwrap(); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reattach, + ) + .unwrap(), + ) + .unwrap(); + let input = translated["input"].as_array().unwrap(); + let total_attached: usize = input + .iter() + .map(|item| { + item["content"] + .as_array() + .map(|parts| { + parts + .iter() + .filter(|part| part["type"] == "input_image") + .count() + }) + .unwrap_or(0) + }) + .sum(); + assert_eq!( + total_attached, 4, + "cap is request-wide, not per tool result" + ); + } + + #[test] + fn reattach_degrades_url_images_with_reason_instead_of_reattaching() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"url","url":"https://example.invalid/a.png"}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reattach, + ) + .unwrap(), + ) + .unwrap(); + let output = translated["input"][1]["output"].as_str().unwrap(); + assert!( + output.starts_with("[image omitted: url]"), + "unexpected output: {output}" + ); + assert!( + output.contains("cannot be gated"), + "reason must explain the skip: {output}" + ); + // No input_image part anywhere. + assert!(!translated.to_string().contains("input_image")); + } + + #[test] + fn reject_mode_restores_old_bail_on_tool_result_images() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]} + ])); + let result = translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reject, + ); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("tool result supports text children only") + ); + } + + #[test] + fn reject_mode_restores_old_bail_on_top_level_user_images() { + let request: MessagesRequest = serde_json::from_value(serde_json::json!({ + "model":"grok-4.5", + "messages":[{"role":"user","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]}] + })) + .unwrap(); + let result = translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reject, + ); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("unsupported content block: image") + ); + } + + /// Build a solid-color PNG of the given size, returned as base64. Used to + /// create dimension-valid but decoded-size-oversized fixtures without a + /// large test file. + fn oversized_png_base64(width: u32, height: u32) -> String { + use base64::Engine; + use std::io::Write as _; + let mut png = Vec::new(); + png.extend_from_slice(&[0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a]); + let mut ihdr = Vec::new(); + ihdr.extend_from_slice(&width.to_be_bytes()); + ihdr.extend_from_slice(&height.to_be_bytes()); + ihdr.extend_from_slice(&[8, 2, 0, 0, 0]); // 8-bit RGB + write_chunk(&mut png, b"IHDR", &ihdr); + // One scanline: filter byte + width * 3 zero bytes, repeated. + let mut raw = Vec::with_capacity((width as usize * 3 + 1) * height as usize); + for _ in 0..height { + raw.push(0u8); + raw.extend(std::iter::repeat_n(0u8, width as usize * 3)); + } + let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(&raw).unwrap(); + let idat = encoder.finish().unwrap(); + write_chunk(&mut png, b"IDAT", &idat); + write_chunk(&mut png, b"IEND", &[]); + base64::engine::general_purpose::STANDARD.encode(png) + } + + fn write_chunk(out: &mut Vec, kind: &[u8; 4], data: &[u8]) { + out.extend_from_slice(&(data.len() as u32).to_be_bytes()); + out.extend_from_slice(kind); + out.extend_from_slice(data); + let mut crc_data = Vec::with_capacity(4 + data.len()); + crc_data.extend_from_slice(kind); + crc_data.extend_from_slice(data); + out.extend_from_slice(&crc32(&crc_data).to_be_bytes()); + } + + fn crc32(data: &[u8]) -> u32 { + let mut crc: u32 = 0xffff_ffff; + for byte in data { + crc ^= *byte as u32; + for _ in 0..8 { + let mask = (crc & 1).wrapping_neg(); + crc = (crc >> 1) ^ (0xedb8_8320 & mask); + } + } + !crc + } } diff --git a/src/traffic.rs b/src/traffic.rs index 780d2b81..1d36bf16 100644 --- a/src/traffic.rs +++ b/src/traffic.rs @@ -357,7 +357,15 @@ fn redact_traffic_with_depth(value: &Value, depth: u16) -> Value { let mut out = Map::new(); for (key, value) in map { let normalized = key.to_lowercase(); - if REDACT_KEYS.contains(&normalized.as_str()) + if normalized == "image_url" { + // Grok/OpenAI `input_image` parts carry data URLs; never + // persist the payload in traffic captures. + out.insert(key.clone(), redact_traffic_value(value)); + } else if normalized == "data" && looks_like_image_source(map) { + // Anthropic image blocks carry raw base64 under + // `source.data`; redact it for the same reason. + out.insert(key.clone(), redact_traffic_value(value)); + } else if REDACT_KEYS.contains(&normalized.as_str()) || matches!( normalized.as_str(), "token" @@ -390,10 +398,27 @@ fn redact_traffic_with_depth(value: &Value, depth: u16) -> Value { .map(|value| redact_traffic_with_depth(value, depth + 1)) .collect(), ), + Value::String(text) if is_data_url(text) => { + Value::String(format!("[redacted data-url len={}]", text.len())) + } _ => value.clone(), } } +fn is_data_url(text: &str) -> bool { + text.starts_with("data:image/") && text.contains(";base64,") +} + +/// An object shaped like an Anthropic image source: `type: "base64"` plus a +/// `media_type`. Used to redact only image payloads, not arbitrary `data` keys. +fn looks_like_image_source(map: &Map) -> bool { + map.get("type").and_then(Value::as_str) == Some("base64") + && map + .get("media_type") + .and_then(Value::as_str) + .is_some_and(|media_type| media_type.starts_with("image/")) +} + fn redact_traffic_value(value: &Value) -> Value { match value { Value::String(s) => Value::String(format!("[redacted len={}]", s.len())), @@ -418,3 +443,77 @@ fn set_mode(path: &Path, mode: u32) { fn _provider_alias(_provider: &str) -> Option { None } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn redact_traffic_strips_input_image_data_urls() { + let value = serde_json::json!({ + "input": [ + {"type": "message", "role": "user", "content": [ + {"type": "input_text", "text": "what color?"}, + {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC"} + ]} + ] + }); + let redacted = redact_traffic(&value); + let rendered = redacted.to_string(); + assert!( + !rendered.contains("iVBORw0KGgo"), + "base64 payload leaked: {rendered}" + ); + assert!( + !rendered.contains("data:image/png;base64,iVBOR"), + "data URL payload leaked: {rendered}" + ); + // The image part is still structurally visible (redacted marker). + assert!(rendered.contains("input_image")); + } + + #[test] + fn redact_traffic_strips_inline_base64_image_values() { + let value = serde_json::json!({ + "output": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC" + }); + let redacted = redact_traffic(&value); + let rendered = redacted.to_string(); + assert!(!rendered.contains("iVBORw0KGgo")); + } + + #[test] + fn redact_traffic_strips_anthropic_image_source_data() { + // The incoming Anthropic request body carries raw base64 under + // `source.data` (no data: URL wrapper) — it must not persist either. + let value = serde_json::json!({ + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "image", "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC" + }} + ] + }] + }); + let redacted = redact_traffic(&value); + let rendered = redacted.to_string(); + assert!( + !rendered.contains("iVBORw0KGgo"), + "source.data base64 leaked: {rendered}" + ); + assert!(rendered.contains("redacted")); + // Structure is preserved. + assert!(rendered.contains("image/png")); + } + + #[test] + fn redact_traffic_keeps_unrelated_data_keys() { + let value = serde_json::json!({"data": "some-non-image-payload", "count": 3}); + let redacted = redact_traffic(&value); + assert_eq!(redacted["data"], "some-non-image-payload"); + } +} From 4a67200930897360a379662e11063ea8c2c5f39f Mon Sep 17 00:00:00 2001 From: Desicool Date: Mon, 20 Jul 2026 13:46:32 +0800 Subject: [PATCH 3/5] grok: opt-in vision via CCP_GROK_TOOL_IMAGE=inline (L2b) Inline mode sends image-bearing function call outputs as untagged arrays of input_text and input_image parts. String and text-only outputs retain their plain-string wire shape. The same dimension, decoded-size, URL, and request-wide image gates apply as reattach mode. Per-image failures remain visible as omission text within mixed arrays. Top-level user images use input_image parts in both vision modes. Token counting charges the image estimate for inline parts, and traffic capture redaction preserves array structure without persisting image payloads. --- src/config.rs | 19 +- src/providers/grok/count_tokens.rs | 35 ++- src/providers/grok/translate/request.rs | 382 +++++++++++++++++++++++- src/traffic.rs | 28 ++ 4 files changed, 428 insertions(+), 36 deletions(-) diff --git a/src/config.rs b/src/config.rs index 1ea92c77..bfd2ed61 100644 --- a/src/config.rs +++ b/src/config.rs @@ -324,20 +324,23 @@ pub fn grok_client_version() -> String { /// top-level user messages). `omit` is the safe default: degrade to the L1 /// placeholder string. `reattach` keeps the placeholder in the tool output and /// additionally appends a user message carrying the images as `input_image` -/// data URLs. `reject` restores the pre-L1 hard error. `inline` is reserved -/// for ticket 04 and currently warns and falls back to `omit`. +/// data URLs. `inline` sends the tool output itself as an array of +/// `input_text` + `input_image` parts (string-only outputs still serialize as +/// plain strings). `reject` restores the pre-L1 hard error. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum GrokToolImageMode { Omit, Reattach, + Inline, Reject, } pub fn parse_grok_tool_image_mode(raw: Option<&str>) -> GrokToolImageMode { match raw.map(str::trim) { Some("reattach") => GrokToolImageMode::Reattach, + Some("inline") => GrokToolImageMode::Inline, Some("reject") => GrokToolImageMode::Reject, - // `inline` (ticket 04) and any unknown/empty value degrade to omit. + // Any unknown/empty value degrades to the safe default. _ => GrokToolImageMode::Omit, } } @@ -346,15 +349,11 @@ pub fn grok_tool_image_mode() -> GrokToolImageMode { parse_grok_tool_image_mode(std::env::var("CCP_GROK_TOOL_IMAGE").ok().as_deref()) } -/// Warn once at startup when a reserved/unknown mode was requested. Called from -/// the Grok provider constructor rather than per request. +/// Warn once at startup when an unknown mode was requested. Called from the +/// Grok provider constructor rather than per request. pub fn warn_grok_tool_image_mode_once(log: &crate::logging::Logger) { match std::env::var("CCP_GROK_TOOL_IMAGE").ok().as_deref() { - Some("inline") => log.warn( - "CCP_GROK_TOOL_IMAGE=inline is not implemented yet; falling back to omit", - None, - ), - Some(other) if !matches!(other, "omit" | "reattach" | "reject") => { + Some(other) if !matches!(other, "omit" | "reattach" | "inline" | "reject") => { let mut fields = serde_json::Map::new(); fields.insert( "value".to_string(), diff --git a/src/providers/grok/count_tokens.rs b/src/providers/grok/count_tokens.rs index c78a418d..9f586497 100644 --- a/src/providers/grok/count_tokens.rs +++ b/src/providers/grok/count_tokens.rs @@ -1,4 +1,6 @@ -use super::translate::request::{GrokContentPart, GrokInputItem, GrokResponsesRequest}; +use super::translate::request::{ + GrokContentPart, GrokInputItem, GrokResponsesRequest, GrokToolOutput, +}; const MESSAGE_OVERHEAD_TOKENS: u64 = 4; const TOOL_OVERHEAD_TOKENS: u64 = 4; @@ -37,24 +39,31 @@ pub fn count_tokens(request: &GrokResponsesRequest) -> u64 { fn count_input_item(item: &GrokInputItem) -> u64 { match item { - GrokInputItem::Message { content, .. } => content - .iter() - .map(|part| match part { - GrokContentPart::InputText { text } | GrokContentPart::OutputText { text } => { - approx_token_count(text) - } - // Images count toward tokens; charge a flat per-image estimate - // instead of counting base64 payload characters. - GrokContentPart::InputImage { .. } => IMAGE_TOKEN_ESTIMATE, - }) - .sum(), + GrokInputItem::Message { content, .. } => count_parts(content), GrokInputItem::FunctionCall { name, arguments, .. } => approx_token_count(name) + approx_token_count(arguments), - GrokInputItem::FunctionCallOutput { output, .. } => approx_token_count(output), + GrokInputItem::FunctionCallOutput { output, .. } => match output { + GrokToolOutput::Text(text) => approx_token_count(text), + GrokToolOutput::Parts(parts) => count_parts(parts), + }, } } +fn count_parts(parts: &[GrokContentPart]) -> u64 { + parts + .iter() + .map(|part| match part { + GrokContentPart::InputText { text } | GrokContentPart::OutputText { text } => { + approx_token_count(text) + } + // Images count toward tokens; charge a flat per-image estimate + // instead of counting base64 payload characters. + GrokContentPart::InputImage { .. } => IMAGE_TOKEN_ESTIMATE, + }) + .sum() +} + const IMAGE_TOKEN_ESTIMATE: u64 = 1_024; fn approx_token_count(text: &str) -> u64 { diff --git a/src/providers/grok/translate/request.rs b/src/providers/grok/translate/request.rs index 9dacf817..ecb46a21 100644 --- a/src/providers/grok/translate/request.rs +++ b/src/providers/grok/translate/request.rs @@ -38,7 +38,20 @@ pub enum GrokInputItem { arguments: String, }, #[serde(rename = "function_call_output")] - FunctionCallOutput { call_id: String, output: String }, + FunctionCallOutput { + call_id: String, + output: GrokToolOutput, + }, +} + +/// Tool output payload: a plain string, or (in `inline` image mode) an array +/// of `input_text` + `input_image` parts. Untagged so string outputs serialize +/// byte-identically to the pre-inline shape. +#[derive(Debug, Clone, Serialize)] +#[serde(untagged)] +pub enum GrokToolOutput { + Text(String), + Parts(Vec), } #[derive(Debug, Clone, Serialize)] @@ -188,7 +201,10 @@ struct ReattachBudget { impl ReattachBudget { fn new(messages: &[Message], image_mode: GrokToolImageMode) -> Self { let mut drops: std::collections::HashMap = std::collections::HashMap::new(); - if image_mode == GrokToolImageMode::Reattach { + if matches!( + image_mode, + GrokToolImageMode::Reattach | GrokToolImageMode::Inline + ) { let passing: Vec = messages .iter() .flat_map(candidate_image_blocks) @@ -555,7 +571,7 @@ fn parse_message( .ok_or_else(|| anyhow::anyhow!("image source is invalid"))?; content.push(GrokContentPart::InputText { text: placeholder }); } - GrokToolImageMode::Reattach => { + GrokToolImageMode::Reattach | GrokToolImageMode::Inline => { let source = parse_image_source(object) .ok_or_else(|| anyhow::anyhow!("image source is invalid"))?; match gate_image(&source) { @@ -641,9 +657,10 @@ fn parse_message( .ok_or_else(|| anyhow::anyhow!("tool result content is required"))?; let mut tool_result_images: Vec = Vec::new(); let output = match value { - Value::String(text) => text.clone(), + Value::String(text) => GrokToolOutput::Text(text.clone()), Value::Array(parts) => { let mut texts = Vec::new(); + let mut inline_parts: Vec = Vec::new(); for part in parts { let part = part.as_object().ok_or_else(|| { anyhow::anyhow!("tool result child must be an object") @@ -709,6 +726,33 @@ fn parse_message( } } } + GrokToolImageMode::Inline => { + let source = parse_image_source(part).ok_or_else(|| { + anyhow::anyhow!("tool result image source is invalid") + })?; + match gate_image(&source) { + Ok(()) => { + let image_url = image_source_to_url(&source); + if budget.admit(&image_url) { + inline_parts.push( + GrokContentPart::InputImage { image_url }, + ); + } else { + inline_parts.push(GrokContentPart::InputText { + text: omit_with_reason( + &source, + &cap_reason(), + ), + }); + } + } + Err(reason) => { + inline_parts.push(GrokContentPart::InputText { + text: omit_with_reason(&source, &reason), + }); + } + } + } } continue; } @@ -720,14 +764,41 @@ fn parse_message( { anyhow::bail!("tool result supports text children only"); } - texts.push( - part.get("text") - .and_then(Value::as_str) - .ok_or_else(|| anyhow::anyhow!("tool result text is invalid"))? - .to_string(), - ); + let text = part + .get("text") + .and_then(Value::as_str) + .ok_or_else(|| anyhow::anyhow!("tool result text is invalid"))? + .to_string(); + if image_mode == GrokToolImageMode::Inline { + inline_parts.push(GrokContentPart::InputText { text }); + } else { + texts.push(text); + } + } + if image_mode == GrokToolImageMode::Inline { + // Text-only results keep the plain-string shape so + // inline mode is invisible unless an image is present. + let mut joined: Option = Some(String::new()); + for part in &inline_parts { + match part { + GrokContentPart::InputText { text } => { + if let Some(acc) = joined.as_mut() { + if !acc.is_empty() { + acc.push('\n'); + } + acc.push_str(text); + } + } + _ => joined = None, + } + } + match joined { + Some(text) => GrokToolOutput::Text(text), + None => GrokToolOutput::Parts(inline_parts), + } + } else { + GrokToolOutput::Text(texts.join("\n")) } - texts.join("\n") } _ => anyhow::bail!("tool result supports text only"), }; @@ -1502,10 +1573,9 @@ mod tests { parse_grok_tool_image_mode(Some("reject")), GrokToolImageMode::Reject ); - // inline is not implemented yet (ticket 04): warn + fall back to omit. assert_eq!( parse_grok_tool_image_mode(Some("inline")), - GrokToolImageMode::Omit + GrokToolImageMode::Inline ); // Unknown values are the safe default. assert_eq!( @@ -1862,4 +1932,290 @@ mod tests { } !crc } + + // --------------------------------------------------------------------- + // L2b: CCP_GROK_TOOL_IMAGE=inline — tool output as a content-part array + // --------------------------------------------------------------------- + + #[test] + fn inline_tool_result_image_emits_array_output_with_input_image_part() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"text","text":"screenshot"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(translated["input"][1]["type"], "function_call_output"); + // output is an untagged array of content parts, not a string. + let parts = translated["input"][1]["output"].as_array().unwrap(); + assert_eq!(parts.len(), 2); + assert_eq!(parts[0]["type"], "input_text"); + assert_eq!(parts[0]["text"], "screenshot"); + assert_eq!(parts[1]["type"], "input_image"); + let url = parts[1]["image_url"].as_str().unwrap(); + assert!( + url.starts_with("data:image/png;base64,"), + "unexpected image_url prefix: {}", + &url[..url.len().min(40)] + ); + assert!(url.contains(PNG_32_RED)); + // No reattached user message: the image rides inside the tool output. + assert_eq!(translated["input"].as_array().unwrap().len(), 2); + } + + #[test] + fn inline_image_only_tool_result_emits_array_with_single_image_part() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + let parts = translated["input"][1]["output"].as_array().unwrap(); + assert_eq!(parts.len(), 1); + assert_eq!(parts[0]["type"], "input_image"); + } + + #[test] + fn inline_text_only_tool_result_serializes_byte_identically_to_omit() { + // String-only outputs (and text-only array results) must keep the + // plain-string shape in every mode — inline included. + let shapes = [ + serde_json::json!({"type":"tool_result","tool_use_id":"call_1","content":"plain string"}), + serde_json::json!({"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"text","text":"first"}, + {"type":"text","text":"second"} + ]}), + ]; + for shape in shapes { + let request = request_with_blocks(serde_json::json!([shape])); + let omit = serde_json::to_string( + &translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Omit, + ) + .unwrap(), + ) + .unwrap(); + let inline = serde_json::to_string( + &translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(omit, inline, "text-only outputs must be byte-identical"); + // And the output field really is a bare JSON string, not an array. + let value: Value = serde_json::from_str(&inline).unwrap(); + assert!(value["input"][1]["output"].is_string()); + } + } + + #[test] + fn inline_string_output_matches_pre_inline_serialization_exactly() { + // Regression: the whole upstream-bound request body for a string tool + // result must serialize exactly as before the GrokToolOutput widening. + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":"result"} + ])); + let body = serde_json::to_string( + &translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + assert!( + body.contains(r#""output":"result""#), + "output must serialize as a bare string: {body}" + ); + assert!(!body.contains(r#""output":["#)); + } + + #[test] + fn inline_all_images_gated_out_collapse_to_string_with_reasons() { + for (data, dims) in [(PNG_1_RED, "1x1"), (PNG_8_RED, "8x8")] { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"text","text":"shot"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":data}} + ]} + ])); + // Gate failures degrade per-image, never a 400 for the whole turn. + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + // All parts ended up as text → collapses back to a plain string. + let output = translated["input"][1]["output"].as_str().unwrap(); + assert!( + output.contains("[image omitted: image/png"), + "unexpected output: {output}" + ); + assert!(output.contains(dims), "reason must cite dims: {output}"); + assert!(output.starts_with("shot\n")); + } + } + + #[test] + fn inline_mixed_passing_and_failing_images_keep_array_shape() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_1_RED}}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + let parts = translated["input"][1]["output"].as_array().unwrap(); + assert_eq!(parts.len(), 2); + // The gated-out image becomes an in-band omit fragment inside the array. + assert_eq!(parts[0]["type"], "input_text"); + assert!( + parts[0]["text"] + .as_str() + .unwrap() + .starts_with("[image omitted: image/png") + ); + assert!(parts[0]["text"].as_str().unwrap().contains("1x1")); + assert_eq!(parts[1]["type"], "input_image"); + } + + #[test] + fn inline_degrades_url_images_to_omit_fragment_inside_array() { + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"url","url":"https://example.invalid/a.png"}}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + let parts = translated["input"][1]["output"].as_array().unwrap(); + assert_eq!(parts.len(), 2); + assert_eq!(parts[0]["type"], "input_text"); + assert!( + parts[0]["text"] + .as_str() + .unwrap() + .contains("cannot be gated") + ); + assert_eq!(parts[1]["type"], "input_image"); + } + + #[test] + fn inline_keeps_only_last_four_images_per_request() { + let mut result_parts = Vec::new(); + for _ in 0..6 { + result_parts.push(serde_json::json!({"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}})); + } + let request: MessagesRequest = serde_json::from_value(serde_json::json!({ + "model":"grok-4.5", + "messages":[ + {"role":"assistant","content":[{"type":"tool_use","id":"call_1","name":"lookup","input":{}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"call_1","content":result_parts}]} + ] + })) + .unwrap(); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + let input = translated["input"].as_array().unwrap(); + // function_call + function_call_output only — no reattached message. + assert_eq!(input.len(), 2); + let parts = input[1]["output"].as_array().unwrap(); + assert_eq!(parts.len(), 6); + let images = parts + .iter() + .filter(|part| part["type"] == "input_image") + .count(); + assert_eq!(images, 4, "only the last 4 images survive"); + let cap_drops = parts + .iter() + .filter(|part| { + part["type"] == "input_text" + && part["text"] + .as_str() + .is_some_and(|text| text.contains("only the last 4 images")) + }) + .count(); + assert_eq!(cap_drops, 2, "cap-dropped images carry a reason"); + } + + #[test] + fn inline_top_level_user_image_becomes_input_image_part() { + let request: MessagesRequest = serde_json::from_value(serde_json::json!({ + "model":"grok-4.5", + "messages":[{"role":"user","content":[ + {"type":"text","text":"what color is this?"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG_32_RED}} + ]}] + })) + .unwrap(); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Inline, + ) + .unwrap(), + ) + .unwrap(); + let parts = translated["input"][0]["content"].as_array().unwrap(); + assert_eq!(parts[0]["type"], "input_text"); + assert_eq!(parts[1]["type"], "input_image"); + assert!( + parts[1]["image_url"] + .as_str() + .unwrap() + .starts_with("data:image/png;base64,") + ); + } } diff --git a/src/traffic.rs b/src/traffic.rs index 1d36bf16..f2cab8f3 100644 --- a/src/traffic.rs +++ b/src/traffic.rs @@ -516,4 +516,32 @@ mod tests { let redacted = redact_traffic(&value); assert_eq!(redacted["data"], "some-non-image-payload"); } + + #[test] + fn redact_traffic_strips_data_urls_in_array_shaped_tool_output() { + // L2b inline mode: function_call_output.output is an array of content + // parts — the image_url inside must be redacted just like message parts. + let value = serde_json::json!({ + "input": [ + {"type": "function_call_output", "call_id": "call_1", "output": [ + {"type": "input_text", "text": "screenshot"}, + {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC"} + ]} + ] + }); + let redacted = redact_traffic(&value); + let rendered = redacted.to_string(); + assert!( + !rendered.contains("iVBORw0KGgo"), + "base64 payload leaked: {rendered}" + ); + assert!( + !rendered.contains("data:image/png;base64,iVBOR"), + "data URL payload leaked: {rendered}" + ); + // Structure and text survive. + assert!(rendered.contains("input_image")); + assert!(rendered.contains("screenshot")); + assert!(rendered.contains("redacted")); + } } From aeadd2774730914938351c0117a1c875f13d8d6d Mon Sep 17 00:00:00 2001 From: Desicool Date: Mon, 20 Jul 2026 16:05:58 +0800 Subject: [PATCH 4/5] grok: harden image modes and traffic redaction Keep startup mode validation consistent with request parsing and apply conservative image gates before opt-in vision payloads reach Grok. Redact Anthropic and provider image representations across nested traffic capture shapes while preserving useful JSON structure. Keep unsupported formats and unverifiable URLs on the per-image omission path instead of failing the request. --- CHANGELOG.md | 25 ++++--- src/config.rs | 8 ++- src/providers/grok/translate/request.rs | 16 +++-- src/traffic.rs | 91 ++++++++++++++++++++++--- 4 files changed, 112 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37a12a3a..2573378c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,17 +54,20 @@ description: Release notes for claude-code-proxy. ## Unreleased -- Grok gains opt-in real vision via `CCP_GROK_TOOL_IMAGE=reattach`: image blocks - that pass the upstream gates (min side 8px, min area 512px, decoded size ≤5MB, - last 4 per request) are reattached as a following user message carrying - `input_image` data URLs, in addition to the omit marker left in the tool - output. Images that fail a gate — or lose the per-request cap — degrade to - the omit marker with a reason; never a request-level 400. URL-source images - cannot be gated and always degrade in this mode. Top-level user-message - images become real `input_image` parts. `CCP_GROK_TOOL_IMAGE=reject` restores - the pre-L1 hard error; `inline` is reserved and currently warns and falls - back to the default `omit`. Data URLs and Anthropic image `source.data` - payloads are redacted from traffic captures. +- Grok gains opt-in real vision via `CCP_GROK_TOOL_IMAGE`: `reattach` keeps + the omit marker in the tool output and re-sends image blocks that pass the + upstream gates (min side 8px, min area 512px², decoded size ≤5MB, last 4 + per request) as a following user message carrying `input_image` data URLs; + `inline` instead sends the tool output itself as an array of `input_text` + + `input_image` parts (string-only outputs still serialize as plain strings, + byte-identical to before). Images that fail a gate — or lose the + per-request cap — degrade to the omit marker with a reason; well-formed + images never cause a request-level 400. URL-source images cannot be gated + and always degrade in `reattach` mode. Top-level user-message images become + real `input_image` parts in both modes. WebP blocks always degrade (their + dimensions are not parsed). `reject` restores the pre-L1 hard error. Data + URLs and Anthropic image `source.data` payloads are redacted from traffic + captures; this redaction now covers `image_url` keys for all providers. - Grok no longer fails requests whose `tool_result` contains image blocks (for example Claude Code `Read` of a PNG): image children degrade to `[image omitted: ]` / `[image omitted: url]` placeholders joined diff --git a/src/config.rs b/src/config.rs index bfd2ed61..595d75fe 100644 --- a/src/config.rs +++ b/src/config.rs @@ -352,8 +352,12 @@ pub fn grok_tool_image_mode() -> GrokToolImageMode { /// Warn once at startup when an unknown mode was requested. Called from the /// Grok provider constructor rather than per request. pub fn warn_grok_tool_image_mode_once(log: &crate::logging::Logger) { - match std::env::var("CCP_GROK_TOOL_IMAGE").ok().as_deref() { - Some(other) if !matches!(other, "omit" | "reattach" | "inline" | "reject") => { + match std::env::var("CCP_GROK_TOOL_IMAGE") + .ok() + .as_deref() + .map(str::trim) + { + Some(other) if !matches!(other, "" | "omit" | "reattach" | "inline" | "reject") => { let mut fields = serde_json::Map::new(); fields.insert( "value".to_string(), diff --git a/src/providers/grok/translate/request.rs b/src/providers/grok/translate/request.rs index ecb46a21..90c5588c 100644 --- a/src/providers/grok/translate/request.rs +++ b/src/providers/grok/translate/request.rs @@ -849,7 +849,10 @@ fn image_placeholder(object: &serde_json::Map) -> Option } // --------------------------------------------------------------------------- -// L2a reattach gates (upstream-verified limits; see grok-tool-image/handoff.md) +// L2a reattach gates (limits verified against cli-chat-proxy.grok.com on +// 2026-07-20: 1x1 and 8x8 rejected, 32x32 accepted; production-scale +// screenshots well above the minimums; both `reattach` and `inline` wire +// shapes returned 200 with correct visual answers) // --------------------------------------------------------------------------- /// Upstream rejects images whose smallest side is under this many pixels. @@ -858,7 +861,7 @@ const MIN_IMAGE_SIDE_PX: u32 = 8; const MIN_IMAGE_AREA_PX: u64 = 512; /// Decoded RGB(A) payload cap; larger images are degraded to the omit marker. const MAX_IMAGE_DECODED_BYTES: u64 = 5 * 1024 * 1024; -/// Only the last few images per tool result are reattached. +/// Only the last few images across the whole request are attached. const MAX_REATTACHED_IMAGES: usize = 4; /// Parse an Anthropic image block into a shared `ImageSource`. Returns `None` @@ -921,9 +924,9 @@ fn gate_image(source: &ImageSource) -> Result<(), String> { "{width}x{height} below minimum area {MIN_IMAGE_AREA_PX}px" )); } - // Decoded raster is at least 3 bytes per pixel (RGB); RGBA is 4. Use 3 as - // a conservative lower bound so borderline images stay under the cap. - let decoded = area.saturating_mul(3); + // Worst-case raster is 8 bytes per pixel (RGBA, 16-bit channels). + // Over-estimating is the safe direction for a pre-flight gate. + let decoded = area.saturating_mul(8); if decoded > MAX_IMAGE_DECODED_BYTES { return Err(format!( "{width}x{height} too large (decoded ~{}MB > {}MB cap)", @@ -951,7 +954,8 @@ fn cap_reason() -> String { } /// Extract pixel dimensions from the encoded image header. Supports PNG, JPEG -/// and GIF; returns `None` for unknown/corrupt formats. +/// and GIF only — other formats (e.g. WebP) always degrade to the omit +/// marker; `None` also covers unknown/corrupt data. fn image_dimensions(bytes: &[u8], media_type: &str) -> Option<(u32, u32)> { match media_type { "image/png" => png_dimensions(bytes), diff --git a/src/traffic.rs b/src/traffic.rs index f2cab8f3..ee50f6ae 100644 --- a/src/traffic.rs +++ b/src/traffic.rs @@ -406,23 +406,31 @@ fn redact_traffic_with_depth(value: &Value, depth: u16) -> Value { } fn is_data_url(text: &str) -> bool { - text.starts_with("data:image/") && text.contains(";base64,") + let lower = text.to_ascii_lowercase(); + lower.starts_with("data:image/") && text.contains(";base64,") } -/// An object shaped like an Anthropic image source: `type: "base64"` plus a -/// `media_type`. Used to redact only image payloads, not arbitrary `data` keys. +/// An object shaped like an Anthropic image source: `type: "base64"` and +/// either an image `media_type` or none at all. Used to redact only image +/// payloads, not arbitrary `data` keys. Missing `media_type` is treated as +/// an image: the capture is written before translation, so a malformed block +/// that the translator would reject must still not persist its payload. fn looks_like_image_source(map: &Map) -> bool { - map.get("type").and_then(Value::as_str) == Some("base64") - && map - .get("media_type") - .and_then(Value::as_str) - .is_some_and(|media_type| media_type.starts_with("image/")) + if map.get("type").and_then(Value::as_str) != Some("base64") { + return false; + } + match map.get("media_type").and_then(Value::as_str) { + Some(media_type) => media_type.to_ascii_lowercase().starts_with("image/"), + None => true, + } } fn redact_traffic_value(value: &Value) -> Value { match value { Value::String(s) => Value::String(format!("[redacted len={}]", s.len())), - Value::Object(_) | Value::Array(_) => Value::String("[redacted]".to_string()), + // Structured values (e.g. Kimi's `{url: ...}` image_url) keep their + // shape so captures stay parseable; only the payload leaves. + Value::Object(_) | Value::Array(_) => redact_traffic(value), _ => Value::String("[redacted]".to_string()), } } @@ -544,4 +552,69 @@ mod tests { assert!(rendered.contains("screenshot")); assert!(rendered.contains("redacted")); } + + #[test] + fn redact_traffic_strips_uppercase_media_type_image_source() { + // Captures are written before translation validates anything, so a + // non-standard casing must still not persist its payload. + let value = serde_json::json!({ + "source": {"type": "base64", "media_type": "IMAGE/PNG", "data": "QUJDREVGRw=="} + }); + let redacted = redact_traffic(&value); + let rendered = redacted.to_string(); + assert!( + !rendered.contains("QUJDREVGRw"), + "payload leaked: {rendered}" + ); + assert!( + redacted["source"]["data"] + .as_str() + .unwrap() + .contains("redacted") + ); + } + + #[test] + fn redact_traffic_strips_image_source_without_media_type() { + // A malformed block the translator would reject must still be redacted + // in the pre-translation capture. + let value = serde_json::json!({ + "source": {"type": "base64", "data": "QUJDREVGRw=="} + }); + let redacted = redact_traffic(&value); + let rendered = redacted.to_string(); + assert!( + !rendered.contains("QUJDREVGRw"), + "payload leaked: {rendered}" + ); + } + + #[test] + fn redact_traffic_strips_uppercase_data_url_scheme() { + let value = serde_json::json!({ + "image_url": "DATA:IMAGE/PNG;base64,iVBORw0KGgoAAAANSUhEUg" + }); + let redacted = redact_traffic(&value); + let rendered = redacted.to_string(); + assert!( + !rendered.contains("iVBORw0KGgo"), + "data URL payload leaked: {rendered}" + ); + } + + #[test] + fn redact_traffic_keeps_structured_image_url_shape() { + // Kimi-style object image_url: redact the payload but keep the object + // structure so captures stay parseable. + let value = serde_json::json!({ + "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg"} + }); + let redacted = redact_traffic(&value); + assert!(redacted["image_url"].is_object(), "shape lost: {redacted}"); + let rendered = redacted.to_string(); + assert!( + !rendered.contains("iVBORw0KGgo"), + "nested payload leaked: {rendered}" + ); + } } From ed3bf32005f28b366850a51c1e487625fce9ad14 Mon Sep 17 00:00:00 2001 From: Raine Virta Date: Tue, 28 Jul 2026 15:25:53 +0200 Subject: [PATCH 5/5] fix verified grok image review findings Use format metadata when estimating decoded image size so ordinary 8-bit screenshots under the upstream limit retain their pixels. Keep conservative alpha accounting for formats that can carry transparency. Redact image source types and data URL markers case-insensitively so malformed pre-translation captures cannot persist image payloads. Pin omission tests to their intended mode so developer environment settings do not affect results. Place the feature notes in the active unreleased changelog section and document the image policy in the current Grok and configuration documentation. --- CHANGELOG.md | 30 +--- docs/src/content/docs/providers/grok.md | 18 ++- .../content/docs/reference/configuration.md | 1 + src/providers/grok/translate/request.rs | 152 +++++++++++++----- src/traffic.rs | 23 ++- 5 files changed, 156 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2573378c..d91f3d1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ title: Changelog description: Release notes for claude-code-proxy. --- +## Unreleased + +- Grok handles images in user messages and tool results without failing requests. + Images are omitted by default, with opt-in vision through + `CCP_GROK_TOOL_IMAGE`. Traffic captures redact image payloads. + ## v0.1.25 (2026-07-24) - Kimi users can select Kimi K3 with the `kimi-k3` or `k3` model name, including @@ -52,30 +58,6 @@ description: Release notes for claude-code-proxy. reasoning-token usage. `CCP_COMPACT_EFFORT` can choose a different cap or disable the behavior. ([#67](https://github.com/raine/claude-code-proxy/pull/67)) -## Unreleased - -- Grok gains opt-in real vision via `CCP_GROK_TOOL_IMAGE`: `reattach` keeps - the omit marker in the tool output and re-sends image blocks that pass the - upstream gates (min side 8px, min area 512px², decoded size ≤5MB, last 4 - per request) as a following user message carrying `input_image` data URLs; - `inline` instead sends the tool output itself as an array of `input_text` + - `input_image` parts (string-only outputs still serialize as plain strings, - byte-identical to before). Images that fail a gate — or lose the - per-request cap — degrade to the omit marker with a reason; well-formed - images never cause a request-level 400. URL-source images cannot be gated - and always degrade in `reattach` mode. Top-level user-message images become - real `input_image` parts in both modes. WebP blocks always degrade (their - dimensions are not parsed). `reject` restores the pre-L1 hard error. Data - URLs and Anthropic image `source.data` payloads are redacted from traffic - captures; this redaction now covers `image_url` keys for all providers. -- Grok no longer fails requests whose `tool_result` contains image blocks (for - example Claude Code `Read` of a PNG): image children degrade to - `[image omitted: ]` / `[image omitted: url]` placeholders joined - with text children, matching the Codex translator. Top-level user-message - image blocks degrade to the same placeholder instead of erroring. -- Grok multi-part text `tool_result` content is now joined with `\n` between - parts (previously concatenated without a separator), mirroring Codex. - ## v0.1.21 (2026-07-15) - The monitor shows session token activity trends at common terminal widths, diff --git a/docs/src/content/docs/providers/grok.md b/docs/src/content/docs/providers/grok.md index 342e8f51..d634db8b 100644 --- a/docs/src/content/docs/providers/grok.md +++ b/docs/src/content/docs/providers/grok.md @@ -44,12 +44,28 @@ Claude Code hosted search tools map to Grok-native tools: ## Multimodal support -Grok support in claude-code-proxy focuses on text, reasoning, function tools, and hosted search. Treat image and other media behavior as unsupported unless the current source and tests explicitly cover it. +`CCP_GROK_TOOL_IMAGE` controls image blocks in user messages and tool results: + +- `omit`, the default, replaces each image with an `[image omitted: ...]` + placeholder. The model does not receive the pixels. +- `reattach` keeps the placeholder in each tool result and sends accepted images + in a following user message. +- `inline` sends accepted tool-result images alongside text as `input_image` + parts. Text-only outputs retain their string shape. +- `reject` returns the image validation error used by older versions. + +Vision modes accept base64 PNG, JPEG, and GIF images with a minimum side of 8 +pixels, a minimum area of 512 square pixels, and a decoded size up to 5 MB. At +most the last four accepted images in a request are sent. Images that fail a +gate, WebP images, and remote URL sources degrade to placeholders with a reason. + +Traffic captures redact Anthropic image data and upstream image data URLs. ## Configuration - `CCP_GROK_BASE_URL` or `grok.baseUrl` changes the API base URL. - `CCP_GROK_CLIENT_VERSION` or `grok.clientVersion` changes the client version header. +- `CCP_GROK_TOOL_IMAGE` selects `omit`, `reattach`, `inline`, or `reject`. See [Configuration](/reference/configuration/) for defaults. diff --git a/docs/src/content/docs/reference/configuration.md b/docs/src/content/docs/reference/configuration.md index dde63afc..66cdb8cb 100644 --- a/docs/src/content/docs/reference/configuration.md +++ b/docs/src/content/docs/reference/configuration.md @@ -98,6 +98,7 @@ All keys are optional. An unreadable file, malformed JSON, or incompatible field | --- | --- | --- | --- | | `CCP_GROK_BASE_URL` | `grok.baseUrl` | `https://cli-chat-proxy.grok.com/v1` | Changes the Responses API base URL. | | `CCP_GROK_CLIENT_VERSION` | `grok.clientVersion` | `0.2.93` | Changes the Grok client version header. | +| `CCP_GROK_TOOL_IMAGE` | none | `omit` | Selects `omit`, `reattach`, `inline`, or `reject` image handling. | ## Cursor Agent diff --git a/src/providers/grok/translate/request.rs b/src/providers/grok/translate/request.rs index 90c5588c..15c4ac2e 100644 --- a/src/providers/grok/translate/request.rs +++ b/src/providers/grok/translate/request.rs @@ -910,8 +910,10 @@ fn gate_image(source: &ImageSource) -> Result<(), String> { let bytes = base64::engine::general_purpose::STANDARD .decode(source.data.as_bytes()) .map_err(|_| "undecodable base64".to_string())?; - let (width, height) = image_dimensions(&bytes, &source.media_type) + let raster = image_raster(&bytes, &source.media_type) .ok_or_else(|| format!("unreadable dimensions for {}", source.media_type))?; + let width = raster.width; + let height = raster.height; let min_side = width.min(height); if min_side < MIN_IMAGE_SIDE_PX { return Err(format!( @@ -924,9 +926,7 @@ fn gate_image(source: &ImageSource) -> Result<(), String> { "{width}x{height} below minimum area {MIN_IMAGE_AREA_PX}px" )); } - // Worst-case raster is 8 bytes per pixel (RGBA, 16-bit channels). - // Over-estimating is the safe direction for a pre-flight gate. - let decoded = area.saturating_mul(8); + let decoded = area.saturating_mul(raster.bytes_per_pixel); if decoded > MAX_IMAGE_DECODED_BYTES { return Err(format!( "{width}x{height} too large (decoded ~{}MB > {}MB cap)", @@ -953,45 +953,77 @@ fn cap_reason() -> String { format!("only the last {MAX_REATTACHED_IMAGES} images are attached per request") } -/// Extract pixel dimensions from the encoded image header. Supports PNG, JPEG -/// and GIF only — other formats (e.g. WebP) always degrade to the omit -/// marker; `None` also covers unknown/corrupt data. -fn image_dimensions(bytes: &[u8], media_type: &str) -> Option<(u32, u32)> { +/// Extract raster dimensions and a conservative decoded byte width from the +/// encoded image header. PNG accounting reserves alpha when the format may +/// carry transparency. GIF accounting reserves an RGBA output pixel. +#[derive(Clone, Copy)] +struct ImageRaster { + width: u32, + height: u32, + bytes_per_pixel: u64, +} + +fn image_raster(bytes: &[u8], media_type: &str) -> Option { match media_type { - "image/png" => png_dimensions(bytes), - "image/jpeg" => jpeg_dimensions(bytes), - "image/gif" => gif_dimensions(bytes), - _ => sniff_dimensions(bytes), + "image/png" => png_raster(bytes), + "image/jpeg" => jpeg_raster(bytes), + "image/gif" => gif_raster(bytes), + _ => sniff_raster(bytes), } } -fn sniff_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { - png_dimensions(bytes) - .or_else(|| jpeg_dimensions(bytes)) - .or_else(|| gif_dimensions(bytes)) +fn sniff_raster(bytes: &[u8]) -> Option { + png_raster(bytes) + .or_else(|| jpeg_raster(bytes)) + .or_else(|| gif_raster(bytes)) } -fn png_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { - // Signature (8) + IHDR length (4) + "IHDR" (4) + width (4) + height (4). - if bytes.len() < 24 || &bytes[..8] != b"\x89PNG\r\n\x1a\n" || &bytes[12..16] != b"IHDR" { +fn png_raster(bytes: &[u8]) -> Option { + // Signature (8) + IHDR length (4) + "IHDR" (4) + IHDR fields. + if bytes.len() < 26 + || &bytes[..8] != b"\x89PNG\r\n\x1a\n" + || u32::from_be_bytes(bytes[8..12].try_into().ok()?) != 13 + || &bytes[12..16] != b"IHDR" + { return None; } let width = u32::from_be_bytes(bytes[16..20].try_into().ok()?); let height = u32::from_be_bytes(bytes[20..24].try_into().ok()?); - Some((width, height)) + let bit_depth = bytes[24]; + let color_type = bytes[25]; + let bytes_per_channel = match bit_depth { + 1 | 2 | 4 | 8 => 1, + 16 => 2, + _ => return None, + }; + let channels = match color_type { + // Grayscale and RGB may carry transparency in a tRNS chunk. + 0 | 4 => 2, + 2 | 3 | 6 => 4, + _ => return None, + }; + Some(ImageRaster { + width, + height, + bytes_per_pixel: bytes_per_channel * channels, + }) } -fn gif_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { +fn gif_raster(bytes: &[u8]) -> Option { // "GIF87a"/"GIF89a" (6) + width (2 LE) + height (2 LE). if bytes.len() < 10 || !matches!(&bytes[..6], b"GIF87a" | b"GIF89a") { return None; } let width = u16::from_le_bytes(bytes[6..8].try_into().ok()?) as u32; let height = u16::from_le_bytes(bytes[8..10].try_into().ok()?) as u32; - Some((width, height)) + Some(ImageRaster { + width, + height, + bytes_per_pixel: 4, + }) } -fn jpeg_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { +fn jpeg_raster(bytes: &[u8]) -> Option { // Walk SOI-delimited segments to the first SOF0..SOF15 frame header. if bytes.len() < 4 || bytes[0] != 0xff || bytes[1] != 0xd8 { return None; @@ -1017,13 +1049,23 @@ fn jpeg_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { 0xc0..=0xc3 | 0xc5..=0xc7 | 0xc9..=0xcb | 0xcd..=0xcf ); if is_sof { - // Segment: length (2) + precision (1) + height (2) + width (2). - if segment_len < 7 { + // Segment: length (2), precision (1), dimensions (4), components (1). + if segment_len < 8 { return None; } + let precision = bytes[cursor + 4]; let height = u16::from_be_bytes(bytes[cursor + 5..cursor + 7].try_into().ok()?) as u32; let width = u16::from_be_bytes(bytes[cursor + 7..cursor + 9].try_into().ok()?) as u32; - return Some((width, height)); + let components = bytes[cursor + 9] as usize; + if components == 0 || segment_len < 8 + 3 * components { + return None; + } + let bytes_per_channel = u64::from(precision).div_ceil(8); + return Some(ImageRaster { + width, + height, + bytes_per_pixel: bytes_per_channel.saturating_mul(components as u64), + }); } cursor += 2 + segment_len; } @@ -1369,6 +1411,16 @@ mod tests { .unwrap() } + fn translated_with_mode( + request: &MessagesRequest, + image_mode: crate::config::GrokToolImageMode, + ) -> Value { + serde_json::to_value( + translate_request_with_mode(request, "grok-4.5".into(), image_mode).unwrap(), + ) + .unwrap() + } + #[test] fn grok_translation_rejects_unknown_tool_block_fields() { let mut request = request_with_blocks(serde_json::json!([ @@ -1390,8 +1442,7 @@ mod tests { {"type":"image","source":{"type":"base64","media_type":"image/png","data":"aGVsbG8="}} ]} ])); - let translated = - serde_json::to_value(translate_request(&request, "grok-4.5".into()).unwrap()).unwrap(); + let translated = translated_with_mode(&request, crate::config::GrokToolImageMode::Omit); assert_eq!(translated["input"][1]["type"], "function_call_output"); assert_eq!( translated["input"][1]["output"], @@ -1406,8 +1457,7 @@ mod tests { {"type":"image","source":{"type":"url","url":"https://example.invalid/a.png"}} ]} ])); - let translated = - serde_json::to_value(translate_request(&request, "grok-4.5".into()).unwrap()).unwrap(); + let translated = translated_with_mode(&request, crate::config::GrokToolImageMode::Omit); assert_eq!(translated["input"][1]["output"], "[image omitted: url]"); } @@ -1420,8 +1470,7 @@ mod tests { {"type":"image","source":{"type":"url","url":"https://example.invalid/a.png"}} ]} ])); - let translated = - serde_json::to_value(translate_request(&request, "grok-4.5".into()).unwrap()).unwrap(); + let translated = translated_with_mode(&request, crate::config::GrokToolImageMode::Omit); assert_eq!( translated["input"][1]["output"], "caption\n[image omitted: image/png]\n[image omitted: url]" @@ -1451,8 +1500,7 @@ mod tests { ]}] })) .unwrap(); - let translated = - serde_json::to_value(translate_request(&request, "grok-4.5".into()).unwrap()).unwrap(); + let translated = translated_with_mode(&request, crate::config::GrokToolImageMode::Omit); assert_eq!( translated["input"][0]["content"], serde_json::json!([ @@ -1689,11 +1737,37 @@ mod tests { } } + #[test] + fn reattach_accepts_rgb_image_under_decoded_size_cap() { + // The 8-bit RGB raster is 2.1MB. A format-independent 8-byte estimate + // would incorrectly reject this common screenshot size. + let image = solid_rgb_png_base64(1000, 700); + let request = request_with_blocks(serde_json::json!([ + {"type":"tool_result","tool_use_id":"call_1","content":[ + {"type":"image","source":{"type":"base64","media_type":"image/png","data":image}} + ]} + ])); + let translated = serde_json::to_value( + translate_request_with_mode( + &request, + "grok-4.5".into(), + crate::config::GrokToolImageMode::Reattach, + ) + .unwrap(), + ) + .unwrap(); + assert_eq!( + translated["input"][1]["output"], + "[image omitted: image/png]" + ); + assert_eq!(translated["input"][2]["content"][0]["type"], "input_image"); + } + #[test] fn reattach_degrades_oversized_image_to_omit_with_reason() { - // 6000x6000 solid PNG: decoded raw size ~108MB > 5MB cap. Deflate of a + // 6000x6000 solid PNG: decoded raw size is at least 108MB. Deflate of a // solid scanline is tiny, so the base64 fixture stays small. - let big = oversized_png_base64(6000, 6000); + let big = solid_rgb_png_base64(6000, 6000); let request = request_with_blocks(serde_json::json!([ {"type":"tool_result","tool_use_id":"call_1","content":[ {"type":"image","source":{"type":"base64","media_type":"image/png","data":big}} @@ -1888,10 +1962,8 @@ mod tests { ); } - /// Build a solid-color PNG of the given size, returned as base64. Used to - /// create dimension-valid but decoded-size-oversized fixtures without a - /// large test file. - fn oversized_png_base64(width: u32, height: u32) -> String { + /// Build a solid 8-bit RGB PNG of the given size, returned as base64. + fn solid_rgb_png_base64(width: u32, height: u32) -> String { use base64::Engine; use std::io::Write as _; let mut png = Vec::new(); diff --git a/src/traffic.rs b/src/traffic.rs index ee50f6ae..36d60efa 100644 --- a/src/traffic.rs +++ b/src/traffic.rs @@ -407,7 +407,7 @@ fn redact_traffic_with_depth(value: &Value, depth: u16) -> Value { fn is_data_url(text: &str) -> bool { let lower = text.to_ascii_lowercase(); - lower.starts_with("data:image/") && text.contains(";base64,") + lower.starts_with("data:image/") && lower.contains(";base64,") } /// An object shaped like an Anthropic image source: `type: "base64"` and @@ -416,7 +416,11 @@ fn is_data_url(text: &str) -> bool { /// an image: the capture is written before translation, so a malformed block /// that the translator would reject must still not persist its payload. fn looks_like_image_source(map: &Map) -> bool { - if map.get("type").and_then(Value::as_str) != Some("base64") { + if !map + .get("type") + .and_then(Value::as_str) + .is_some_and(|source_type| source_type.eq_ignore_ascii_case("base64")) + { return false; } match map.get("media_type").and_then(Value::as_str) { @@ -574,6 +578,19 @@ mod tests { ); } + #[test] + fn redact_traffic_strips_uppercase_image_source_type() { + let value = serde_json::json!({ + "source": {"type": "BASE64", "media_type": "image/png", "data": "QUJDREVGRw=="} + }); + let redacted = redact_traffic(&value); + let rendered = redacted.to_string(); + assert!( + !rendered.contains("QUJDREVGRw"), + "payload leaked: {rendered}" + ); + } + #[test] fn redact_traffic_strips_image_source_without_media_type() { // A malformed block the translator would reject must still be redacted @@ -607,7 +624,7 @@ mod tests { // Kimi-style object image_url: redact the payload but keep the object // structure so captures stay parseable. let value = serde_json::json!({ - "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg"} + "image_url": {"url": "DATA:IMAGE/PNG;BASE64,iVBORw0KGgoAAAANSUhEUg"} }); let redacted = redact_traffic(&value); assert!(redacted["image_url"].is_object(), "shape lost: {redacted}");