From d4a05585e1b01a5427d7afa45fc30455b287107e Mon Sep 17 00:00:00 2001 From: "luzeyang (INT)" Date: Wed, 29 Jul 2026 15:51:41 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E5=8C=96=E5=9B=BE=E7=89=87=E6=B6=88=E6=81=AF=E8=BE=93=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Op::SendMessage 新增可选结构化输入 UserMessageInput（Text/LocalImage），纯文本路径行为不变 - ContentBlock 新增 LocalImage 本地图片引用，会话持久化只存 workspace 相对路径与元数据，不存 Base64 - turn loop 构建 provider 请求前对消息克隆物化：workspace 相对路径校验、20MB 上限、魔数 MIME 嗅探，文件缺失明确报错 - Anthropic 遇 data: URL 转换为原生 base64 source；Chat/Responses 图片块序列化补契约测试 - 收口 Base64 泄漏面：mouse_ui 占位渲染、runtime_api 返回 metadata、context_report 固定估算 - image_analyze 加固：20MB 上限、真实文件头检测、更新工具描述（仅用于未随当前消息原生提供的图片） - image_analyze 请求不再携带 temperature：kimi-for-coding 等模型只接受默认值，显式 temperature 会被 400 拒绝 Signed-off-by: luzeyang (INT) --- crates/tui/src/client/anthropic.rs | 71 +++- crates/tui/src/client/chat.rs | 62 ++++ crates/tui/src/client/responses.rs | 41 +++ crates/tui/src/compaction.rs | 11 +- crates/tui/src/context_report.rs | 17 +- crates/tui/src/core/engine.rs | 133 +++++++- crates/tui/src/core/engine/tests.rs | 315 +++++++++++++++++ crates/tui/src/core/engine/turn_loop.rs | 26 +- crates/tui/src/core/ops.rs | 38 +++ crates/tui/src/main.rs | 3 +- crates/tui/src/models.rs | 16 + crates/tui/src/purge.rs | 12 + crates/tui/src/rlm/session.rs | 12 + crates/tui/src/runtime_api/sessions.rs | 28 +- crates/tui/src/runtime_threads.rs | 1 + crates/tui/src/seam_manager.rs | 3 +- crates/tui/src/session_manager.rs | 82 ++++- crates/tui/src/tui/mouse_ui.rs | 17 +- crates/tui/src/tui/notifications.rs | 2 +- crates/tui/src/tui/session_picker.rs | 3 + crates/tui/src/tui/ui.rs | 1 + crates/tui/src/utils.rs | 3 +- crates/tui/src/vision/image_input.rs | 437 ++++++++++++++++++++++++ crates/tui/src/vision/mod.rs | 1 + crates/tui/src/vision/tools.rs | 161 ++++----- crates/tui/src/working_set.rs | 6 +- 26 files changed, 1391 insertions(+), 111 deletions(-) create mode 100644 crates/tui/src/vision/image_input.rs diff --git a/crates/tui/src/client/anthropic.rs b/crates/tui/src/client/anthropic.rs index 00f6597e2b..e311ec8d15 100644 --- a/crates/tui/src/client/anthropic.rs +++ b/crates/tui/src/client/anthropic.rs @@ -381,10 +381,32 @@ fn content_block_to_anthropic(block: &ContentBlock) -> Option { } Some(value) } - ContentBlock::ImageUrl { image_url } => Some(json!({ - "type": "image", - "source": { "type": "url", "url": image_url.url }, - })), + ContentBlock::ImageUrl { image_url } => { + // Anthropic Messages does not accept a `url` source carrying a + // `data:` URL — inline images (materialized local images) must be + // split into the native base64 source shape. Remote URLs keep the + // existing passthrough. + Some( + match crate::vision::image_input::split_data_url(&image_url.url) { + Some((media_type, data)) => json!({ + "type": "image", + "source": { + "type": "base64", + "media_type": media_type, + "data": data, + }, + }), + None => json!({ + "type": "image", + "source": { "type": "url", "url": image_url.url }, + }), + }, + ) + } + // Local image references are materialized into `ImageUrl` before a + // request reaches the client; like server-tool blocks they have no + // wire equivalent of their own. + ContentBlock::LocalImage { .. } => None, // Server-tool block types are DeepSeek/internal concepts with no // Anthropic client-side wire equivalent. ContentBlock::ServerToolUse { .. } @@ -1141,6 +1163,47 @@ mod tests { assert_eq!(message, "upstream blew up"); } + #[test] + fn image_block_with_data_url_uses_native_base64_source() { + // Contract: Anthropic Messages rejects a `url` source carrying a + // `data:` URL — inline images must be split into the native base64 + // source shape. + let block = ContentBlock::ImageUrl { + image_url: crate::models::ImageUrlContent { + url: "data:image/jpeg;base64,/9j/4AAQ".to_string(), + }, + }; + + let value = content_block_to_anthropic(&block).expect("image block converts"); + + assert_eq!(value["type"], json!("image")); + assert_eq!( + value["source"], + json!({ + "type": "base64", + "media_type": "image/jpeg", + "data": "/9j/4AAQ", + }), + "data: URL must be split into Anthropic's native base64 source; got {value}" + ); + } + + #[test] + fn image_block_with_remote_url_keeps_url_source() { + let block = ContentBlock::ImageUrl { + image_url: crate::models::ImageUrlContent { + url: "https://example.com/cat.png".to_string(), + }, + }; + + let value = content_block_to_anthropic(&block).expect("image block converts"); + + assert_eq!( + value["source"], + json!({ "type": "url", "url": "https://example.com/cat.png" }) + ); + } + #[test] fn messages_url_tolerates_v1_suffix() { assert_eq!( diff --git a/crates/tui/src/client/chat.rs b/crates/tui/src/client/chat.rs index 9456676856..88ad1a4c6e 100644 --- a/crates/tui/src/client/chat.rs +++ b/crates/tui/src/client/chat.rs @@ -1721,6 +1721,10 @@ fn build_chat_messages_with_reasoning( ContentBlock::ServerToolUse { .. } | ContentBlock::ToolSearchToolResult { .. } | ContentBlock::CodeExecutionToolResult { .. } => {} + // Local image references are materialized into `ImageUrl` + // before a request reaches the client; they have no wire + // equivalent of their own. + ContentBlock::LocalImage { .. } => {} } } @@ -3928,6 +3932,64 @@ mod stream_decoder_tests { assert_eq!(built[0]["content"], "internal runtime event"); } + #[test] + fn request_builder_emits_openai_image_url_parts_for_inline_images() { + // Contract: OpenAI Chat / vLLM-compatible wire shape keeps inline + // images as `image_url` parts carrying the data URL verbatim. + let data_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg=="; + let messages = vec![Message { + role: "user".to_string(), + content: vec![ + ContentBlock::Text { + text: "what is in this picture?".to_string(), + cache_control: None, + }, + ContentBlock::ImageUrl { + image_url: crate::models::ImageUrlContent { + url: data_url.to_string(), + }, + }, + ], + }]; + + let built = build_chat_messages(None, &messages, "deepseek-v4-flash"); + + assert_eq!(built.len(), 1); + assert_eq!(built[0]["role"], "user"); + assert_eq!( + built[0]["content"], + json!([ + { "type": "text", "text": "what is in this picture?" }, + { "type": "image_url", "image_url": { "url": data_url } }, + ]), + "chat wire must carry image_url parts with the data URL; got {}", + built[0]["content"] + ); + } + + #[test] + fn request_builder_emits_pure_image_user_message_without_text_part() { + let data_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg=="; + let messages = vec![Message { + role: "user".to_string(), + content: vec![ContentBlock::ImageUrl { + image_url: crate::models::ImageUrlContent { + url: data_url.to_string(), + }, + }], + }]; + + let built = build_chat_messages(None, &messages, "deepseek-v4-flash"); + + assert_eq!(built.len(), 1); + assert_eq!( + built[0]["content"], + json!([{ "type": "image_url", "image_url": { "url": data_url } }]), + "pure-image message must not fabricate a text part; got {}", + built[0]["content"] + ); + } + fn tool_use_message(id: &str, name: &str, input: Value) -> Message { Message { role: "assistant".to_string(), diff --git a/crates/tui/src/client/responses.rs b/crates/tui/src/client/responses.rs index 628a62b3cc..f64559780b 100644 --- a/crates/tui/src/client/responses.rs +++ b/crates/tui/src/client/responses.rs @@ -500,6 +500,10 @@ fn convert_messages_to_responses_input(request: &MessageRequest) -> Vec { "image_url": image_url.url, })); } + // Local image references are materialized into + // `ImageUrl` before a request reaches the client; + // they have no wire equivalent of their own. + ContentBlock::LocalImage { .. } => {} ContentBlock::ToolResult { tool_use_id, content, @@ -789,6 +793,43 @@ mod tests { } } + #[test] + fn responses_input_emits_input_image_for_inline_images() { + // Contract: OpenAI Responses wire shape carries inline images as + // `input_image` items with the data URL in `image_url`. + let data_url = "data:image/webp;base64,UklGRhIAAABXRUJQ"; + let mut request = minimal_responses_request(); + request.messages = vec![Message { + role: "user".to_string(), + content: vec![ + ContentBlock::Text { + text: "describe".to_string(), + cache_control: None, + }, + ContentBlock::ImageUrl { + image_url: crate::models::ImageUrlContent { + url: data_url.to_string(), + }, + }, + ], + }]; + + let items = convert_messages_to_responses_input(&request); + + assert_eq!( + items, + vec![json!({ + "type": "message", + "role": "user", + "content": [ + { "type": "input_text", "text": "describe" }, + { "type": "input_image", "image_url": data_url }, + ], + })], + "responses wire must carry input_image items; got {items:?}" + ); + } + fn test_codex_config(server: &MockServer) -> Config { Config { provider: Some("openai-codex".to_string()), diff --git a/crates/tui/src/compaction.rs b/crates/tui/src/compaction.rs index 248caa8092..64b2fecd5a 100644 --- a/crates/tui/src/compaction.rs +++ b/crates/tui/src/compaction.rs @@ -317,7 +317,8 @@ fn message_text(msg: &Message) -> String { ContentBlock::ServerToolUse { .. } | ContentBlock::ToolSearchToolResult { .. } | ContentBlock::CodeExecutionToolResult { .. } - | ContentBlock::ImageUrl { .. } => {} + | ContentBlock::ImageUrl { .. } + | ContentBlock::LocalImage { .. } => {} } } text @@ -342,7 +343,8 @@ fn extract_paths_from_message(message: &Message, workspace: Option<&Path>) -> Ve ContentBlock::ServerToolUse { .. } | ContentBlock::ToolSearchToolResult { .. } | ContentBlock::CodeExecutionToolResult { .. } - | ContentBlock::ImageUrl { .. } => Vec::new(), + | ContentBlock::ImageUrl { .. } + | ContentBlock::LocalImage { .. } => Vec::new(), }; paths.extend(candidates); } @@ -627,7 +629,7 @@ fn estimate_tokens_for_message(message: &Message, include_thinking: bool) -> usi // sessions. Use a conservative flat per-image estimate (vision // tiles are typically ~1k tokens); erring high compacts slightly // early rather than overflowing. - ContentBlock::ImageUrl { .. } => IMAGE_TOKEN_ESTIMATE, + ContentBlock::ImageUrl { .. } | ContentBlock::LocalImage { .. } => IMAGE_TOKEN_ESTIMATE, ContentBlock::ServerToolUse { .. } | ContentBlock::ToolSearchToolResult { .. } | ContentBlock::CodeExecutionToolResult { .. } => 0, @@ -1497,7 +1499,8 @@ fn build_formatted_summary_request( ContentBlock::ServerToolUse { .. } | ContentBlock::ToolSearchToolResult { .. } | ContentBlock::CodeExecutionToolResult { .. } - | ContentBlock::ImageUrl { .. } => {} + | ContentBlock::ImageUrl { .. } + | ContentBlock::LocalImage { .. } => {} } } } diff --git a/crates/tui/src/context_report.rs b/crates/tui/src/context_report.rs index 348043de05..3f4419c6f7 100644 --- a/crates/tui/src/context_report.rs +++ b/crates/tui/src/context_report.rs @@ -624,7 +624,14 @@ fn add_message_entries(builder: &mut ReportBuilder, messages: &[Message]) { for (index, message) in messages.iter().enumerate() { for block in &message.content { - let tokens = estimate_text_tokens_conservative(&content_block_text(block)); + // Images never expand to their URL/payload here: a flat per-image + // estimate keeps the report honest without cloning Base64. + let tokens = match block { + ContentBlock::ImageUrl { .. } | ContentBlock::LocalImage { .. } => { + IMAGE_TOKEN_ESTIMATE + } + _ => estimate_text_tokens_conservative(&content_block_text(block)), + }; match block { ContentBlock::ToolResult { .. } | ContentBlock::ToolSearchToolResult { .. } @@ -687,10 +694,16 @@ fn content_block_text(block: &ContentBlock) -> String { ContentBlock::ToolUse { input, .. } | ContentBlock::ServerToolUse { input, .. } => { input.to_string() } - ContentBlock::ImageUrl { image_url } => image_url.url.clone(), + // Image payloads are never inlined for estimation; the caller applies + // `IMAGE_TOKEN_ESTIMATE` instead. + ContentBlock::ImageUrl { .. } | ContentBlock::LocalImage { .. } => String::new(), } } +/// Flat per-image token estimate, mirroring compaction's image heuristic +/// (vision tiles are typically ~1k tokens) without reading the image payload. +const IMAGE_TOKEN_ESTIMATE: usize = 1000; + fn pressure_label(percent: Option) -> &'static str { // Delegate to the unified pressure thresholds so this diagnostic label can't // drift from `context_budget::PressureLevel`. `None` (unknown window) keeps diff --git a/crates/tui/src/core/engine.rs b/crates/tui/src/core/engine.rs index ef76f23427..d85f2f0357 100644 --- a/crates/tui/src/core/engine.rs +++ b/crates/tui/src/core/engine.rs @@ -71,7 +71,8 @@ use super::authority::agent_approval_mode_for_turn; use super::authority::{TurnAuthority, effective_input_policy, shell_policy_for_mode}; use super::events::{Event, TurnOutcomeStatus, TurnRoute}; use super::ops::{ - Op, ProviderRuntimeStatus, SessionSnapshot, USER_SHELL_TOOL_ID_PREFIX, UserInputProvenance, + Op, ProviderRuntimeStatus, SessionSnapshot, USER_SHELL_TOOL_ID_PREFIX, UserInputBlock, + UserInputProvenance, UserMessageInput, }; use super::session::Session; use super::tool_parser; @@ -1552,6 +1553,7 @@ impl Engine { EngineRunInput::Operation(op) => match *op { Op::SendMessage { content, + input, mode, route, compaction, @@ -1575,6 +1577,7 @@ impl Engine { } => { self.handle_send_message( content, + input, mode, *route, *compaction, @@ -2091,6 +2094,7 @@ impl Engine { let mode = self.current_mode; self.handle_send_message( new_message, + None, mode, route, self.config.compaction.clone(), @@ -2431,6 +2435,73 @@ impl Engine { } } + /// Build a user message from structured input blocks, validating every + /// local image reference (workspace boundary, size cap, real signature) + /// before it enters session history. Persisted blocks stay references — + /// Base64 only ever exists on the short-lived materialized request clone. + /// + /// Block order: user text/images first, `turn_meta` last — the same + /// prefix-cache contract as + /// [`Self::user_text_message_with_turn_metadata_for_route_and_provenance`]. + /// A pure-image message (no text block) is allowed. + async fn user_structured_message_with_turn_metadata( + &self, + blocks: Vec, + meta_text: &str, + routed_model: &str, + auto_model: bool, + reasoning_effort: Option<&str>, + reasoning_effort_auto: bool, + provenance: UserInputProvenance, + ) -> Result { + let turn_metadata = self.turn_metadata_block( + routed_model, + auto_model, + reasoning_effort, + reasoning_effort_auto, + provenance, + meta_text, + ); + let mut content = Vec::with_capacity(blocks.len() + 1); + for block in blocks { + match block { + UserInputBlock::Text { text } => { + // Drop empty text blocks: Anthropic rejects zero-length + // text parts, and they carry no meaning. + if !text.trim().is_empty() { + content.push(ContentBlock::Text { + text, + cache_control: None, + }); + } + } + UserInputBlock::LocalImage { + relative_path, + mime_type, + display_name, + } => { + let byte_size = crate::vision::image_input::validate_local_image_reference( + &self.session.workspace, + &relative_path, + &mime_type, + ) + .await?; + content.push(ContentBlock::LocalImage { + relative_path, + mime_type, + display_name, + byte_size, + }); + } + } + } + content.push(turn_metadata); + Ok(Message { + role: "user".to_string(), + content, + }) + } + async fn handle_idle_subagent_completion(&mut self, first: SubAgentCompletion) { let mut completions = Vec::new(); if let Some(completion) = @@ -2487,6 +2558,7 @@ impl Engine { let recorded = self .handle_send_message( content, + None, self.current_mode, route, self.config.compaction.clone(), @@ -2613,6 +2685,7 @@ impl Engine { async fn handle_send_message( &mut self, content: String, + input: Option, mode: AppMode, route: ResolvedRuntimeRoute, compaction: CompactionConfig, @@ -2756,15 +2829,54 @@ impl Engine { .observe_user_message(&content, &self.session.workspace); let force_update_plan_first = should_force_update_plan_first(input_policy.mode, &content); - // Add user message to session - let user_msg = self.user_text_message_with_turn_metadata_for_route_and_provenance( - content, - &model, - auto_model, - reasoning_effort.as_deref(), - reasoning_effort_auto, - provenance, - ); + // Add user message to session. Structured input keeps local images as + // workspace-relative references (never Base64); every reference is + // preflight-validated before it enters history, and an invalid image + // fails the turn instead of being silently dropped. + let user_msg = match input { + Some(input) if !input.blocks.is_empty() => { + match self + .user_structured_message_with_turn_metadata( + input.blocks, + &content, + &model, + auto_model, + reasoning_effort.as_deref(), + reasoning_effort_auto, + provenance, + ) + .await + { + Ok(message) => message, + Err(err) => { + let message = err.to_string(); + let _ = self + .tx_event + .send(Event::error(ErrorEnvelope::classify(message.clone(), true))) + .await; + let _ = self + .tx_event + .send(Event::TurnComplete { + usage: turn.usage.clone(), + status: TurnOutcomeStatus::Failed, + error: Some(message), + tool_catalog: None, + base_url: None, + }) + .await; + return false; + } + } + } + _ => self.user_text_message_with_turn_metadata_for_route_and_provenance( + content, + &model, + auto_model, + reasoning_effort.as_deref(), + reasoning_effort_auto, + provenance, + ), + }; self.session.add_message(user_msg); let previous_goal_objective = self.config.goal_objective.clone(); @@ -3095,6 +3207,7 @@ impl Engine { .tx_op .send(Op::SendMessage { content: continuation, + input: None, mode, route: Box::new(route), compaction: Box::new(self.config.compaction.clone()), diff --git a/crates/tui/src/core/engine/tests.rs b/crates/tui/src/core/engine/tests.rs index d4458fc45c..505103d4c9 100644 --- a/crates/tui/src/core/engine/tests.rs +++ b/crates/tui/src/core/engine/tests.rs @@ -256,6 +256,7 @@ async fn exact_turn_snapshot_restores_custom_endpoint_and_turn_receipt_after_bui handle .send(Op::SendMessage { content: "verify exact route".to_string(), + input: None, mode: AppMode::Agent, route: Box::new( resolve_runtime_route(&config, ApiProvider::Custom, Some("local-model")) @@ -350,6 +351,7 @@ async fn goal_continuation_resolves_updated_authoritative_route_after_active_tur handle .send(Op::SendMessage { content: "first turn".to_string(), + input: None, mode: AppMode::Agent, route: resolved_route_for_test(&config, "local-model"), compaction: Box::new(CompactionConfig::default()), @@ -465,6 +467,7 @@ async fn host_managed_engine_does_not_self_dispatch_goal_continuation() { handle .send(Op::SendMessage { content: "one host-owned turn".to_string(), + input: None, mode: AppMode::Agent, route: resolved_route_for_test(&config, "local-model"), compaction: Box::new(CompactionConfig::default()), @@ -573,6 +576,7 @@ async fn host_managed_engine_defers_idle_subagent_completion_to_explicit_turn() handle .send(Op::SendMessage { content: "claim the next turn".to_string(), + input: None, mode: AppMode::Agent, route: resolved_route_for_test(&config, "local-model"), compaction: Box::new(CompactionConfig::default()), @@ -1160,6 +1164,7 @@ fn resolved_route_for_test( fn external_user_message_op(content: &str, mode: AppMode, config: &Config) -> Op { Op::SendMessage { content: content.to_string(), + input: None, mode, route: resolved_route_for_test(config, crate::config::DEFAULT_TEXT_MODEL), compaction: Box::new(CompactionConfig::default()), @@ -1304,6 +1309,310 @@ async fn injected_model_drives_real_engine_navigation_trajectory() { task.await.expect("engine task"); } +/// PNG signature plus a few payload bytes — enough for magic-byte sniffing. +const TEST_PNG_BYTES: [u8; 11] = [ + 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x01, 0x02, 0x03, +]; + +fn with_structured_input(op: &mut Op, blocks: Vec) { + if let Op::SendMessage { input, .. } = op { + *input = Some(UserMessageInput { blocks }); + } +} + +#[tokio::test] +async fn structured_image_input_persists_reference_and_materializes_request() { + use crate::llm_client::mock::{MockLlmClient, canned}; + + let workspace = tempdir().expect("tempdir"); + let attachments = workspace.path().join("attachments"); + fs::create_dir_all(&attachments).expect("mkdir attachments"); + fs::write(attachments.join("photo.png"), TEST_PNG_BYTES).expect("write png"); + + let mock = std::sync::Arc::new(MockLlmClient::new(vec![canned::simple_text_turn( + "I see the image.", + )])); + let client: crate::core::model_client::SharedModelClient = mock.clone(); + let (engine, handle) = Engine::new_with_model_client( + deterministic_engine_config(workspace.path()), + &Config::default(), + client, + ); + let task = tokio::spawn(engine.run()); + + let mut op = external_user_message_op("look at this picture", AppMode::Agent, &Config::default()); + with_structured_input( + &mut op, + vec![ + UserInputBlock::Text { + text: "look at this picture".to_string(), + }, + UserInputBlock::LocalImage { + relative_path: PathBuf::from("attachments/photo.png"), + mime_type: "image/png".to_string(), + display_name: "photo.png".to_string(), + }, + ], + ); + handle.send(op).await.expect("send structured image turn"); + + let mut rx = handle.rx_event.write().await; + while let Some(event) = tokio::time::timeout(model_turn_event_timeout(), rx.recv()) + .await + .expect("timed out waiting for structured image turn") + { + if let Event::TurnComplete { status, error, .. } = event { + assert_eq!(status, TurnOutcomeStatus::Completed, "{error:?}"); + break; + } + } + drop(rx); + + // The provider request carries the materialized inline image on a clone... + let requests = mock.captured_requests(); + assert_eq!(requests.len(), 1); + let user_message = requests[0] + .messages + .iter() + .find(|m| m.role == "user") + .expect("user message in request"); + let image_url = user_message + .content + .iter() + .find_map(|block| match block { + ContentBlock::ImageUrl { image_url } => Some(image_url.url.clone()), + _ => None, + }) + .expect("materialized image block in request"); + assert!( + image_url.starts_with("data:image/png;base64,"), + "request must inline the data URL; got {image_url}" + ); + assert!( + user_message + .content + .iter() + .all(|b| !matches!(b, ContentBlock::LocalImage { .. })), + "no local-image reference may reach the provider request" + ); + // ...and turn_meta stays the last block (prefix-cache contract). + assert!( + matches!(user_message.content.last(), Some(ContentBlock::Text { text, .. }) if text.contains("")), + "turn_meta must remain the final block; got {:?}", + user_message.content.last() + ); + + // ...while the session persists only the lightweight reference. + let (tx, snapshot_rx) = tokio::sync::oneshot::channel(); + handle + .send(Op::GetSessionSnapshot { + tx: std::sync::Arc::new(std::sync::Mutex::new(Some(tx))), + }) + .await + .expect("snapshot request"); + let snapshot = tokio::time::timeout(Duration::from_secs(2), snapshot_rx) + .await + .expect("snapshot response") + .expect("snapshot"); + let persisted = serde_json::to_string(&snapshot.messages).expect("serialize messages"); + assert!( + !persisted.contains("data:image"), + "session must never persist a data URL: {persisted}" + ); + let (relative_path, mime_type, display_name, byte_size) = snapshot + .messages + .iter() + .flat_map(|m| &m.content) + .find_map(|b| match b { + ContentBlock::LocalImage { + relative_path, + mime_type, + display_name, + byte_size, + } => Some(( + relative_path.clone(), + mime_type.clone(), + display_name.clone(), + *byte_size, + )), + _ => None, + }) + .expect("local image reference in session"); + assert_eq!(relative_path, PathBuf::from("attachments/photo.png")); + assert_eq!(mime_type, "image/png"); + assert_eq!(display_name, "photo.png"); + assert_eq!(byte_size, TEST_PNG_BYTES.len() as u64); + + handle.send(Op::Shutdown).await.expect("shutdown engine"); + task.await.expect("engine task"); +} + +#[test] +fn forkguard_structured_image_input_persists_reference_and_materializes_request() { + // [pinvou3-fork] forkguard 锚点:结构化图片输入的关键结果式行为——provider 请求内联 + // data URL 且不含本地引用、turn_meta 保持消息末尾、session 只持久化轻量引用。 + // 断言本体在上方同名测试(#[tokio::test] 展开后可直接同步调用), + // 此包装使其纳入 `cargo test forkguard_` 守护面。 + structured_image_input_persists_reference_and_materializes_request(); +} + +#[tokio::test] +async fn structured_pure_image_message_omits_text_block() { + use crate::llm_client::mock::{MockLlmClient, canned}; + + let workspace = tempdir().expect("tempdir"); + let attachments = workspace.path().join("attachments"); + fs::create_dir_all(&attachments).expect("mkdir attachments"); + fs::write(attachments.join("only.png"), TEST_PNG_BYTES).expect("write png"); + + let mock = std::sync::Arc::new(MockLlmClient::new(vec![canned::simple_text_turn( + "described", + )])); + let client: crate::core::model_client::SharedModelClient = mock.clone(); + let (engine, handle) = Engine::new_with_model_client( + deterministic_engine_config(workspace.path()), + &Config::default(), + client, + ); + let task = tokio::spawn(engine.run()); + + let mut op = external_user_message_op("", AppMode::Agent, &Config::default()); + with_structured_input( + &mut op, + vec![UserInputBlock::LocalImage { + relative_path: PathBuf::from("attachments/only.png"), + mime_type: "image/png".to_string(), + display_name: "only.png".to_string(), + }], + ); + handle.send(op).await.expect("send pure image turn"); + + let mut rx = handle.rx_event.write().await; + while let Some(event) = tokio::time::timeout(model_turn_event_timeout(), rx.recv()) + .await + .expect("timed out waiting for pure image turn") + { + if let Event::TurnComplete { status, error, .. } = event { + assert_eq!(status, TurnOutcomeStatus::Completed, "{error:?}"); + break; + } + } + drop(rx); + + let requests = mock.captured_requests(); + assert_eq!(requests.len(), 1); + let user_message = requests[0] + .messages + .iter() + .find(|m| m.role == "user") + .expect("user message in request"); + // A pure-image message must not fabricate user text: the only text block + // is the trailing turn_meta. + let non_meta_text_blocks = user_message + .content + .iter() + .filter(|b| matches!(b, ContentBlock::Text { text, .. } if !text.contains(""))) + .count(); + assert_eq!(non_meta_text_blocks, 0, "{:?}", user_message.content); + assert!( + user_message + .content + .iter() + .any(|b| matches!(b, ContentBlock::ImageUrl { .. })), + "pure image message must still carry the materialized image" + ); + + handle.send(Op::Shutdown).await.expect("shutdown engine"); + task.await.expect("engine task"); +} + +#[tokio::test] +async fn structured_image_input_invalid_file_fails_turn_without_history() { + use crate::llm_client::mock::MockLlmClient; + + let workspace = tempdir().expect("tempdir"); + let attachments = workspace.path().join("attachments"); + fs::create_dir_all(&attachments).expect("mkdir attachments"); + // JPEG bytes staged with a declared `image/png` type. + fs::write( + attachments.join("mismatch.png"), + [0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10], + ) + .expect("write jpeg bytes"); + + let mock = std::sync::Arc::new(MockLlmClient::new(Vec::new())); + let client: crate::core::model_client::SharedModelClient = mock.clone(); + let (engine, handle) = Engine::new_with_model_client( + deterministic_engine_config(workspace.path()), + &Config::default(), + client, + ); + let task = tokio::spawn(engine.run()); + + let cases: Vec<(Vec, &str)> = vec![ + ( + vec![UserInputBlock::LocalImage { + relative_path: PathBuf::from("attachments/gone.png"), + mime_type: "image/png".to_string(), + display_name: "gone.png".to_string(), + }], + "原图片文件已不存在", + ), + ( + vec![UserInputBlock::LocalImage { + relative_path: PathBuf::from("attachments/mismatch.png"), + mime_type: "image/png".to_string(), + display_name: "mismatch.png".to_string(), + }], + "不符", + ), + ]; + for (blocks, expected_hint) in cases { + let mut op = external_user_message_op("", AppMode::Agent, &Config::default()); + with_structured_input(&mut op, blocks); + handle.send(op).await.expect("send invalid image turn"); + + let mut rx = handle.rx_event.write().await; + while let Some(event) = tokio::time::timeout(model_turn_event_timeout(), rx.recv()) + .await + .expect("timed out waiting for failed image turn") + { + if let Event::TurnComplete { status, error, .. } = event { + assert_eq!(status, TurnOutcomeStatus::Failed); + let error = error.expect("failed turn carries an error"); + assert!( + error.contains(expected_hint), + "error must contain '{expected_hint}'; got {error}" + ); + break; + } + } + drop(rx); + } + + // Nothing reached the provider, and no broken message entered history. + assert_eq!(mock.call_count(), 0); + let (tx, snapshot_rx) = tokio::sync::oneshot::channel(); + handle + .send(Op::GetSessionSnapshot { + tx: std::sync::Arc::new(std::sync::Mutex::new(Some(tx))), + }) + .await + .expect("snapshot request"); + let snapshot = tokio::time::timeout(Duration::from_secs(2), snapshot_rx) + .await + .expect("snapshot response") + .expect("snapshot"); + assert!( + snapshot.messages.is_empty(), + "invalid image turns must not pollute session history: {:?}", + snapshot.messages + ); + + handle.send(Op::Shutdown).await.expect("shutdown engine"); + task.await.expect("engine task"); +} + #[tokio::test] async fn injected_model_receives_malformed_tool_feedback_and_recovers() { use crate::llm_client::mock::{MockLlmClient, canned}; @@ -3859,6 +4168,7 @@ async fn operate_model_shell_uses_normal_approval_and_workspace_sandbox() { handle .send(Op::SendMessage { content: "write the requested local fixture".to_string(), + input: None, mode: AppMode::Operate, route: resolved_route_for_test(&api_config, crate::config::DEFAULT_TEXT_MODEL), compaction: Box::new(CompactionConfig::default()), @@ -4003,6 +4313,7 @@ async fn yolo_mode_does_not_prompt_for_model_driven_typed_ask_rule() { handle .send(Op::SendMessage { content: "please exercise the shell path".to_string(), + input: None, mode: AppMode::Yolo, route: resolved_route_for_test(&api_config, crate::config::DEFAULT_TEXT_MODEL), compaction: Box::new(CompactionConfig::default()), @@ -4129,6 +4440,7 @@ async fn yolo_mode_still_prompts_for_background_destructive_shell() { handle .send(Op::SendMessage { content: "please run a background shell".to_string(), + input: None, mode: AppMode::Yolo, route: resolved_route_for_test(&api_config, crate::config::DEFAULT_TEXT_MODEL), compaction: Box::new(CompactionConfig::default()), @@ -4283,6 +4595,7 @@ async fn yolo_mode_does_not_prompt_for_background_shell() { handle .send(Op::SendMessage { content: "please run a background shell".to_string(), + input: None, mode: AppMode::Yolo, route: resolved_route_for_test(&api_config, crate::config::DEFAULT_TEXT_MODEL), compaction: Box::new(CompactionConfig::default()), @@ -4417,6 +4730,7 @@ async fn yolo_mode_prompts_for_publish_like_shell_safety_floor() { handle .send(Op::SendMessage { content: "please publish this crate".to_string(), + input: None, mode: AppMode::Yolo, route: resolved_route_for_test(&api_config, crate::config::DEFAULT_TEXT_MODEL), compaction: Box::new(CompactionConfig::default()), @@ -4569,6 +4883,7 @@ async fn yolo_mode_does_not_prompt_for_mcp_action() { handle .send(Op::SendMessage { content: "please open the PR".to_string(), + input: None, mode: AppMode::Yolo, route: resolved_route_for_test(&api_config, crate::config::DEFAULT_TEXT_MODEL), compaction: Box::new(CompactionConfig::default()), diff --git a/crates/tui/src/core/engine/turn_loop.rs b/crates/tui/src/core/engine/turn_loop.rs index a1221402fb..5d8b452e64 100644 --- a/crates/tui/src/core/engine/turn_loop.rs +++ b/crates/tui/src/core/engine/turn_loop.rs @@ -605,9 +605,33 @@ impl Engine { } } + // Materialize workspace-local image references into inline + // `data:` URL blocks on this request's private clone of the + // message list. The session (and anything persisted or recorded) + // keeps the lightweight `LocalImage` reference; Base64 only ever + // lives on this short-lived request copy. Re-materializing on + // every request build is what makes retries and resumed sessions + // re-read the file and fail clearly when it is gone — never + // silently drop an image. + let mut request_messages = self.messages_with_turn_metadata(); + if let Err(err) = crate::vision::image_input::materialize_messages_local_images( + &mut request_messages, + &self.session.workspace, + ) + .await + { + let message = err.to_string(); + turn_error = Some(message.clone()); + let _ = self + .tx_event + .send(Event::error(ErrorEnvelope::classify(message, true))) + .await; + return (TurnOutcomeStatus::Failed, turn_error); + } + let request = MessageRequest { model: self.session.model.clone(), - messages: self.messages_with_turn_metadata(), + messages: request_messages, max_tokens: effective_max_output_tokens_for_route( self.api_provider, &self.session.model, diff --git a/crates/tui/src/core/ops.rs b/crates/tui/src/core/ops.rs index 8d01a831a6..08c04e58e4 100644 --- a/crates/tui/src/core/ops.rs +++ b/crates/tui/src/core/ops.rs @@ -62,6 +62,36 @@ pub enum UserInputProvenance { AssistantGenerated, } +/// Structured user message input: ordered text and local-image blocks. +/// +/// Pure-text callers keep using only `Op::SendMessage::content`; hosts that +/// stage image attachments into the session workspace additionally pass this +/// structure so the engine can persist image *references* (never Base64) and +/// materialize them only while building a provider request. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UserMessageInput { + pub blocks: Vec, +} + +/// One block of structured user input. +#[allow(dead_code)] // Variants are constructed by external hosts (e.g. pinvou3-app) landing after the engine gate. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum UserInputBlock { + /// Plain user text. + Text { + text: String, + }, + /// An image staged inside the session workspace. `relative_path` must be + /// a workspace-relative path; the engine re-validates it before every + /// request and refuses escapes. `mime_type` is the host-declared type and + /// is verified against the real file bytes before sending. + LocalImage { + relative_path: PathBuf, + mime_type: String, + display_name: String, + }, +} + impl UserInputProvenance { pub fn as_str(self) -> &'static str { match self { @@ -85,6 +115,14 @@ pub enum Op { /// Send a message to the AI SendMessage { content: String, + /// Structured input carrying local-image references alongside text. + /// `None` keeps the legacy pure-text behavior: the message is built + /// from `content` alone. When `Some`, the engine builds the user + /// message from these blocks (persisting workspace-relative image + /// references, never Base64) while `content` remains the plain-text + /// rendition used for policy checks, snapshots, and transcript + /// summaries. + input: Option, mode: AppMode, /// Exact, structurally resolved route authority for this turn. The /// engine activates its client before mutating turn state; injected diff --git a/crates/tui/src/main.rs b/crates/tui/src/main.rs index 48bd894c2d..68d079ae2f 100644 --- a/crates/tui/src/main.rs +++ b/crates/tui/src/main.rs @@ -8515,7 +8515,7 @@ fn exec_stream_input_analysis( &mut analysis.tool_result_estimated_tokens, ); } - ContentBlock::ImageUrl { .. } => {} + ContentBlock::ImageUrl { .. } | ContentBlock::LocalImage { .. } => {} } } } @@ -8923,6 +8923,7 @@ async fn run_exec_agent( engine_handle .send(Op::SendMessage { content: prompt.to_string(), + input: None, mode, route: Box::new(validated_route.into_resolved()), compaction: Box::new(compaction.clone()), diff --git a/crates/tui/src/models.rs b/crates/tui/src/models.rs index 092fc5db9e..e7e064bd88 100644 --- a/crates/tui/src/models.rs +++ b/crates/tui/src/models.rs @@ -91,6 +91,22 @@ pub enum ContentBlock { }, #[serde(rename = "image_url")] ImageUrl { image_url: ImageUrlContent }, + /// Reference to an image file stored inside the session workspace. + /// + /// This is the only image representation allowed in persisted session + /// state: it records the workspace-relative path, declared MIME type, + /// display name, and byte size — never Base64. Before a provider request + /// is built, the engine clones the message list and materializes this + /// block into [`ContentBlock::ImageUrl`] with a `data:` URL (see + /// `vision::image_input`); provider clients therefore never see it and + /// treat it as having no wire equivalent. + #[serde(rename = "local_image")] + LocalImage { + relative_path: std::path::PathBuf, + mime_type: String, + display_name: String, + byte_size: u64, + }, #[serde(rename = "thinking")] Thinking { thinking: String, diff --git a/crates/tui/src/purge.rs b/crates/tui/src/purge.rs index 26077c7813..164461b632 100644 --- a/crates/tui/src/purge.rs +++ b/crates/tui/src/purge.rs @@ -249,6 +249,18 @@ fn format_content_block(buf: &mut String, blk_idx: usize, block: &ContentBlock) ); } ContentBlock::ImageUrl { .. } => {} + ContentBlock::LocalImage { + mime_type, + byte_size, + .. + } => { + // Inspection surfaces show metadata only — never URLs or Base64. + let _ = writeln!( + buf, + " [{blk_idx}] {}", + crate::vision::image_input::image_placeholder(mime_type, *byte_size) + ); + } } } diff --git a/crates/tui/src/rlm/session.rs b/crates/tui/src/rlm/session.rs index 2430b89522..bec676e214 100644 --- a/crates/tui/src/rlm/session.rs +++ b/crates/tui/src/rlm/session.rs @@ -407,6 +407,18 @@ fn compact_content_block(block: &ContentBlock) -> Value { "content": content, }), ContentBlock::ImageUrl { .. } => serde_json::Value::Null, + ContentBlock::LocalImage { + relative_path, + mime_type, + display_name, + byte_size, + } => json!({ + "type": "local_image", + "relative_path": relative_path, + "mime_type": mime_type, + "display_name": display_name, + "byte_size": byte_size, + }), } } diff --git a/crates/tui/src/runtime_api/sessions.rs b/crates/tui/src/runtime_api/sessions.rs index 3f7c4a83b2..7b7ae540b3 100644 --- a/crates/tui/src/runtime_api/sessions.rs +++ b/crates/tui/src/runtime_api/sessions.rs @@ -699,7 +699,33 @@ pub(super) fn session_to_detail(session: SavedSession) -> SessionDetailResponse } => { json!({ "type": "tool_result", "tool_use_id": tool_use_id, "content": content }) } - crate::models::ContentBlock::ImageUrl { .. } => Value::Null, + crate::models::ContentBlock::ImageUrl { image_url } => { + // Never echo the image payload: expose only metadata. + match crate::vision::image_input::split_data_url(&image_url.url) { + Some((mime_type, data)) => json!({ + "type": "image_url", + "mime_type": mime_type, + "byte_size": (data.len() / 4) * 3, + }), + None => json!({ + "type": "image_url", + "mime_type": Value::Null, + "byte_size": Value::Null, + }), + } + } + crate::models::ContentBlock::LocalImage { + relative_path, + mime_type, + display_name, + byte_size, + } => json!({ + "type": "local_image", + "relative_path": relative_path, + "mime_type": mime_type, + "display_name": display_name, + "byte_size": byte_size, + }), }) .collect(); json!({ diff --git a/crates/tui/src/runtime_threads.rs b/crates/tui/src/runtime_threads.rs index 64d4f9ed60..07edac4202 100644 --- a/crates/tui/src/runtime_threads.rs +++ b/crates/tui/src/runtime_threads.rs @@ -2877,6 +2877,7 @@ impl RuntimeThreadManager { let auto_approve = req.auto_approve.unwrap_or(thread.auto_approve); let op = Op::SendMessage { content: prompt, + input: None, mode, route: Box::new(route), compaction: Box::new(compaction), diff --git a/crates/tui/src/seam_manager.rs b/crates/tui/src/seam_manager.rs index 741d66f657..29a95d969d 100644 --- a/crates/tui/src/seam_manager.rs +++ b/crates/tui/src/seam_manager.rs @@ -391,7 +391,8 @@ impl SeamManager { ContentBlock::ServerToolUse { .. } | ContentBlock::ToolSearchToolResult { .. } | ContentBlock::CodeExecutionToolResult { .. } - | ContentBlock::ImageUrl { .. } => {} + | ContentBlock::ImageUrl { .. } + | ContentBlock::LocalImage { .. } => {} } } } diff --git a/crates/tui/src/session_manager.rs b/crates/tui/src/session_manager.rs index 6f3749e309..94c9a1a35b 100644 --- a/crates/tui/src/session_manager.rs +++ b/crates/tui/src/session_manager.rs @@ -1229,8 +1229,7 @@ mod tests { forked_from_message_count: None, cumulative_turn_secs: 0, }, - system_prompt: None, - context_references: Vec::new(), + system_prompt: None, context_references: Vec::new(), artifacts: Vec::new(), work_state: None, }; @@ -1265,6 +1264,85 @@ mod tests { assert_eq!(loaded.messages.len(), 2); } + #[test] + fn saved_session_with_local_image_persists_reference_without_base64() { + let tmp = tempdir().expect("tempdir"); + let manager = SessionManager::new(tmp.path().join("sessions")).expect("new"); + + let messages = vec![Message { + role: "user".to_string(), + content: vec![ + ContentBlock::Text { + text: "look at this picture".to_string(), + cache_control: None, + }, + ContentBlock::LocalImage { + relative_path: PathBuf::from("attachments/photo.png"), + mime_type: "image/png".to_string(), + display_name: "photo.png".to_string(), + byte_size: 106_138, + }, + ], + }]; + let session = create_saved_session(&messages, "test-model", tmp.path(), 100, None); + let session_id = session.metadata.id.clone(); + + let path = manager.save_session(&session).expect("save"); + let raw = fs::read_to_string(&path).expect("read saved session"); + + // The persisted record keeps the lightweight reference only. + assert!(raw.contains("local_image"), "{raw}"); + assert!(raw.contains("attachments/photo.png"), "{raw}"); + assert!(raw.contains("106138"), "{raw}"); + assert!( + !raw.contains("data:image"), + "persisted session must never contain a data URL: {raw}" + ); + // Defense in depth: no long Base64 run anywhere in the file. + let mut run = 0usize; + for ch in raw.chars() { + if ch.is_ascii_alphanumeric() || matches!(ch, '+' | '/' | '=') { + run += 1; + assert!(run < 64, "persisted session contains a Base64-looking run: {raw}"); + } else { + run = 0; + } + } + + // Round-trip keeps the reference intact. + let loaded = manager.load_session(&session_id).expect("load"); + let reference = loaded.messages[0] + .content + .iter() + .find_map(|b| match b { + ContentBlock::LocalImage { + relative_path, + mime_type, + display_name, + byte_size, + } => Some(( + relative_path.clone(), + mime_type.clone(), + display_name.clone(), + *byte_size, + )), + _ => None, + }) + .expect("local image reference survives round-trip"); + assert_eq!(reference.0, PathBuf::from("attachments/photo.png")); + assert_eq!(reference.1, "image/png"); + assert_eq!(reference.2, "photo.png"); + assert_eq!(reference.3, 106_138); + } + + #[test] + fn forkguard_saved_session_with_local_image_persists_reference_without_base64() { + // [pinvou3-fork] forkguard 锚点:含本地图片的会话落盘只存轻量引用,绝无 + // data URL / Base64 泄漏,且往返加载引用完好。断言本体在上方同名测试, + // 此包装使其纳入 `cargo test forkguard_` 守护面。 + saved_session_with_local_image_persists_reference_without_base64(); + } + #[test] fn save_and_load_session_preserves_rich_update_plan_tool_payload() { let tmp = tempdir().expect("tempdir"); diff --git a/crates/tui/src/tui/mouse_ui.rs b/crates/tui/src/tui/mouse_ui.rs index 4d8b6254ca..f4e22abb86 100644 --- a/crates/tui/src/tui/mouse_ui.rs +++ b/crates/tui/src/tui/mouse_ui.rs @@ -930,7 +930,22 @@ fn agent_message_text(message: &Message) -> String { text.push_str(&format!("{label} ({tool_use_id})\n{content}\n")); } ContentBlock::ImageUrl { image_url } => { - text.push_str(&format!("[image: {}]\n", image_url.url)); + // Transcript views render metadata placeholders only — never + // the data URL / Base64 payload. + text.push_str(&format!( + "{}\n", + crate::vision::image_input::data_url_placeholder(&image_url.url) + )); + } + ContentBlock::LocalImage { + mime_type, + byte_size, + .. + } => { + text.push_str(&format!( + "{}\n", + crate::vision::image_input::image_placeholder(mime_type, *byte_size) + )); } // Thinking blocks are deliberately not surfaced in the main TUI // and should not leak through a worker detail view either. diff --git a/crates/tui/src/tui/notifications.rs b/crates/tui/src/tui/notifications.rs index 33fbd5f3d2..ae606cb286 100644 --- a/crates/tui/src/tui/notifications.rs +++ b/crates/tui/src/tui/notifications.rs @@ -760,7 +760,7 @@ pub fn latest_assistant_text(messages: &[Message]) -> Option { | ContentBlock::ServerToolUse { .. } | ContentBlock::ToolSearchToolResult { .. } | ContentBlock::CodeExecutionToolResult { .. } => None, - ContentBlock::ImageUrl { .. } => None, + ContentBlock::ImageUrl { .. } | ContentBlock::LocalImage { .. } => None, }) .collect::>() .join("\n"); diff --git a/crates/tui/src/tui/session_picker.rs b/crates/tui/src/tui/session_picker.rs index b832dc8850..379562f590 100644 --- a/crates/tui/src/tui/session_picker.rs +++ b/crates/tui/src/tui/session_picker.rs @@ -938,6 +938,9 @@ fn message_text_for_history(message: &crate::models::Message) -> String { format!("tool result: {}", truncate(&content.to_string(), 220)) } crate::models::ContentBlock::ImageUrl { .. } => String::from("[image]"), + crate::models::ContentBlock::LocalImage { display_name, .. } => { + format!("[image: {display_name}]") + } }; let part = part.trim(); if !part.is_empty() { diff --git a/crates/tui/src/tui/ui.rs b/crates/tui/src/tui/ui.rs index 54fc77d310..ee925acb36 100644 --- a/crates/tui/src/tui/ui.rs +++ b/crates/tui/src/tui/ui.rs @@ -8021,6 +8021,7 @@ async fn dispatch_user_message( engine_handle .send(Op::SendMessage { content, + input: None, mode: app.mode, route: Box::new(turn_route), compaction: Box::new(turn_compaction.clone()), diff --git a/crates/tui/src/utils.rs b/crates/tui/src/utils.rs index 6a677826e0..161606b441 100644 --- a/crates/tui/src/utils.rs +++ b/crates/tui/src/utils.rs @@ -623,7 +623,8 @@ pub fn estimate_message_chars(messages: &[Message]) -> usize { ContentBlock::ServerToolUse { .. } | ContentBlock::ToolSearchToolResult { .. } | ContentBlock::CodeExecutionToolResult { .. } - | ContentBlock::ImageUrl { .. } => {} + | ContentBlock::ImageUrl { .. } + | ContentBlock::LocalImage { .. } => {} } } } diff --git a/crates/tui/src/vision/image_input.rs b/crates/tui/src/vision/image_input.rs new file mode 100644 index 0000000000..195c8c49f7 --- /dev/null +++ b/crates/tui/src/vision/image_input.rs @@ -0,0 +1,437 @@ +//! Shared workspace-image validation and request-time materialization. +//! +//! Sessions persist images as [`crate::models::ContentBlock::LocalImage`] +//! references (workspace-relative path + MIME + display name + byte size) — +//! never Base64. Right before a provider request is built, the engine clones +//! the message list and calls [`materialize_messages_local_images`] to turn +//! those references into `data:` URL image blocks. The materialized clone is +//! short-lived and must never be persisted, logged, or fed back into the op +//! mailbox. +//! +//! The `image_analyze` tool (`vision::tools`) reuses the same path-boundary +//! and file-validation helpers so both entry points enforce identical rules. + +use std::path::{Component, Path, PathBuf}; + +use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; + +use crate::models::{ContentBlock, ImageUrlContent, Message}; + +/// Unified upper bound for one image, checked before any Base64 encoding. +/// Aligned with the application-side attachment limit (`MAX_FILE_BYTES`). +pub const MAX_IMAGE_BYTES: u64 = 20 * 1024 * 1024; + +/// Number of leading bytes needed to sniff every supported signature +/// (WEBP needs 12: `RIFF` + size + `WEBP`). +const SNIFF_HEADER_BYTES: usize = 16; + +/// User-facing validation/materialization failure for local image input. +/// +/// Messages are bilingual (Chinese first) so hosts can surface them verbatim; +/// they never contain Base64 or full request bodies. +#[derive(Debug)] +pub struct ImageInputError { + message: String, +} + +impl ImageInputError { + fn new(message: impl Into) -> Self { + Self { + message: message.into(), + } + } +} + +impl std::fmt::Display for ImageInputError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for ImageInputError {} + +/// Resolve a workspace-relative image path, rejecting absolute paths, `..` +/// traversal, and symlink escapes. Shared by native image input and the +/// `image_analyze` tool. +pub fn resolve_workspace_image_path( + workspace: &Path, + relative_path: &Path, +) -> Result { + if relative_path.components().any(|c| { + matches!( + c, + Component::Prefix(_) | Component::RootDir | Component::ParentDir + ) + }) { + return Err(ImageInputError::new( + "图片路径必须是工作区内的相对路径，不能越界 \ + (image path must be a relative path within the workspace and cannot escape it)", + )); + } + + let workspace = workspace.canonicalize().map_err(|e| { + ImageInputError::new(format!( + "无法解析工作区路径 (failed to resolve workspace path): {e}" + )) + })?; + let candidate = workspace.join(relative_path); + let resolved = candidate.canonicalize().map_err(|e| { + if e.kind() == std::io::ErrorKind::NotFound { + ImageInputError::new(format!( + "原图片文件已不存在，无法发送 (the original image file no longer exists): {}", + relative_path.display() + )) + } else { + ImageInputError::new(format!( + "无法解析图片文件 (failed to resolve image file): {e}" + )) + } + })?; + if !resolved.starts_with(&workspace) { + return Err(ImageInputError::new( + "图片路径越界：解析后的位置不在工作区内 \ + (image path must resolve within the workspace and cannot escape it)", + )); + } + Ok(resolved) +} + +/// Detect the real image MIME type from magic bytes. Returns `None` when the +/// header does not match any supported format (PNG/JPEG/GIF/WebP/BMP). +#[must_use] +pub fn sniff_image_mime(header: &[u8]) -> Option<&'static str> { + if header.starts_with(b"\x89PNG\r\n\x1a\n") { + return Some("image/png"); + } + if header.starts_with(b"\xff\xd8\xff") { + return Some("image/jpeg"); + } + if header.starts_with(b"GIF87a") || header.starts_with(b"GIF89a") { + return Some("image/gif"); + } + if header.len() >= 12 && header.starts_with(b"RIFF") && &header[8..12] == b"WEBP" { + return Some("image/webp"); + } + if header.starts_with(b"BM") { + return Some("image/bmp"); + } + None +} + +/// Normalize a declared MIME type for comparison (lowercase, trimmed, common +/// alias folding). +fn normalize_mime(mime: &str) -> String { + let lowered = mime.trim().to_ascii_lowercase(); + match lowered.as_str() { + "image/jpg" | "image/pjpeg" => "image/jpeg".to_string(), + other => other.to_string(), + } +} + +fn format_size_limit() -> String { + format!("{} MB", MAX_IMAGE_BYTES / (1024 * 1024)) +} + +fn check_image_bytes(declared_mime: &str, bytes: &[u8]) -> Result<(), ImageInputError> { + let sniffed = sniff_image_mime(bytes).ok_or_else(|| { + ImageInputError::new( + "无法识别的图片格式：文件头不是支持的图片类型，已拒绝发送 \ + (unsupported image format: unrecognized file signature)", + ) + })?; + if normalize_mime(declared_mime) != sniffed { + return Err(ImageInputError::new(format!( + "图片真实格式（{sniffed}）与声明类型（{declared_mime}）不符，已拒绝发送 \ + (declared MIME type does not match the real file signature)" + ))); + } + Ok(()) +} + +/// Read an already-resolved image file with the size cap enforced both before +/// and after the read, detecting the real MIME type from magic bytes. +pub async fn read_image_sniffed( + resolved_path: &Path, +) -> Result<(Vec, &'static str), ImageInputError> { + let metadata = tokio::fs::metadata(resolved_path).await.map_err(|e| { + if e.kind() == std::io::ErrorKind::NotFound { + ImageInputError::new(format!( + "原图片文件已不存在，无法发送 (the original image file no longer exists): {}", + resolved_path.display() + )) + } else { + ImageInputError::new(format!( + "无法读取图片文件信息 (failed to stat image file): {e}" + )) + } + })?; + if metadata.len() > MAX_IMAGE_BYTES { + return Err(ImageInputError::new(format!( + "图片超过大小上限：{} 字节，单张图片最大允许 {} (image exceeds the maximum allowed size)", + metadata.len(), + format_size_limit(), + ))); + } + + let bytes = tokio::fs::read(resolved_path).await.map_err(|e| { + ImageInputError::new(format!( + "无法读取图片文件 (failed to read image file): {e}" + )) + })?; + // Re-check after the read: the file may have grown between stat and read. + if bytes.len() as u64 > MAX_IMAGE_BYTES { + return Err(ImageInputError::new(format!( + "图片超过大小上限：{} 字节，单张图片最大允许 {} (image exceeds the maximum allowed size)", + bytes.len(), + format_size_limit(), + ))); + } + + let sniffed = sniff_image_mime(&bytes).ok_or_else(|| { + ImageInputError::new( + "无法识别的图片格式：文件头不是支持的图片类型，已拒绝发送 \ + (unsupported image format: unrecognized file signature)", + ) + })?; + Ok((bytes, sniffed)) +} + +/// Read an already-resolved image file with the size cap enforced, then +/// verify the real signature against `declared_mime`. Returns the bytes and +/// the canonical sniffed MIME type. +pub async fn read_validated_image( + resolved_path: &Path, + declared_mime: &str, +) -> Result<(Vec, &'static str), ImageInputError> { + let (bytes, sniffed) = read_image_sniffed(resolved_path).await?; + if normalize_mime(declared_mime) != sniffed { + return Err(ImageInputError::new(format!( + "图片真实格式（{sniffed}）与声明类型（{declared_mime}）不符，已拒绝发送 \ + (declared MIME type does not match the real file signature)" + ))); + } + Ok((bytes, sniffed)) +} + +/// Preflight validation for a workspace-local image reference. Resolves the +/// path inside `workspace`, enforces the size cap, and verifies the real file +/// signature against `declared_mime` without keeping the file contents. +/// Returns the actual byte size for persistence metadata. +pub async fn validate_local_image_reference( + workspace: &Path, + relative_path: &Path, + declared_mime: &str, +) -> Result { + let resolved = resolve_workspace_image_path(workspace, relative_path)?; + let metadata = tokio::fs::metadata(&resolved) + .await + .map_err(|e| ImageInputError::new(format!("无法读取图片文件信息 (failed to stat image file): {e}")))?; + if metadata.len() > MAX_IMAGE_BYTES { + return Err(ImageInputError::new(format!( + "图片超过大小上限：{} 字节，单张图片最大允许 {} (image exceeds the maximum allowed size)", + metadata.len(), + format_size_limit(), + ))); + } + + let mut file = tokio::fs::File::open(&resolved) + .await + .map_err(|e| ImageInputError::new(format!("无法读取图片文件 (failed to open image file): {e}")))?; + let mut header = [0u8; SNIFF_HEADER_BYTES]; + let read = { + use tokio::io::AsyncReadExt as _; + file.read(&mut header) + .await + .map_err(|e| ImageInputError::new(format!("无法读取图片文件 (failed to read image file): {e}")))? + }; + check_image_bytes(declared_mime, &header[..read])?; + Ok(metadata.len()) +} + +/// Build the OpenAI-compatible `data:` URL for validated image bytes. +#[must_use] +pub fn data_url(mime_type: &str, bytes: &[u8]) -> String { + format!("data:{mime_type};base64,{}", BASE64.encode(bytes)) +} + +/// Split a `data:;base64,` URL into its parts, borrowing from the +/// input so callers never clone the payload. Returns `None` for any other +/// URL shape (e.g. remote `https:` URLs). +#[must_use] +pub fn split_data_url(url: &str) -> Option<(&str, &str)> { + let rest = url.strip_prefix("data:")?; + let (mime, data) = rest.split_once(";base64,")?; + if mime.is_empty() || data.is_empty() { + return None; + } + Some((mime, data)) +} + +/// Materialize one local image reference into an inline `ImageUrl` block by +/// reading and validating the file fresh from disk. +pub async fn materialize_local_image( + workspace: &Path, + relative_path: &Path, + declared_mime: &str, +) -> Result { + let resolved = resolve_workspace_image_path(workspace, relative_path)?; + // The data URL carries the canonical sniffed type, not the host's + // declaration (which may be an alias like `image/jpg`). + let (bytes, sniffed_mime) = read_validated_image(&resolved, declared_mime).await?; + Ok(ContentBlock::ImageUrl { + image_url: ImageUrlContent { + url: data_url(sniffed_mime, &bytes), + }, + }) +} + +/// Replace every [`ContentBlock::LocalImage`] in a cloned request message +/// list with its materialized inline image. Operates in place; on failure the +/// caller must abort the request — never fall back to dropping the image. +/// +/// Re-runs on every request build, which is what makes retries and resumed +/// sessions re-read the file from its relative path and report a clear error +/// when the original image is gone. +pub async fn materialize_messages_local_images( + messages: &mut [Message], + workspace: &Path, +) -> Result<(), ImageInputError> { + for message in messages.iter_mut() { + if !message + .content + .iter() + .any(|block| matches!(block, ContentBlock::LocalImage { .. })) + { + continue; + } + for block in message.content.iter_mut() { + if let ContentBlock::LocalImage { + relative_path, + mime_type, + display_name, + .. + } = block + { + let materialized = materialize_local_image(workspace, relative_path, mime_type) + .await + .map_err(|e| ImageInputError::new(format!("{e}（图片：{display_name}）")))?; + *block = materialized; + } + } + } + Ok(()) +} + +/// Placeholder for transcript/debug rendering of an image reference, e.g. +/// `[image:image/jpeg,106138 bytes]`. Never includes URLs or Base64. +#[must_use] +pub fn image_placeholder(mime_type: &str, byte_size: u64) -> String { + format!("[image:{mime_type},{byte_size} bytes]") +} + +/// Placeholder for an already-materialized inline image block, deriving MIME +/// and approximate byte size from the data URL without copying the payload. +#[must_use] +pub fn data_url_placeholder(url: &str) -> String { + match split_data_url(url) { + Some((mime, data)) => { + let approx_bytes = (data.len() / 4) * 3; + image_placeholder(mime, approx_bytes as u64) + } + None => "[image:remote-url]".to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const PNG_HEADER: [u8; 8] = [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]; + + #[test] + fn sniff_detects_supported_signatures() { + assert_eq!(sniff_image_mime(&PNG_HEADER), Some("image/png")); + assert_eq!(sniff_image_mime(b"\xff\xd8\xff\xe0rest"), Some("image/jpeg")); + assert_eq!(sniff_image_mime(b"GIF87a...."), Some("image/gif")); + assert_eq!(sniff_image_mime(b"GIF89a...."), Some("image/gif")); + assert_eq!( + sniff_image_mime(b"RIFF\x04\x00\x00\x00WEBPvp8 "), + Some("image/webp") + ); + assert_eq!(sniff_image_mime(b"BM6\x00"), Some("image/bmp")); + } + + #[test] + fn sniff_rejects_unknown_and_truncated_headers() { + assert_eq!(sniff_image_mime(b"plain text"), None); + assert_eq!(sniff_image_mime(b""), None); + // WEBP requires the full 12-byte signature. + assert_eq!(sniff_image_mime(b"RIFF\x04\x00\x00\x00"), None); + assert_eq!(sniff_image_mime(b"RIFF\x04\x00\x00\x00WAVE"), None); + } + + #[test] + fn split_data_url_parses_canonical_shape() { + let (mime, data) = + split_data_url("data:image/png;base64,iVBORw0K").expect("valid data url"); + assert_eq!(mime, "image/png"); + assert_eq!(data, "iVBORw0K"); + assert!(split_data_url("https://example.com/x.png").is_none()); + assert!(split_data_url("data:;base64,AAAA").is_none()); + assert!(split_data_url("data:image/png;base64,").is_none()); + } + + #[test] + fn placeholders_never_contain_payload() { + assert_eq!(image_placeholder("image/jpeg", 106_138), "[image:image/jpeg,106138 bytes]"); + assert_eq!( + data_url_placeholder("data:image/png;base64,AAAAAAAA"), + "[image:image/png,6 bytes]" + ); + assert_eq!( + data_url_placeholder("https://example.com/x.png"), + "[image:remote-url]" + ); + } + + #[tokio::test] + async fn materialize_uses_canonical_sniffed_mime_for_aliases() { + let tmp = tempfile::tempdir().expect("tempdir"); + let attachments = tmp.path().join("attachments"); + std::fs::create_dir_all(&attachments).expect("mkdir"); + std::fs::write(attachments.join("photo.jpg"), [0xFF, 0xD8, 0xFF, 0xE0, 0x01]) + .expect("write jpeg bytes"); + + let block = materialize_local_image( + tmp.path(), + Path::new("attachments/photo.jpg"), + "image/jpg", // alias declaration + ) + .await + .expect("alias declaration matches sniffed jpeg"); + + let ContentBlock::ImageUrl { image_url } = block else { + panic!("expected materialized ImageUrl block"); + }; + assert!( + image_url.url.starts_with("data:image/jpeg;base64,"), + "canonical sniffed MIME must win over the alias; got {}", + image_url.url + ); + } + + #[tokio::test] + async fn materialize_rejects_workspace_escape_and_missing_file() { + let tmp = tempfile::tempdir().expect("tempdir"); + + let err = materialize_local_image(tmp.path(), Path::new("../escape.png"), "image/png") + .await + .expect_err("parent-dir escape must reject"); + assert!(err.to_string().contains("越界"), "{err}"); + + let err = materialize_local_image(tmp.path(), Path::new("gone.png"), "image/png") + .await + .expect_err("missing file must reject"); + assert!(err.to_string().contains("已不存在"), "{err}"); + } +} diff --git a/crates/tui/src/vision/mod.rs b/crates/tui/src/vision/mod.rs index a7d4bba149..7d8082a661 100644 --- a/crates/tui/src/vision/mod.rs +++ b/crates/tui/src/vision/mod.rs @@ -3,4 +3,5 @@ //! Provides the `image_analyze` tool that sends images to an //! OpenAI-compatible vision model API and returns text descriptions. +pub mod image_input; pub mod tools; diff --git a/crates/tui/src/vision/tools.rs b/crates/tui/src/vision/tools.rs index de06240c21..a8a2748f91 100644 --- a/crates/tui/src/vision/tools.rs +++ b/crates/tui/src/vision/tools.rs @@ -1,6 +1,6 @@ //! `image_analyze` tool — analyze images using a dedicated vision model. -use std::path::{Component, Path, PathBuf}; +use std::path::Path; use std::time::Duration; use async_trait::async_trait; @@ -12,6 +12,7 @@ use crate::llm_client::{LlmError, RetryConfig, sanitize_http_error_body, with_re use crate::tools::spec::{ ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec, required_str, }; +use crate::vision::image_input::{read_image_sniffed, resolve_workspace_image_path}; const DEFAULT_VISION_MAX_OUTPUT_TOKENS: u32 = 4096; @@ -31,60 +32,13 @@ impl ImageAnalyzeTool { } async fn read_image_file(path: &Path) -> Result<(String, String), ToolError> { - let bytes = tokio::fs::read(path) + // Size cap and real-signature detection are enforced inside the + // shared helper — the file extension is never trusted. + let (bytes, mime_type) = read_image_sniffed(path) .await - .map_err(|e| ToolError::execution_failed(format!("Failed to read image file: {e}")))?; - - let mime_type = Self::detect_mime_type(path)?; + .map_err(|e| ToolError::execution_failed(e.to_string()))?; let base64_data = BASE64.encode(&bytes); - Ok((base64_data, mime_type)) - } - - fn resolve_image_path(workspace: &Path, image_path: &str) -> Result { - let image_path_buf = Path::new(image_path); - if image_path_buf.components().any(|c| { - matches!( - c, - Component::Prefix(_) | Component::RootDir | Component::ParentDir - ) - }) { - return Err(ToolError::execution_failed( - "image_path must be a relative path within the workspace and cannot escape it.", - )); - } - - let workspace = workspace.canonicalize().map_err(|e| { - ToolError::execution_failed(format!("Failed to resolve workspace path: {e}")) - })?; - let candidate = workspace.join(image_path_buf); - let resolved = candidate.canonicalize().map_err(|e| { - ToolError::execution_failed(format!("Failed to resolve image file: {e}")) - })?; - if !resolved.starts_with(&workspace) { - return Err(ToolError::execution_failed( - "image_path must resolve within the workspace and cannot escape it.", - )); - } - Ok(resolved) - } - - fn detect_mime_type(path: &Path) -> Result { - let extension = path - .extension() - .and_then(|e| e.to_str()) - .unwrap_or("") - .to_lowercase(); - - match extension.as_str() { - "png" => Ok("image/png".to_string()), - "jpg" | "jpeg" => Ok("image/jpeg".to_string()), - "gif" => Ok("image/gif".to_string()), - "webp" => Ok("image/webp".to_string()), - "bmp" => Ok("image/bmp".to_string()), - _ => Err(ToolError::execution_failed(format!( - "Unsupported image format: {extension}" - ))), - } + Ok((base64_data, mime_type.to_string())) } fn base_url(&self) -> String { @@ -122,6 +76,10 @@ impl ImageAnalyzeTool { } fn request_payload(&self, prompt: &str, image_data: &str, mime_type: &str) -> Value { + // 不带 temperature:部分模型(kimi-for-coding、gpt-5 系推理模型等)只接受 + // 默认值,显式 temperature 会被 400 拒绝("only 1 is allowed");省略时 + // provider 应用各自默认值,对所有模型都合法。模型目录 supported_parameters + // 也无一声明 temperature。 let mut payload = json!({ "model": self.config.model, "messages": [ @@ -137,8 +95,7 @@ impl ImageAnalyzeTool { } ] } - ], - "temperature": 0.7 + ] }); let token_limit_field = if Self::uses_max_completion_tokens(&self.config) { @@ -159,8 +116,12 @@ impl ToolSpec for ImageAnalyzeTool { } fn description(&self) -> &str { - "Analyze an image using the configured vision model. \ - Supports PNG, JPEG, GIF, WebP, and BMP formats." + "Analyze an image in the session workspace using the configured vision model. \ + Supports PNG, JPEG, GIF, WebP, and BMP formats. \ + Use this only for workspace images that were NOT provided as native image blocks \ + of the current user message; images already attached to the current message are \ + visible to the model directly and must not be re-analyzed. \ + 仅用于读取 workspace 中未作为当前消息原生图片块提供的图片；已随当前消息提供的图片不需要重复调用。" } fn input_schema(&self) -> Value { @@ -191,7 +152,8 @@ impl ToolSpec for ImageAnalyzeTool { .and_then(|v| v.as_str()) .unwrap_or("Describe this image in detail."); - let resolved_path = Self::resolve_image_path(&context.workspace, image_path)?; + let resolved_path = resolve_workspace_image_path(&context.workspace, Path::new(image_path)) + .map_err(|e| ToolError::execution_failed(e.to_string()))?; let (image_data, mime_type) = Self::read_image_file(&resolved_path).await?; let payload = self.request_payload(prompt, &image_data, &mime_type); @@ -311,30 +273,51 @@ mod tests { assert!(tool.capabilities().contains(&ToolCapability::ReadOnly)); } - #[test] - fn mime_type_detection_covers_common_formats() { - for (ext, expected) in [ - ("png", "image/png"), - ("PNG", "image/png"), - ("jpg", "image/jpeg"), - ("jpeg", "image/jpeg"), - ("gif", "image/gif"), - ("webp", "image/webp"), - ("bmp", "image/bmp"), - ] { - let path = std::path::PathBuf::from(format!("test.{ext}")); - let mime = ImageAnalyzeTool::detect_mime_type(&path) - .unwrap_or_else(|_| panic!("must detect {ext}")); - assert_eq!(mime, expected); - } + #[tokio::test] + async fn read_image_file_sniffs_real_signature_regardless_of_extension() { + // A PNG file named `.jpg` must be detected as image/png — the real + // file header wins over the extension. + let tmp = tempdir().expect("tempdir"); + let path = tmp.path().join("mislabeled.jpg"); + std::fs::write(&path, [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00]) + .expect("write png bytes"); + + let (_data, mime) = ImageAnalyzeTool::read_image_file(&path) + .await + .expect("png signature detected"); + assert_eq!(mime, "image/png"); } - #[test] - fn mime_type_detection_rejects_unsupported_extension() { - let path = std::path::PathBuf::from("test.svg"); - let err = ImageAnalyzeTool::detect_mime_type(&path) - .expect_err("svg is intentionally out of scope for vision tool"); - assert!(err.to_string().contains("Unsupported image format")); + #[tokio::test] + async fn read_image_file_rejects_forged_extension() { + let tmp = tempdir().expect("tempdir"); + let path = tmp.path().join("fake.png"); + std::fs::write(&path, b"this is not a real image").expect("write garbage"); + + let err = ImageAnalyzeTool::read_image_file(&path) + .await + .expect_err("forged extension must reject"); + assert!( + err.to_string().contains("unsupported image format"), + "error must call out the unrecognized signature; got {err}" + ); + } + + #[tokio::test] + async fn read_image_file_enforces_size_cap_before_reading() { + let tmp = tempdir().expect("tempdir"); + let path = tmp.path().join("huge.png"); + let file = std::fs::File::create(&path).expect("create"); + file.set_len(crate::vision::image_input::MAX_IMAGE_BYTES + 1) + .expect("set len"); + + let err = ImageAnalyzeTool::read_image_file(&path) + .await + .expect_err("oversized image must reject before reading"); + assert!( + err.to_string().contains("大小上限"), + "error must call out the size limit; got {err}" + ); } #[test] @@ -382,6 +365,24 @@ mod tests { assert!(payload.get("max_tokens").is_none()); } + /// pinvou3 fork 行为锚点:payload 绝不携带 temperature——kimi-for-coding 等模型 + /// 只接受默认值,显式 temperature 会被 400 拒绝。 + #[test] + fn forkguard_vision_payload_omits_temperature() { + for model in ["deepseek-v4-pro", "kimi-for-coding", "gpt-5.6-terra", "mimo-v2.5"] { + let mut config = fake_config(); + config.model = model.to_string(); + let tool = ImageAnalyzeTool::new(config); + + let payload = tool.request_payload("describe", "abc123", "image/png"); + + assert!( + payload.get("temperature").is_none(), + "temperature must be omitted for {model}; got {payload}" + ); + } + } + #[tokio::test] async fn execute_rejects_absolute_path() { // Trust-boundary pin: image_path must stay inside the workspace diff --git a/crates/tui/src/working_set.rs b/crates/tui/src/working_set.rs index b5d05faae1..0604ef1256 100644 --- a/crates/tui/src/working_set.rs +++ b/crates/tui/src/working_set.rs @@ -1442,7 +1442,8 @@ fn extract_paths_from_message(message: &Message) -> Vec { | ContentBlock::ServerToolUse { .. } | ContentBlock::ToolSearchToolResult { .. } | ContentBlock::CodeExecutionToolResult { .. } - | ContentBlock::ImageUrl { .. } => {} + | ContentBlock::ImageUrl { .. } + | ContentBlock::LocalImage { .. } => {} } } paths @@ -1626,7 +1627,8 @@ fn message_mentions_any_path(message: &Message, needles: &[String], max_scan_cha | ContentBlock::ServerToolUse { .. } | ContentBlock::ToolSearchToolResult { .. } | ContentBlock::CodeExecutionToolResult { .. } - | ContentBlock::ImageUrl { .. } => {} + | ContentBlock::ImageUrl { .. } + | ContentBlock::LocalImage { .. } => {} } } false