diff --git a/Cargo.toml b/Cargo.toml index 31758c6..f6d24ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vibekeys" -version = "0.1.3" +version = "0.2.0" authors = ["csh <458761603@qq.com>"] edition = "2021" resolver = "2" @@ -88,3 +88,7 @@ embuild = "0.33" remote_component = { name = "espressif/esp-sr", version = "^2.0.0" } bindings_header = "components/esp_sr/bindgen.h" bindings_module = "esp_sr" + +[[package.metadata.esp-idf-sys.extra_components]] +remote_component = { name = "espressif/esp_new_jpeg", version = "^1.0.0" } +bindings_header = "components/esp_new_jpeg/bindgen.h" diff --git a/components/esp_new_jpeg/bindgen.h b/components/esp_new_jpeg/bindgen.h new file mode 100644 index 0000000..3708d69 --- /dev/null +++ b/components/esp_new_jpeg/bindgen.h @@ -0,0 +1,2 @@ +#include "esp_jpeg_common.h" +#include "esp_jpeg_dec.h" \ No newline at end of file diff --git a/components_esp32s3.lock b/components_esp32s3.lock index 9b692d5..d57cc0e 100644 --- a/components_esp32s3.lock +++ b/components_esp32s3.lock @@ -37,12 +37,29 @@ dependencies: registry_url: https://components.espressif.com/ type: service version: 2.3.1 + espressif/esp_new_jpeg: + component_hash: 2f06f88e94fbc781b26a9be4291ff4ffc88596c90436a1e1358a7a38296b44ae + dependencies: [] + source: + registry_url: https://components.espressif.com/ + type: service + targets: + - esp32 + - esp32s2 + - esp32s3 + - esp32p4 + - esp32c2 + - esp32c3 + - esp32c5 + - esp32c6 + version: 1.0.0 idf: source: type: idf version: 5.4.1 direct_dependencies: - espressif/esp-sr -manifest_hash: 02fa0b8da76d97234f1906120eca12c93f5d7a9e2bb8fef29c38a5e8ce1fa669 +- espressif/esp_new_jpeg +manifest_hash: 1faf3ba12053ac8c70fd3574b63896030ba5489133e6c91e37e6c1339f556bc9 target: esp32s3 version: 2.0.0 diff --git a/src/app.rs b/src/app.rs index 0cc93de..74a22a4 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,6 +1,7 @@ use embedded_graphics::prelude::WebColors; use crate::{ + bt_keyboard_mode::{self, KeymapConfig}, lcd::{self, ColorFormat}, protocol::{self}, }; @@ -62,6 +63,7 @@ pub async fn run( uri: String, ui: &mut crate::lcd::UI, mut rx: crate::audio::EventRx, + keymaps: &KeymapConfig, ) -> anyhow::Result<()> { let server = crate::ws::Server::new(uri).await; if server.is_err() { @@ -74,68 +76,140 @@ pub async fn run( let mut server = server.unwrap(); let mut start_submit_audio = false; - ui.show_notification( - ColorFormat::CSS_DARK_GREEN, - "Server Connected\nPress Voice Key to start talking", - )?; - ui.start_input("Ready for input")?; - while let Some(evt) = select_event(&mut server, &mut rx).await { match evt { - SelectResult::Event(e) => { - match e { - Event::MicAudioChunk(chunk) => { - if !start_submit_audio { - start_submit_audio = true; - log::info!("Starting to submit audio chunks to server"); - server - .send(protocol::ClientMessage::voice_input_start(Some(16000))) - .await?; - ui.refresh_input_if_waiting()?; - } - let audio_buffer_u8 = unsafe { - std::slice::from_raw_parts(chunk.as_ptr() as *const u8, chunk.len() * 2) - }; + SelectResult::Event(e) => match e { + Event::MicAudioChunk(chunk) => { + if !start_submit_audio { + start_submit_audio = true; + log::info!("Starting to submit audio chunks to server"); + server + .send(protocol::ClientMessage::voice_input_start(Some(16000))) + .await?; + // ui.show_notification(ColorFormat::CSS_DARK_GREEN, "Voice input started")?; + ui.start_input("")?; + } + let audio_buffer_u8 = unsafe { + std::slice::from_raw_parts(chunk.as_ptr() as *const u8, chunk.len() * 2) + }; + server + .send(protocol::ClientMessage::voice_input_chunk( + audio_buffer_u8.to_vec(), + )) + .await?; + } + Event::MicAudioChunkEnd => { + start_submit_audio = false; + server + .send(protocol::ClientMessage::voice_input_end()) + .await?; + ui.refresh_input_display()?; + } + Event::RotateUp => { + if ui.is_input_mode() { + ui.move_cursor_left()?; + } else { + server.send(protocol::ClientMessage::ScrollUp).await?; + } + } + Event::RotateDown => { + if ui.is_input_mode() { + ui.move_cursor_right()?; + } else { + server.send(protocol::ClientMessage::ScrollDown).await?; + } + } + Event::Esc => { + if ui.is_input_mode() { + ui.clear_input()?; + } else { server - .send(protocol::ClientMessage::voice_input_chunk( - audio_buffer_u8.to_vec(), - )) + .send(protocol::ClientMessage::PtyInput(vec![0x1b])) .await?; } - Event::MicAudioChunkEnd => { - start_submit_audio = false; - ui.refresh_input_if_waiting()?; + } + Event::Accept => { + if ui.is_input_mode() { + let input = ui.take_waiting_input_prompt(); + server.send(protocol::ClientMessage::Input(input)).await?; + } else { server - .send(protocol::ClientMessage::voice_input_end()) + .send(protocol::ClientMessage::PtyInput(vec![0x0d])) .await?; } - evt => { - log::info!("Received event: {:?}", evt); - - match ui.state() { - lcd::UiState::WaitingInput { .. } => { - ui.handle_key_event_on_waiting_input(evt, &mut server) - .await?; - } - lcd::UiState::WaitingChoice { .. } => { - ui.handle_key_event_on_choice_selection(evt, &mut server) - .await?; - } - &lcd::UiState::WaitingChoiceAllowCustom { .. } => { - ui.handle_key_event_on_choice_selection(evt, &mut server) - .await?; - } - lcd::UiState::ShowingNotification { .. } => { - ui.handle_key_event_on_displaying_text(evt, &mut server) - .await?; - } - _ => { - log::info!("Received event {:?} in state {:?}, handling with default handler", evt, ui.state()); - } - } + } + Event::NEXT => { + if let Some(bytes) = keymaps + .keys + .get(KeymapConfig::KEY_NEXT) + .and_then(|action| key_action_to_ansi(action)) + { + server + .send(protocol::ClientMessage::PtyInput(bytes)) + .await?; + } else { + // Fallback to default DOWN arrow + server + .send(protocol::ClientMessage::PtyInput(b"\x1b[B".to_vec())) + .await?; } } - } + Event::Backspace => { + if ui.is_input_mode() { + ui.delete_char_before_cursor()?; + } else { + server + .send(protocol::ClientMessage::PtyInput(vec![0x08])) + .await?; + } + } + Event::SwitchMode => { + if let Some(bytes) = keymaps + .keys + .get(KeymapConfig::KEY_SWITCH) + .and_then(|action| key_action_to_ansi(action)) + { + server + .send(protocol::ClientMessage::PtyInput(bytes)) + .await?; + } else { + // Fallback to default DOWN arrow + server + .send(protocol::ClientMessage::PtyInput(b"\x1b[Z".to_vec())) + .await?; + } + } + Event::RotatePush => { + if let Some(bytes) = keymaps + .keys + .get(KeymapConfig::KEY_ROTATE) + .and_then(|action| key_action_to_ansi(action)) + { + server + .send(protocol::ClientMessage::PtyInput(bytes)) + .await?; + } else { + server + .send(protocol::ClientMessage::PtyInput(b"/".to_vec())) + .await?; + } + } + Event::Custom => { + if let Some(bytes) = keymaps + .keys + .get(KeymapConfig::KEY_CUSTOM) + .and_then(|action| key_action_to_ansi(action)) + { + server + .send(protocol::ClientMessage::PtyInput(bytes)) + .await?; + } else { + server + .send(protocol::ClientMessage::PtyInput(b"/compact".to_vec())) + .await?; + } + } + }, SelectResult::ServerMessage(msg) => match msg { protocol::ServerMessage::PtyOutput(..) => { log::trace!("Received PTY output, ignoring for now"); @@ -152,141 +226,161 @@ pub async fn run( Ok(()) } -impl lcd::UI { - async fn handle_key_event_on_waiting_input( - &mut self, - evt: Event, - server: &mut crate::ws::Server, - ) -> anyhow::Result<()> { - match evt { - Event::Esc => { - self.clear_input()?; - } - Event::RotateDown => { - self.move_cursor_right()?; - } - Event::RotateUp => { - self.move_cursor_left()?; - } - Event::Backspace => { - self.remove_input_char()?; - } - Event::Accept => { - let input = self.get_input().unwrap_or_default(); - if input.is_empty() { - log::info!("Input is empty, ignoring submit"); - return Ok(()); +/// Convert KeyAction to ANSI escape sequences for terminal input +/// +/// # Arguments +/// * `action` - The key action to convert +/// +/// # Returns +/// * `Some(bytes)` - ANSI bytes to send +/// * `None` - Nothing to send (unknown key) +pub fn key_action_to_ansi(action: &bt_keyboard_mode::KeyAction) -> Option> { + use bt_keyboard_mode::KeyAction; + + match action { + KeyAction::Combo { modifiers, key, .. } => { + let key_upper = key.to_uppercase(); + let mut result = Vec::new(); + + // Check each modifier and apply corresponding ANSI sequence + let mut has_ctrl = false; + let mut has_alt = false; + let mut has_shift = false; + + for mod_name in modifiers { + match mod_name.as_str() { + "ctrl" => has_ctrl = true, + "shift" => has_shift = true, + "alt" | "option" => has_alt = true, + "meta" | "command" | "cmd" | "win" | "gui" => { + // Meta not supported in ANSI, ignore + } + _ => {} } - log::info!("Submitting input: {}", input); - server.send(protocol::ClientMessage::input(input)).await?; - } - Event::SwitchMode => { - // shift + tab - server - .send(protocol::ClientMessage::PtyInput(b"\x1b[Z".to_vec())) - .await?; } - _ => { - log::warn!("Unexpected event in WaitingInput state"); - } - } - - Ok(()) - } - async fn handle_key_event_on_choice_selection( - &mut self, - evt: Event, - server: &mut crate::ws::Server, - ) -> anyhow::Result<()> { - match evt { - Event::RotateDown => { - if self.allow_input() { - self.move_cursor_right()?; - } else { - self.scroll_down()?; - } - } - Event::RotateUp => { - if self.allow_input() { - self.move_cursor_left()?; - } else { - self.scroll_up()?; + // Get the base key character + let base_char = match key_upper.as_str() { + // Letters + "A" => Some(b'a'), + "B" => Some(b'b'), + "C" => Some(b'c'), + "D" => Some(b'd'), + "E" => Some(b'e'), + "F" => Some(b'f'), + "G" => Some(b'g'), + "H" => Some(b'h'), + "I" => Some(b'i'), + "J" => Some(b'j'), + "K" => Some(b'k'), + "L" => Some(b'l'), + "M" => Some(b'm'), + "N" => Some(b'n'), + "O" => Some(b'o'), + "P" => Some(b'p'), + "Q" => Some(b'q'), + "R" => Some(b'r'), + "S" => Some(b's'), + "T" => Some(b't'), + "U" => Some(b'u'), + "V" => Some(b'v'), + "W" => Some(b'w'), + "X" => Some(b'x'), + "Y" => Some(b'y'), + "Z" => Some(b'z'), + // Numbers + "0" => Some(b'0'), + "1" => Some(b'1'), + "2" => Some(b'2'), + "3" => Some(b'3'), + "4" => Some(b'4'), + "5" => Some(b'5'), + "6" => Some(b'6'), + "7" => Some(b'7'), + "8" => Some(b'8'), + "9" => Some(b'9'), + // Special keys + "SPACE" => Some(b' '), + "ENTER" | "RETURN" => Some(0x0d), + "TAB" => Some(0x09), + "ESC" | "ESCAPE" => Some(0x1b), + "BACKSPACE" => Some(0x08), + "DELETE" | "DEL" => Some(0x7f), + // Arrow keys (ANSI escape sequences) + "UP" => return Some(b"\x1b[A".to_vec()), + "DOWN" => return Some(b"\x1b[B".to_vec()), + "RIGHT" => return Some(b"\x1b[C".to_vec()), + "LEFT" => return Some(b"\x1b[D".to_vec()), + // Function keys + "F1" => return Some(b"\x1bOP".to_vec()), + "F2" => return Some(b"\x1bOQ".to_vec()), + "F3" => return Some(b"\x1bOR".to_vec()), + "F4" => return Some(b"\x1bOS".to_vec()), + "F5" => return Some(b"\x1b[15~".to_vec()), + "F6" => return Some(b"\x1b[17~".to_vec()), + "F7" => return Some(b"\x1b[18~".to_vec()), + "F8" => return Some(b"\x1b[19~".to_vec()), + "F9" => return Some(b"\x1b[20~".to_vec()), + "F10" => return Some(b"\x1b[21~".to_vec()), + "F11" => return Some(b"\x1b[23~".to_vec()), + "F12" => return Some(b"\x1b[24~".to_vec()), + // Symbols (basic set) + "MINUS" | "-" => Some(b'-'), + "EQUAL" | "=" => Some(b'='), + "LEFT_BRACKET" | "[" => Some(b'['), + "RIGHT_BRACKET" | "]" => Some(b']'), + "BACKSLASH" | "\\" => Some(b'\\'), + "SEMICOLON" | ";" => Some(b';'), + "QUOTE" | "'" => Some(b'\''), + "COMMA" | "," => Some(b','), + "PERIOD" | "." => Some(b'.'), + "SLASH" | "/" => Some(b'/'), + "GRAVE" | "`" => Some(b'`'), + _ => { + log::warn!("Unknown key for ANSI conversion: {}", key); + None } - } - Event::RotatePush => { - self.reset_scroll()?; - } - Event::NEXT => self.next_choice()?, - Event::Backspace => { - if self.allow_input() { - self.remove_input_char()?; + }; + + if let Some(mut ch) = base_char { + // Apply shift modifier (for letters and symbols) + if has_shift && ch.is_ascii_lowercase() { + ch = ch.to_ascii_uppercase(); } - } - Event::Accept => { - if self.is_confirm_dialog() { - server - .send(protocol::ClientMessage::pty_input(b"\r".to_vec())) - .await?; - } else { - if let Some(choice) = self.confirm_choice() { - server.send(choice).await?; - log::info!("Confirmed choice, sent to server"); - } else { - log::debug!("No choice selected, ignoring accept event"); + + // Handle Ctrl modifier - sends control character (0x00-0x1F) + // Ctrl+A = 0x01, Ctrl+B = 0x02, ..., Ctrl+Z = 0x1A + if has_ctrl { + if ch.is_ascii_lowercase() || ch.is_ascii_uppercase() { + // A-Z maps to 0x01-0x1A + ch = ch.to_ascii_uppercase(); + ch = ch - b'@'; // A=0x41, 0x41-0x40=0x01 + } else if ch == b' ' { + ch = 0x00; // Ctrl+Space = NUL } + result.push(ch); } - } - Event::Esc => { - if !self.allow_input() { - server - .send(protocol::ClientMessage::pty_input(b"\x1b".to_vec())) - .await?; - } else { - self.clear_input()?; - } - } - _ => { - log::warn!("Unexpected event in ChoiceSelection state"); - } - } - Ok(()) - } + // Handle Alt modifier - sends ESC prefix + character + // Alt+T = ESC + t + if has_alt { + result.push(0x1b); // ESC + result.push(ch); + } - async fn handle_key_event_on_displaying_text( - &mut self, - evt: Event, - server: &mut crate::ws::Server, - ) -> anyhow::Result<()> { - match evt { - Event::RotateDown => { - self.scroll_down()?; - } - Event::RotateUp => { - self.scroll_up()?; - } - Event::RotatePush => { - self.reset_scroll()?; - } - Event::Accept => { - self.scroll_up()?; - } - Event::SwitchMode => { - // shift + tab - server - .send(protocol::ClientMessage::PtyInput(b"\x1b[Z".to_vec())) - .await?; - } - Event::Custom => { - server.send(protocol::ClientMessage::Sync).await?; + // If no modifiers or only shift, just send the character + if !has_ctrl && !has_alt { + result.push(ch); + } } - _ => { - log::warn!("Unexpected event in DisplayingText state"); + + if result.is_empty() { + None + } else { + Some(result) } } - - Ok(()) + KeyAction::Text { value, .. } => Some(value.as_bytes().to_vec()), } } diff --git a/src/bt_keyboard_mode.rs b/src/bt_keyboard_mode.rs index 003f582..1c223cc 100644 --- a/src/bt_keyboard_mode.rs +++ b/src/bt_keyboard_mode.rs @@ -54,6 +54,21 @@ pub struct KeymapConfig { } impl KeymapConfig { + // Key name constants + pub const KEY_MIC: &'static str = "MIC"; + pub const KEY_CUSTOM: &'static str = "CUSTOM"; + pub const KEY_ESC: &'static str = "ESC"; + pub const KEY_NEXT: &'static str = "NEXT"; + pub const KEY_BACKSPACE: &'static str = "BACKSPACE"; + pub const KEY_SWITCH: &'static str = "SWITCH"; + pub const KEY_ACCEPT: &'static str = "ACCEPT"; + pub const KEY_ROTATE: &'static str = "ROTATE"; + + pub fn clear_nvs(nvs: &mut esp_idf_svc::nvs::EspDefaultNvs) -> anyhow::Result<()> { + nvs.remove("keymap_config")?; + Ok(()) + } + pub fn from_json(json: &str) -> anyhow::Result { Ok(serde_json::from_str(json)?) } @@ -94,14 +109,14 @@ impl KeymapConfig { /// Get key name from KeysPin index pub fn get_key_name(pin_index: u8) -> &'static str { match pin_index { - KeysPin::MIC => "MIC", - KeysPin::CUSTOM => "CUSTOM", - KeysPin::ESC => "ESC", - KeysPin::NEXT => "NEXT", - KeysPin::BACKSPACE => "BACKSPACE", - KeysPin::SWITCH => "SWITCH", - KeysPin::ACCEPT => "ACCEPT", - KeysPin::ROTATE_BUTTON => "ROTATE", + KeysPin::MIC => Self::KEY_MIC, + KeysPin::CUSTOM => Self::KEY_CUSTOM, + KeysPin::ESC => Self::KEY_ESC, + KeysPin::NEXT => Self::KEY_NEXT, + KeysPin::BACKSPACE => Self::KEY_BACKSPACE, + KeysPin::SWITCH => Self::KEY_SWITCH, + KeysPin::ACCEPT => Self::KEY_ACCEPT, + KeysPin::ROTATE_BUTTON => Self::KEY_ROTATE, _ => "UNKNOWN", } } diff --git a/src/bt_wifi_mode.rs b/src/bt_wifi_mode.rs index 9abe1e3..9effc06 100644 --- a/src/bt_wifi_mode.rs +++ b/src/bt_wifi_mode.rs @@ -23,6 +23,16 @@ pub struct Setting { } impl Setting { + pub fn clear_nvs(nvs: &mut esp_idf_svc::nvs::EspDefaultNvs) -> anyhow::Result<()> { + nvs.remove("ssid")?; + nvs.remove("pass")?; + nvs.remove("server_url")?; + nvs.remove("background_png")?; + nvs.remove("mic_model")?; + nvs.remove("state")?; + Ok(()) + } + pub fn load_from_nvs(nvs: &esp_idf_svc::nvs::EspDefaultNvs) -> anyhow::Result { let mut str_buf = [0; 128]; diff --git a/src/lcd.rs b/src/lcd.rs index b80b0b6..70b92b2 100644 --- a/src/lcd.rs +++ b/src/lcd.rs @@ -373,6 +373,139 @@ pub fn display_png( Ok(()) } +mod new_jpg { + use esp_idf_svc::sys::*; + + struct JpegDecoder { + handle: jpeg_dec_handle_t, + } + + impl JpegDecoder { + fn open(config: &jpeg_dec_config_t) -> Result { + unsafe { + let mut handle: jpeg_dec_handle_t = std::ptr::null_mut(); + let ret = jpeg_dec_open( + config as *const jpeg_dec_config_t as *mut jpeg_dec_config_t, + &mut handle, + ); + if ret != jpeg_error_t_JPEG_ERR_OK { + return Err(ret); + } + Ok(JpegDecoder { handle }) + } + } + } + + impl Drop for JpegDecoder { + fn drop(&mut self) { + if !self.handle.is_null() { + unsafe { + jpeg_dec_close(self.handle); + } + } + } + } + + pub struct JpegBuffer { + ptr: *mut u8, + size: usize, + } + + impl JpegBuffer { + fn new(size: usize, aligned: std::ffi::c_int) -> anyhow::Result { + unsafe { + let ptr = jpeg_calloc_align(size, aligned); + if ptr.is_null() { + return Err(anyhow::anyhow!("Failed to allocate JPEG buffer")); + } + Ok(JpegBuffer { + ptr: ptr as *mut u8, + size, + }) + } + } + + pub fn flush_to_lcd(&self) -> i32 { + let ptr = unsafe { std::slice::from_raw_parts(self.ptr.cast_const(), self.size) }; + super::flush_display(ptr, 0, 0, 288, 80) + } + } + + impl Drop for JpegBuffer { + fn drop(&mut self) { + if !self.ptr.is_null() { + unsafe { + jpeg_free_align(self.ptr as *mut _); + } + } + } + } + + pub fn esp_jpeg_decode_one_picture(data: &[u8]) -> anyhow::Result { + unsafe { + use esp_idf_svc::sys::*; + + // Generate default configuration + let mut config = jpeg_dec_config_t::default(); + config.output_type = jpeg_pixel_format_t_JPEG_PIXEL_FORMAT_RGB565_LE; + config.clipper.height = 80; + config.clipper.width = 288; + + // Create jpeg_dec handle + let decoder = JpegDecoder::open(&config) + .map_err(|e| anyhow::anyhow!("Failed to open JPEG decoder: error code {}", e))?; + + // Create io_callback handle + let mut jpeg_io = Box::new(jpeg_dec_io_t::default()); + + // Create out_info handle + let mut out_info = Box::new(jpeg_dec_header_info_t::default()); + + // Set input buffer and buffer len to io_callback + jpeg_io.inbuf = data.as_ptr() as *mut u8; + jpeg_io.inbuf_len = data.len() as i32; + + // Parse jpeg picture header and get picture for user and decoder + let ret = jpeg_dec_parse_header(decoder.handle, jpeg_io.as_mut(), out_info.as_mut()); + if ret != jpeg_error_t_JPEG_ERR_OK { + return Err(anyhow::anyhow!( + "Failed to parse JPEG header: error code {}", + ret + )); + } + + // Calculate output length based on pixel format + // Default to RGB565 (2 bytes per pixel) + let out_len = (*out_info).width * (*out_info).height * 2; + + // Allocate aligned output buffer + let out_buf = JpegBuffer::new(out_len as usize, 16)?; + + jpeg_io.outbuf = out_buf.ptr; + + // Start decode jpeg + let ret = jpeg_dec_process(decoder.handle, jpeg_io.as_mut()); + if ret != jpeg_error_t_JPEG_ERR_OK { + return Err(anyhow::anyhow!("Failed to decode JPEG: error code {}", ret)); + } + + Ok(out_buf) + } + } +} + +pub fn display_jpeg(jpeg: &[u8]) -> anyhow::Result<()> { + let jpeg_buffer = new_jpg::esp_jpeg_decode_one_picture(jpeg)?; + let e = jpeg_buffer.flush_to_lcd(); + if e != 0 { + return Err(anyhow::anyhow!( + "Failed to flush JPEG to LCD: error code {}", + e + )); + } + Ok(()) +} + pub fn display_text( display_target: &mut FrameBuffer, text: &str, @@ -394,7 +527,7 @@ pub fn display_text( MyTextStyle { font_style: U8g2TextStyle::new( u8g2_fonts::fonts::u8g2_font_wqy12_t_gb2312, - ColorFormat::CSS_BLACK, + ColorFormat::CSS_WHEAT, ), vertical_offset: 3, bg_color: None, @@ -421,6 +554,7 @@ pub enum UiMessage { ScreenImage { data: Vec, format: ImageFormat, + is_last: bool, }, /// 通知消息 @@ -430,33 +564,22 @@ pub enum UiMessage { title: Option, }, - /// 请求输入 - GetInput { - prompt: String, - }, - - /// 提供选择项 - Choices { - id: String, - title: String, - options: Vec, - multi_select: bool, - allow_custom_input: bool, - }, - /// ASR 结果 AsrResult(String), - - Status(String), } impl Debug for UiMessage { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - UiMessage::ScreenImage { data, format } => f + UiMessage::ScreenImage { + data, + format, + is_last, + } => f .debug_struct("ScreenImage") .field("format", format) .field("data_len", &data.len()) + .field("is_last", is_last) .finish(), UiMessage::Notification { message, @@ -468,32 +591,10 @@ impl Debug for UiMessage { .field("message", &message.chars().take(20).collect::()) .field("title", title) .finish(), - UiMessage::GetInput { prompt } => f - .debug_tuple("GetInput") - .field(&prompt.chars().take(20).collect::()) - .finish(), - UiMessage::Choices { - id, - title, - options, - multi_select, - allow_custom_input, - } => f - .debug_struct("Choices") - .field("id", id) - .field("title", &title.chars().take(20).collect::()) - .field("options_count", &options.len()) - .field("multi_select", &multi_select) - .field("allow_custom_input", &allow_custom_input) - .finish(), UiMessage::AsrResult(text) => f .debug_tuple("AsrResult") .field(&text.chars().take(20).collect::()) .finish(), - UiMessage::Status(status) => f - .debug_tuple("Status") - .field(&status.chars().take(20).collect::()) - .finish(), } } } @@ -529,44 +630,6 @@ impl NotificationLevel { // ========== UI 状态 ========== -/// UI 当前状态 -#[derive(Clone, Debug)] -pub enum UiState { - /// 空闲状态 - Idle, - - /// 显示图片 - ShowingImage, - - /// 显示通知 - ShowingNotification { color: ColorFormat, message: String }, - - /// 等待输入 - WaitingInput { - prompt: String, - current_input: String, - cursor_pos: usize, - }, - - /// 等待选择 - WaitingChoice { - id: String, - title: String, - options: Vec, - selected_index: i32, - }, - WaitingChoiceAllowCustom { - id: String, - title: String, - options: Vec, - multi_select: bool, - current_selected: i32, - selected_indices: HashSet, - custom_input: String, - cursor_pos: usize, - }, -} - // ========== UI 组件 ========== /// UI 渲染配置 @@ -579,7 +642,7 @@ pub struct UiConfig { impl Default for UiConfig { fn default() -> Self { Self { - text_color: ColorFormat::CSS_BLACK, + text_color: ColorFormat::CSS_WHEAT, } } } @@ -592,26 +655,27 @@ impl Default for UiConfig { pub struct UI { /// 显示缓冲区 display: FrameBuffer, - /// 当前 UI 状态 - state: UiState, - status_bar: String, + /// UI 配置 config: UiConfig, - scroll_offset: i32, - waiting_input_prompt: String, + asr_input: String, + asr_cursor_pos: usize, + input_mode: bool, + + image_buffer: Vec, } impl UI { /// 创建新的 UI 实例 pub fn new() -> Self { Self { - display: FrameBuffer::new(ColorFormat::CSS_WHITE), - state: UiState::Idle, + display: FrameBuffer::new(ColorFormat::new(30, 30, 30)), + input_mode: false, + image_buffer: Vec::with_capacity(1024), config: UiConfig::default(), - scroll_offset: 0, - waiting_input_prompt: String::new(), - status_bar: "[N]".to_string(), + asr_input: String::new(), + asr_cursor_pos: 0, } } @@ -619,76 +683,65 @@ impl UI { pub fn new_with_target(display: FrameBuffer) -> Self { Self { display, - state: UiState::Idle, + input_mode: false, + image_buffer: Vec::with_capacity(1024), config: UiConfig::default(), - scroll_offset: 0, - waiting_input_prompt: String::new(), - status_bar: "[N]".to_string(), + asr_input: String::new(), + asr_cursor_pos: 0, } } /// 处理 UI 消息 (对应 protocol.rs 的 ServerMessage) pub fn handle_message(&mut self, msg: UiMessage) -> anyhow::Result<()> { - log::info!("Handling UI message: {:?}", msg); match msg { - UiMessage::ScreenImage { data, format } => self.show_image(&data, format), - UiMessage::Notification { message, color, .. } => { - self.show_notification(color, &message) - } - UiMessage::GetInput { prompt } => self.start_input(&prompt), - UiMessage::Choices { - title, - options, - id, - multi_select, - allow_custom_input, + UiMessage::ScreenImage { + data, + format, + is_last, } => { - if allow_custom_input { - self.show_allow_custom_choices(&id, &title, &options, multi_select) - } else { - self.show_choices(&id, &title, &options) + self.image_buffer.extend_from_slice(&data); + if is_last { + self.show_self_image_buffer(format)?; + self.image_buffer.clear(); } + Ok(()) } - UiMessage::AsrResult(text) => self.show_asr_result(&text), - UiMessage::Status(status) => self.set_status(&status), + UiMessage::Notification { message, color, .. } => { + // self.show_notification(color, &message) + log::info!("[TODO] Showing notification: {}", message); + Ok(()) + } + UiMessage::AsrResult(text) => self.input_asr_result(&text), } } - pub fn reset_scroll(&mut self) -> anyhow::Result<()> { - self.scroll_offset = 0; - match &self.state { - UiState::ShowingNotification { .. } => self.refresh_notification(), - UiState::WaitingChoice { .. } => self.refresh_choices_display(), - UiState::WaitingChoiceAllowCustom { .. } => self.refresh_allow_custom_choices_display(), - _ => Ok(()), - } - } + pub fn show_self_image_buffer(&mut self, format: ImageFormat) -> anyhow::Result<()> { + let data = &self.image_buffer; - pub fn scroll_up(&mut self) -> anyhow::Result<()> { - self.scroll_offset -= 14; - match &self.state { - UiState::ShowingNotification { .. } => self.refresh_notification(), - UiState::WaitingChoice { .. } => self.refresh_choices_display(), - UiState::WaitingChoiceAllowCustom { .. } => self.refresh_allow_custom_choices_display(), - _ => Ok(()), + match format { + ImageFormat::Png => { + let img_reader = image::ImageReader::with_format( + std::io::Cursor::new(data), + image::ImageFormat::Png, + ); + let img = img_reader.decode()?.to_rgb8(); + self.draw_rgb888(&img)?; + self.display.flush()?; + } + ImageFormat::Jpeg => { + display_jpeg(data)?; + } + ImageFormat::Gif => { + // GIF 动画处理可以在这里扩展 + log::warn!("GIF format not fully supported yet"); + } } - } - pub fn scroll_down(&mut self) -> anyhow::Result<()> { - self.scroll_offset += 14; - match &self.state { - UiState::ShowingNotification { .. } => self.refresh_notification(), - UiState::WaitingChoice { .. } => self.refresh_choices_display(), - UiState::WaitingChoiceAllowCustom { .. } => self.refresh_allow_custom_choices_display(), - _ => Ok(()), - } + Ok(()) } /// 显示图片 pub fn show_image(&mut self, data: &[u8], format: ImageFormat) -> anyhow::Result<()> { - self.state = UiState::ShowingImage; - self.scroll_offset = 0; - match format { ImageFormat::Png => { let img_reader = image::ImageReader::with_format( @@ -697,14 +750,10 @@ impl UI { ); let img = img_reader.decode()?.to_rgb8(); self.draw_rgb888(&img)?; + self.display.flush()?; } ImageFormat::Jpeg => { - let img_reader = image::ImageReader::with_format( - std::io::Cursor::new(data), - image::ImageFormat::Jpeg, - ); - let img = img_reader.decode()?.to_rgb8(); - self.draw_rgb888(&img)?; + display_jpeg(data)?; } ImageFormat::Gif => { // GIF 动画处理可以在这里扩展 @@ -712,7 +761,6 @@ impl UI { } } - self.display.flush()?; Ok(()) } @@ -736,93 +784,18 @@ impl UI { Ok(()) } - /// 显示通知 - pub fn show_notification(&mut self, color: ColorFormat, message: &str) -> anyhow::Result<()> { - if message.is_empty() { - if let UiState::ShowingNotification { color: color_, .. } = &mut self.state { - *color_ = color; - self.refresh_notification()?; - return Ok(()); - } - } - - self.state = UiState::ShowingNotification { - color, - message: message.to_string(), - }; - self.scroll_offset = 0; - - // self.display.fill_color(self.config.notification_bg)?; - - const LINE_HEIGHT: i32 = 14; - - // 绘制顶部颜色条表示级别 - let bounding_box = self.display.bounding_box(); - let top_bar = Rectangle::new( - Point::new(0, 0), - Size::new(bounding_box.size.width, LINE_HEIGHT as u32), - ); - top_bar.draw_styled(&PrimitiveStyle::with_fill(color), &mut self.display)?; - - let status_bar_str = format!("{}", self.status_bar.clone()); - - self.draw_text( - &status_bar_str, - Point::new(4, 2), - ColorFormat::CSS_WHITE, - None, - false, - )?; - - // 显示消息 - self.draw_text_wrapped(message, Point::new(2, LINE_HEIGHT), self.config.text_color)?; - - self.display.flush()?; - Ok(()) - } - /// 开始输入模式 pub fn start_input(&mut self, prompt: &str) -> anyhow::Result<()> { - if matches!(self.state, UiState::WaitingInput { .. }) { - return Ok(()); // 已经在输入模式,不重复设置 - } - - // TODO: change state bar + self.input_mode = true; - if let UiState::ShowingNotification { color, .. } = &mut self.state { - *color = ColorFormat::new(255, 150, 0); // 切换到输入模式,先把通知颜色改为橙色 - self.waiting_input_prompt = prompt.to_string(); - self.refresh_notification()?; - return Ok(()); // 正在显示通知,先保存输入提示,等刷新时再切换到输入模式 - } - - if cfg!(debug_assertions) { - unreachable!("Unexpected state when starting input: {:?}", self.state); - } else { - // unreachable in current design, but just in case - self.state = UiState::WaitingInput { - prompt: prompt.to_string(), - current_input: String::new(), - cursor_pos: 0, - }; - self.refresh_input_display()?; - Ok(()) - } + self.input_asr_result(prompt)?; + Ok(()) } /// 刷新输入显示 - fn refresh_input_display(&mut self) -> anyhow::Result<()> { + pub fn refresh_input_display(&mut self) -> anyhow::Result<()> { // 提取需要的数据,避免借用冲突 - let (_prompt, current_input, cursor_pos) = if let UiState::WaitingInput { - prompt, - current_input, - cursor_pos, - } = &self.state - { - (prompt.clone(), current_input.clone(), *cursor_pos) - } else { - return Ok(()); - }; + let cursor_pos = self.asr_cursor_pos; // 检查麦克风状态 let is_mic_on = crate::audio::MIC_ON.load(std::sync::atomic::Ordering::Relaxed); @@ -846,21 +819,20 @@ impl UI { let bounding_box = self.display.bounding_box(); let top_bar = Rectangle::new(Point::new(0, 0), Size::new(bounding_box.size.width, 14)); top_bar.draw_styled(&PrimitiveStyle::with_fill(mic_color), &mut self.display)?; - let status_bar_str = self.status_bar.clone(); self.draw_text( - &status_bar_str, + &"Waiting", Point::new(4, 2), ColorFormat::CSS_WHITE, None, - false, + true, )?; 14 }; - let display_text = if current_input.is_empty() { + let display_text = if self.asr_input.is_empty() { "\x1b[44m_\x1b[49m".to_string() } else { - let chars: Vec = current_input.chars().collect(); + let chars: Vec = self.asr_input.chars().collect(); let mut input_with_cursor = String::new(); for (i, c) in chars.iter().enumerate() { if i == cursor_pos { @@ -887,707 +859,84 @@ impl UI { Ok(()) } - #[allow(unused)] - /// 添加输入字符(在光标位置插入) - pub fn add_input_char(&mut self, c: char) -> anyhow::Result<()> { - match &mut self.state { - UiState::WaitingInput { - current_input, - cursor_pos, - .. - } => { - let mut chars: Vec = current_input.chars().collect(); - chars.insert(*cursor_pos, c); - *current_input = chars.into_iter().collect(); - *cursor_pos += 1; - self.refresh_input_display()?; - } - UiState::WaitingChoiceAllowCustom { - custom_input, - cursor_pos, - .. - } => { - let mut chars: Vec = custom_input.chars().collect(); - chars.insert(*cursor_pos, c); - *custom_input = chars.into_iter().collect(); - *cursor_pos += 1; - self.refresh_allow_custom_choices_display()?; - } - _ => {} - } - Ok(()) - } - - /// 删除光标前的字符(backspace) - pub fn remove_input_char(&mut self) -> anyhow::Result<()> { - match &mut self.state { - UiState::WaitingInput { - current_input, - cursor_pos, - .. - } => { - if *cursor_pos > 0 { - let mut chars: Vec = current_input.chars().collect(); - chars.remove(*cursor_pos - 1); - *current_input = chars.into_iter().collect(); - *cursor_pos -= 1; - self.refresh_input_display()?; - } - } - UiState::WaitingChoiceAllowCustom { - custom_input, - cursor_pos, - .. - } => { - if *cursor_pos > 0 { - let mut chars: Vec = custom_input.chars().collect(); - chars.remove(*cursor_pos - 1); - *custom_input = chars.into_iter().collect(); - *cursor_pos -= 1; - self.refresh_allow_custom_choices_display()?; - } - } - _ => {} - } - Ok(()) - } - - #[allow(unused)] - /// 删除光标后的字符(delete) - pub fn delete_char_at_cursor(&mut self) -> anyhow::Result<()> { - match &mut self.state { - UiState::WaitingInput { - current_input, - cursor_pos, - .. - } => { - let mut chars: Vec = current_input.chars().collect(); - if *cursor_pos < chars.len() { - chars.remove(*cursor_pos); - *current_input = chars.into_iter().collect(); - self.refresh_input_display()?; - } - } - UiState::WaitingChoiceAllowCustom { - custom_input, - cursor_pos, - .. - } => { - let mut chars: Vec = custom_input.chars().collect(); - if *cursor_pos < chars.len() { - chars.remove(*cursor_pos); - *custom_input = chars.into_iter().collect(); - self.refresh_allow_custom_choices_display()?; - } - } - _ => {} - } - Ok(()) + pub fn is_input_mode(&self) -> bool { + self.input_mode } - /// 光标左移 - pub fn move_cursor_left(&mut self) -> anyhow::Result<()> { - match &mut self.state { - UiState::WaitingInput { cursor_pos, .. } => { - *cursor_pos = cursor_pos.saturating_sub(1); - self.refresh_input_display()?; - } - UiState::WaitingChoiceAllowCustom { cursor_pos, .. } => { - *cursor_pos = cursor_pos.saturating_sub(1); - self.refresh_allow_custom_choices_display()?; - } - _ => {} - } - Ok(()) - } - - /// 光标右移 - pub fn move_cursor_right(&mut self) -> anyhow::Result<()> { - match &mut self.state { - UiState::WaitingInput { - current_input, - cursor_pos, - .. - } => { - let max_pos = current_input.chars().count(); - *cursor_pos = (*cursor_pos + 1).min(max_pos); - self.refresh_input_display()?; - } - UiState::WaitingChoiceAllowCustom { - custom_input, - cursor_pos, - .. - } => { - let max_pos = custom_input.chars().count(); - *cursor_pos = (*cursor_pos + 1).min(max_pos); - self.refresh_allow_custom_choices_display()?; - } - _ => {} - } - Ok(()) - } - - /// 在光标位置插入文本(用于 ASR 结果) - pub fn insert_text_at_cursor(&mut self, text: &str) -> anyhow::Result<()> { - match &mut self.state { - UiState::WaitingInput { - current_input, - cursor_pos, - .. - } => { - let mut chars: Vec = current_input.chars().collect(); - let insert_chars: Vec = text.chars().collect(); - for c in insert_chars { - chars.insert(*cursor_pos, c); - *cursor_pos += 1; - } - *current_input = chars.into_iter().collect(); - self.refresh_input_display()?; - } - UiState::WaitingChoiceAllowCustom { - custom_input, - cursor_pos, - .. - } => { - let mut chars: Vec = custom_input.chars().collect(); - let insert_chars: Vec = text.chars().collect(); - for c in insert_chars { - chars.insert(*cursor_pos, c); - *cursor_pos += 1; - } - *custom_input = chars.into_iter().collect(); - self.refresh_allow_custom_choices_display()?; - } - _ => {} - } - Ok(()) - } - - pub fn insert_text_at_start(&mut self, text: &str) -> anyhow::Result<()> { - match &mut self.state { - UiState::WaitingInput { current_input, .. } => { - *current_input = format!("{}{}", text, current_input); - self.refresh_input_display()?; - } - UiState::WaitingChoiceAllowCustom { custom_input, .. } => { - *custom_input = format!("{}{}", text, custom_input); - self.refresh_allow_custom_choices_display()?; - } - _ => {} - } - Ok(()) - } - - /// 获取当前输入并返回 - pub fn get_input(&self) -> Option { - if let UiState::WaitingInput { current_input, .. } = &self.state { - Some(current_input.clone()) - } else { - None - } - } - - /// 清空当前输入 - pub fn clear_input(&mut self) -> anyhow::Result<()> { - let allow_input = self.allow_input(); - match &mut self.state { - UiState::WaitingInput { - current_input, - cursor_pos, - .. - } => { - self.scroll_offset = 0; - - *current_input = String::new(); - *cursor_pos = 0; - self.refresh_input_display()?; - } - UiState::WaitingChoiceAllowCustom { - custom_input, - cursor_pos, - .. - } if allow_input => { - *custom_input = String::new(); - *cursor_pos = 0; - self.refresh_allow_custom_choices_display()?; - } - _ => {} - } - Ok(()) - } - - #[allow(unused)] - /// 获取光标位置 - pub fn get_cursor_pos(&self) -> Option { - if let UiState::WaitingInput { cursor_pos, .. } = &self.state { - Some(*cursor_pos) - } else { - None - } - } - - /// 刷新输入界面(用于麦克风状态变化时) - pub fn refresh_input_if_waiting(&mut self) -> anyhow::Result<()> { - match self.state { - UiState::WaitingInput { .. } => self.refresh_input_display(), - UiState::WaitingChoiceAllowCustom { .. } | UiState::WaitingChoice { .. } => Ok(()), - _ => { - if self.waiting_input_prompt.is_empty() { - Ok(()) - } else { - // 之前正在显示通知时收到输入请求,先切换到输入模式 - self.state = UiState::WaitingInput { - prompt: self.take_waiting_input_prompt(), - current_input: String::new(), - cursor_pos: 0, - }; - self.refresh_input_display() - } - } - } - } - - /// 显示选择项 - pub fn show_choices( - &mut self, - id: &str, - title: &str, - options: &[String], - ) -> anyhow::Result<()> { - if let UiState::WaitingChoice { - id: existing_id, .. - } = &self.state - { - if existing_id == id { - return Ok(()); // 已经在选择模式,不重复设置 - } - } - - self.state = UiState::WaitingChoice { - id: id.to_string(), - title: title.to_string(), - options: options.to_vec(), - selected_index: 0, - }; - self.refresh_choices_display()?; - Ok(()) - } - - /// 刷新选择项显示 - fn refresh_choices_display(&mut self) -> anyhow::Result<()> { - // 提取需要的数据,避免借用冲突 - let (title, options, selected_index) = if let UiState::WaitingChoice { - title, - options, - selected_index, - .. - } = &self.state - { - (title.clone(), options.clone(), *selected_index) - } else { - return Ok(()); - }; - - // 使用 ANSI 代码构建选择项显示文本 - // 标题用灰色背景,选中项用蓝色背景 - let mut display_text = format!("{}\n", title); - - if !options.is_empty() { - for (i, option) in options.iter().enumerate() { - if i as i32 == selected_index { - // 选中项:蓝色背景,白色文字 - display_text.push_str(&format!("\x1b[44;97m[ {} ]\x1b[49m\n", option)); - } else { - // 未选中项:普通文字 - display_text.push_str(&format!(" {}\n", option)); - } - } - } else { - // Confirm/Cancel 固定显示 - display_text.push_str("\n\x1b[44m[Accept] Confirm\x1b[0m\t\t\t"); - display_text.push_str("\x1b[41m[ESC] Cancel\x1b[0m\n"); - } - - const LINE_HEIGHT: i32 = 14; - - let color = ColorFormat::new(255, 150, 0); - - // 绘制顶部颜色条表示级别 - let bounding_box = self.display.bounding_box(); - let top_bar = Rectangle::new( - Point::new(0, 0), - Size::new(bounding_box.size.width, LINE_HEIGHT as u32), - ); - top_bar.draw_styled(&PrimitiveStyle::with_fill(color), &mut self.display)?; - self.draw_text( - "● Choose an action", - Point::new(0, 2), - ColorFormat::CSS_WHITE, - None, - true, - )?; - - self.draw_text_wrapped( - &display_text, - Point::new(2, 2 + LINE_HEIGHT), - self.config.text_color, - )?; - + pub fn show_notification(&mut self, color: ColorFormat, message: &str) -> anyhow::Result<()> { + self.draw_text_wrapped(message, Point::new(2, 2), color)?; self.display.flush()?; Ok(()) } - pub fn show_allow_custom_choices( - &mut self, - id: &str, - title: &str, - options: &[String], - multi_select: bool, - ) -> anyhow::Result<()> { - if let UiState::WaitingChoiceAllowCustom { - id: existing_id, .. - } = &self.state - { - if existing_id == id { - return Ok(()); // 已经在选择模式,不重复设置 - } - } - - self.state = UiState::WaitingChoiceAllowCustom { - id: id.to_string(), - title: title.to_string(), - options: options.to_vec(), - multi_select, - current_selected: 0, - selected_indices: HashSet::new(), - custom_input: String::new(), - cursor_pos: 0, - }; - self.refresh_allow_custom_choices_display()?; - Ok(()) - } - - /// 刷新选择项显示 - fn refresh_allow_custom_choices_display(&mut self) -> anyhow::Result<()> { - let display_text = if let UiState::WaitingChoiceAllowCustom { - title, - options, - current_selected, - selected_indices, - custom_input, - cursor_pos, - multi_select, - .. - } = &self.state - { - // 使用 ANSI 代码构建选择项显示文本 - // 标题用灰色背景,选中项用蓝色背景 - let mut display_text = format!("{}\n", title); - - // 格式化 custom_input 显示光标(类似 refresh_input_display) - let custom_display = if custom_input.is_empty() { - "\x1b[44m_\x1b[49m".to_string() - } else { - let chars: Vec = custom_input.chars().collect(); - let mut input_with_cursor = String::new(); - for (i, c) in chars.iter().enumerate() { - if i == *cursor_pos { - input_with_cursor.push_str(&format!("\x1b[44m{}\x1b[49m", c)); - } else { - input_with_cursor.push(*c); - } - } - if *cursor_pos == chars.len() { - input_with_cursor.push_str("\x1b[44m_\x1b[49m"); - } - input_with_cursor - }; - - let extra_option = if *multi_select { - vec![ - format!("[Custom]: {}", custom_display), - "Submit".to_string(), - ] - } else { - vec![format!("[Custom]: {}", custom_display)] - }; - - for (i, option) in options.iter().chain(&extra_option).enumerate() { - let option = if *multi_select && selected_indices.contains(&(i as i32)) { - format!("{} (*)", option) - } else { - option.clone() - }; - - if i as i32 == *current_selected { - // 选中项:蓝色背景,白色文字 - display_text.push_str(&format!("\x1b[44;37m> {} <\x1b[49m\n", option)); - } else { - // 未选中项:普通文字 - display_text.push_str(&format!(" {}\n", option)); - } - } - - display_text - } else { - return Ok(()); - }; - - const LINE_HEIGHT: i32 = 14; + pub fn input_asr_result(&mut self, text: &str) -> anyhow::Result<()> { + log::info!("Inserting ASR result: {}", text); - let color = ColorFormat::new(255, 150, 0); + self.input_mode = true; - // 绘制顶部颜色条表示级别 - let bounding_box = self.display.bounding_box(); - let top_bar = Rectangle::new( - Point::new(0, 0), - Size::new(bounding_box.size.width, LINE_HEIGHT as u32), - ); - top_bar.draw_styled(&PrimitiveStyle::with_fill(color), &mut self.display)?; - self.draw_text( - "● Choose action", - Point::new(0, 2), - ColorFormat::CSS_WHITE, - None, - true, - )?; - - self.draw_text_wrapped( - &display_text, - Point::new(2, 2 + LINE_HEIGHT), - self.config.text_color, - )?; + // 将字符索引转换为字节索引(支持中文等多字节字符) + let byte_pos = self + .asr_input + .char_indices() + .nth(self.asr_cursor_pos) + .map(|(i, _)| i) + .unwrap_or(self.asr_input.len()); - self.display.flush()?; + self.asr_input.insert_str(byte_pos, text); + self.asr_cursor_pos += text.chars().count(); + self.refresh_input_display()?; Ok(()) } - pub fn allow_input(&self) -> bool { - match &self.state { - UiState::WaitingInput { .. } => true, - UiState::WaitingChoiceAllowCustom { - options, - current_selected, - .. - } => { - // 只有选中 Custom 选项时才允许输入 - *current_selected == options.len() as i32 - } - _ => false, - } - } - - /// 选择下一项(可循环) - pub fn next_choice(&mut self) -> anyhow::Result<()> { - match &mut self.state { - UiState::WaitingChoice { - options, - selected_index, - .. - } => { - // 空选项时不做处理(Confirm/Cancel 只能通过按键选择) - if options.is_empty() { - return Ok(()); - } - *selected_index = (*selected_index + 1) % options.len() as i32; - self.refresh_choices_display()?; - } - UiState::WaitingChoiceAllowCustom { - options, - current_selected, - multi_select, - .. - } => { - // 空选项时不做处理 - if options.is_empty() { - return Ok(()); - } - let extra_options = if *multi_select { 2 } else { 1 }; - *current_selected = - (*current_selected + 1) % (options.len() + extra_options) as i32; - self.refresh_allow_custom_choices_display()?; - } - _ => {} + /// 向左移动光标 + pub fn move_cursor_left(&mut self) -> anyhow::Result<()> { + if self.asr_cursor_pos > 0 { + self.asr_cursor_pos -= 1; + self.refresh_input_display()?; } Ok(()) } - #[allow(unused)] - /// 选择上一项(可循环) - pub fn prev_choice(&mut self) -> anyhow::Result<()> { - match &mut self.state { - UiState::WaitingChoiceAllowCustom { - options, - current_selected, - .. - } => { - // 空选项时不做处理 - if options.is_empty() { - return Ok(()); - } - let len = options.len(); - *current_selected = if *current_selected == 0 { - (len - 1) as i32 - } else { - *current_selected - 1 - }; - self.refresh_allow_custom_choices_display()?; - } - _ => {} + /// 向右移动光标 + pub fn move_cursor_right(&mut self) -> anyhow::Result<()> { + let max_pos = self.asr_input.chars().count(); + if self.asr_cursor_pos < max_pos { + self.asr_cursor_pos += 1; + self.refresh_input_display()?; } Ok(()) } - /// 确认选择并返回选中的索引 - pub fn confirm_choice(&mut self) -> Option { - match &mut self.state { - UiState::WaitingChoice { selected_index, .. } => { - Some(ClientMessage::choice(*selected_index)) - } - UiState::WaitingChoiceAllowCustom { - selected_indices, - custom_input, - multi_select, - current_selected, - options, - .. - } => { - if *multi_select { - if *current_selected == options.len() as i32 + 1 { - selected_indices.remove(&(options.len() as i32)); - Some(ClientMessage::Choices { - index: selected_indices.iter().cloned().collect(), - custom_input: Some(custom_input.clone()), - multi_select: true, - }) - } else { - if selected_indices.contains(current_selected) { - selected_indices.remove(current_selected); - } else { - selected_indices.insert(*current_selected); - } - if let Err(e) = self.refresh_allow_custom_choices_display() { - log::error!("Failed to refresh choices display: {:?}", e); - } - None - } - } else { - Some(ClientMessage::Choices { - index: vec![*current_selected], - custom_input: Some(custom_input.clone()), - multi_select: false, - }) - } - } - _ => None, - } - } - - /// 检查是否为 confirm/cancel 对话框(空选项) - pub fn is_confirm_dialog(&self) -> bool { - if let UiState::WaitingChoice { options, .. } = &self.state { - options.is_empty() - } else { - false - } - } - - /// 刷新通知显示(用于滚动) - pub fn refresh_notification(&mut self) -> anyhow::Result<()> { - if let UiState::ShowingNotification { color, message } = &self.state { - // self.display.fill_color(self.config.notification_bg)?; - let message = message.clone(); - let color = *color; - - const LINE_HEIGHT: i32 = 14; - - // 绘制顶部颜色条表示级别 - let bounding_box = self.display.bounding_box(); - let top_bar = Rectangle::new( - Point::new(0, 0), - Size::new(bounding_box.size.width, LINE_HEIGHT as u32), - ); - top_bar.draw_styled(&PrimitiveStyle::with_fill(color), &mut self.display)?; - if !self.waiting_input_prompt.is_empty() { - self.draw_text( - "● Waiting for next step", - Point::new(4, 2), - ColorFormat::CSS_WHITE, - None, - true, - )?; - } - - let status_bar_str = self.status_bar.clone(); - - self.draw_text( - &status_bar_str, - Point::new(4, 2), - ColorFormat::CSS_WHITE, - None, - false, - )?; - - let y_offset = LINE_HEIGHT; - - // 显示消息 - self.draw_text_wrapped(&message, Point::new(2, y_offset), self.config.text_color)?; - - self.display.flush()?; + pub fn delete_char_before_cursor(&mut self) -> anyhow::Result<()> { + if self.asr_cursor_pos > 0 { + // 将字符索引转换为字节索引(支持中文等多字节字符) + let byte_pos = self + .asr_input + .char_indices() + .nth(self.asr_cursor_pos - 1) + .map(|(i, _)| i) + .unwrap_or(0); + + self.asr_input.remove(byte_pos); + self.asr_cursor_pos -= 1; + self.refresh_input_display()?; } Ok(()) } - /// 显示 ASR 结果(如果在输入模式,直接插入到光标位置) - pub fn show_asr_result(&mut self, text: &str) -> anyhow::Result<()> { - // 如果当前在输入模式,直接插入到光标位置 - - if self.allow_input() { - self.insert_text_at_cursor(text) - } else if !self.waiting_input_prompt.is_empty() || matches!(self.state, UiState::Idle) { - // 如果之前显示过文本或通知,并且有未进入输入模式的提示,先切换到输入模式再插入 - self.scroll_offset = 0; - self.state = UiState::WaitingInput { - prompt: self.take_waiting_input_prompt(), - current_input: String::new(), - cursor_pos: 0, - }; - self.insert_text_at_cursor(text) - } else { - self.scroll_offset = 0; - // 不在输入模式时,可以显示一个简短的通知 - let preview = if text.chars().count() > 20 { - format!("{}...", text.chars().take(17).collect::()) - } else { - text.to_string() - }; - self.show_notification(NotificationLevel::Info.to_color(), &preview) - } - } - - /// 获取当前状态 - pub fn state(&self) -> &UiState { - &self.state - } - - pub fn set_status(&mut self, status: &str) -> anyhow::Result<()> { - self.status_bar = status.to_string(); - match &self.state { - UiState::WaitingInput { .. } => self.refresh_input_display(), - _ => Ok(()), - } + pub fn clear_input(&mut self) -> anyhow::Result<()> { + self.input_mode = false; + self.asr_input.clear(); + self.asr_cursor_pos = 0; + self.refresh_input_display()?; + Ok(()) } - fn take_waiting_input_prompt(&mut self) -> String { - std::mem::take(&mut self.waiting_input_prompt) - } + pub fn take_waiting_input_prompt(&mut self) -> String { + self.asr_cursor_pos = 0; + self.input_mode = false; - /// 清屏并重置到空闲状态 - pub fn clear(&mut self) -> anyhow::Result<()> { - self.state = UiState::Idle; - self.display.fill_color(ColorFormat::CSS_WHITE)?; - self.display.flush()?; - Ok(()) + std::mem::take(&mut self.asr_input) } // ========== 辅助方法 ========== @@ -1673,7 +1022,6 @@ impl UI { textbox_style, ) .add_plugin(crate::ansi_plugin::MyAnsiPlugin::new()) - .set_vertical_offset(self.scroll_offset) .draw(&mut self.display)?; Ok(()) @@ -1698,6 +1046,7 @@ impl From for UiMessage { UiMessage::ScreenImage { data: data.data, format, + is_last: data.is_last, } } crate::protocol::ServerMessage::Notification(data) => { @@ -1724,15 +1073,10 @@ impl From for UiMessage { title: data.title, } } - crate::protocol::ServerMessage::GetInput(data) => UiMessage::GetInput { - prompt: data.prompt, - }, - crate::protocol::ServerMessage::Choices(data) => UiMessage::Choices { - id: data.id.unwrap_or_default(), - title: data.title, - options: data.options, - multi_select: data.multi_select, - allow_custom_input: data.allow_custom_input, + crate::protocol::ServerMessage::Title(text) => UiMessage::Notification { + color: NotificationLevel::Info.to_color(), + message: text, + title: None, }, crate::protocol::ServerMessage::AsrResult(text) => UiMessage::AsrResult(text), crate::protocol::ServerMessage::PtyOutput(_) => { @@ -1746,7 +1090,6 @@ impl From for UiMessage { } } } - crate::protocol::ServerMessage::Status(s) => UiMessage::Status(s), } } } diff --git a/src/main.rs b/src/main.rs index f0982ce..5519442 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,6 @@ use std::sync::{Arc, Mutex}; -use embedded_graphics::prelude::RgbColor; +use embedded_graphics::prelude::WebColors; use esp_idf_svc::hal::gpio::{AnyIOPin, PinDriver}; use crate::lcd::DisplayTargetDrive; @@ -107,7 +107,7 @@ fn main() -> anyhow::Result<()> { peripherals.pins.gpio14, )?; - let mut target = lcd::FrameBuffer::new(lcd::ColorFormat::WHITE); + let mut target = lcd::FrameBuffer::new(lcd::ColorFormat::CSS_BLACK); target.flush()?; lcd::display_text(&mut target, "VibeKeys Starting...\n Read setting", 0)?; @@ -181,9 +181,19 @@ fn main() -> anyhow::Result<()> { esp_idf_svc::hal::gpio::InterruptType::AnyEdge, )?; - let nvs = esp_idf_svc::nvs::EspDefaultNvs::new(partition, "setting", true)?; + let mut nvs = esp_idf_svc::nvs::EspDefaultNvs::new(partition, "setting", true)?; + + if btn3.is_low() { + lcd::display_text(&mut target, "Clear all config", 0)?; + bt_wifi_mode::Setting::clear_nvs(&mut nvs)?; + bt_keyboard_mode::KeymapConfig::clear_nvs(&mut nvs)?; + std::thread::sleep(std::time::Duration::from_secs(1)); + } let setting = bt_wifi_mode::Setting::load_from_nvs(&nvs)?; + // Load keymap config before moving nvs + let mut keymap = bt_keyboard_mode::KeymapConfig::load_from_nvs(&nvs)?; + log::info!("Loaded keymap config with {} keys", keymap.keys.len()); let mut wifi = esp_idf_svc::wifi::EspWifi::new(peripherals.modem, sysloop.clone(), None)?; let mac = wifi.sta_netif().get_mac().unwrap(); @@ -233,6 +243,15 @@ fn main() -> anyhow::Result<()> { ota.mark_running_slot_valid()?; } + if mode == 1 && setting.need_init() { + lcd::display_text( + &mut target, + "Remote Control mode requires network/server config", + 0, + )?; + std::thread::sleep(std::time::Duration::from_secs(1)); + } + if mode == 3 || setting.need_init() { lcd::display_text(&mut target, "Starting in keyboard mode...", 0)?; std::thread::sleep(std::time::Duration::from_secs(1)); @@ -240,10 +259,6 @@ fn main() -> anyhow::Result<()> { let (tx, rx) = tokio::sync::mpsc::channel(64); let (setting_tx, setting_rx) = tokio::sync::mpsc::channel(8); - // Load keymap config before moving nvs - let mut keymap = bt_keyboard_mode::KeymapConfig::load_from_nvs(&nvs)?; - log::info!("Loaded keymap config with {} keys", keymap.keys.len()); - let mut setting_arc = Arc::new(Mutex::new((setting.clone(), nvs))); esp32_nimble::BLEDevice::set_device_name("VibeKeys-MAX")?; @@ -415,7 +430,7 @@ fn main() -> anyhow::Result<()> { let mut ui = lcd::UI::new_with_target(target); - let app_fut = app::run(setting.server_url, &mut ui, rx); + let app_fut = app::run(setting.server_url, &mut ui, rx, &keymap); let r = runtime.block_on(app_fut); if let Err(e) = r { log::error!("App error: {:?}", e); @@ -610,7 +625,7 @@ pub fn handle_key_event( KeysPin::ACCEPT => { keyboard.press(b'\n'); } - KeysPin::ROTATE_BUTTON => keyboard.press(b'\n'), + KeysPin::ROTATE_BUTTON => keyboard.write("/"), _ => {} } } diff --git a/src/protocol.rs b/src/protocol.rs index fd09fbc..134511d 100644 --- a/src/protocol.rs +++ b/src/protocol.rs @@ -32,19 +32,11 @@ pub enum ClientMessage { #[serde(rename = "input_text")] Input(String), - /// 客户端选择 - #[serde(rename = "choice")] - Choice { - /// 选项索引(choices 数组的索引) - index: i32, - }, - - #[serde(rename = "choices")] - Choices { - index: Vec, - custom_input: Option, - multi_select: bool, - }, + #[serde(rename = "scroll_up")] + ScrollUp, + + #[serde(rename = "scroll_down")] + ScrollDown, /// 切换工作目录 #[serde(rename = "change_dir")] @@ -68,19 +60,8 @@ impl Debug for ClientMessage { .finish(), ClientMessage::VoiceInputEnd(_) => f.debug_tuple("VoiceInputEnd").finish(), ClientMessage::Input(text) => f.debug_tuple("Input").field(text).finish(), - ClientMessage::Choice { index } => { - f.debug_struct("Choice").field("index", index).finish() - } - ClientMessage::Choices { - index, - custom_input, - multi_select, - } => f - .debug_struct("Choices") - .field("index", index) - .field("custom_input", custom_input) - .field("multi_select", multi_select) - .finish(), + ClientMessage::ScrollUp => f.debug_tuple("ScrollUp").finish(), + ClientMessage::ScrollDown => f.debug_tuple("ScrollDown").finish(), ClientMessage::ChangeDir(path) => f.debug_tuple("ChangeDir").field(path).finish(), } } @@ -108,39 +89,34 @@ pub enum ServerMessage { #[serde(rename = "pty_out")] PtyOutput(Vec), - /// 屏幕显示图片 + /// 屏幕显示图片(分片) #[serde(rename = "screen_image")] - ScreenImage(ScreenImageData), + ScreenImage(ScreenImageChunk), /// 通知消息 #[serde(rename = "notification")] Notification(NotificationData), - /// 请求输入 - #[serde(rename = "get_input")] - GetInput(GetInputData), - /// ASR 结果 #[serde(rename = "asr_result")] AsrResult(String), - /// 提供选择项 - #[serde(rename = "choices")] - Choices(ChoicesData), - - /// 设置状态栏 - #[serde(rename = "status")] - Status(String), + /// 终端标题变化 + #[serde(rename = "title")] + Title(String), } -/// 屏幕显示图片数据 +/// 屏幕图片分片数据 #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ScreenImageData { - /// 图片数据(原始字节) - pub data: Vec, - +pub struct ScreenImageChunk { /// 图片格式 pub format: ImageFormat, + + /// 是否为最后一个分片 + pub is_last: bool, + + /// 分片数据 + pub data: Vec, } /// 通知消息数据 @@ -162,33 +138,6 @@ pub struct NotificationData { pub color: u32, } -/// 请求输入数据 -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct GetInputData { - /// 提示语 - pub prompt: String, -} - -/// 提供选择项数据 -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ChoicesData { - /// 工具调用 ID(用于识别是否是同一个 tool 请求) - #[serde(skip_serializing_if = "Option::is_none")] - pub id: Option, - - /// 标题/问题 - pub title: String, - - /// 选择项列表, 如果是空则表示选择 confirm/cancel - pub options: Vec, - - /// 是否允许多选(如果为 true,客户端可以选择多个选项,index 将是一个数组) - pub multi_select: bool, - - /// 是否允许用户输入自定义选项(如果为 true,客户端可以提供一个文本输入,文本输入内容将通过 input 消息发送) - pub allow_custom_input: bool, -} - // ========== 辅助类型 ========== /// 图片格式 @@ -241,11 +190,6 @@ impl ClientMessage { Self::VoiceInputEnd(VoiceInputEnd {}) } - /// 创建客户端选择消息 - pub fn choice(index: i32) -> Self { - Self::Choice { index } - } - /// 创建文本输入消息 pub fn input(text: impl Into) -> Self { Self::Input(text.into()) @@ -267,8 +211,12 @@ impl ServerMessage { } /// 创建屏幕图片消息 - pub fn screen_image(data: Vec, format: ImageFormat) -> Self { - Self::ScreenImage(ScreenImageData { data, format }) + pub fn screen_image_chunk(format: ImageFormat, is_last: bool, data: Vec) -> Self { + Self::ScreenImage(ScreenImageChunk { + format, + is_last, + data, + }) } /// 创建通知消息 @@ -295,54 +243,14 @@ impl ServerMessage { }) } - /// 创建请求输入消息 - pub fn get_input(prompt: impl Into) -> Self { - Self::GetInput(GetInputData { - prompt: prompt.into(), - }) - } - - /// 创建提供选择项消息 - pub fn choices( - title: impl Into, - options: Vec, - multi_select: bool, - allow_custom_input: bool, - ) -> Self { - Self::Choices(ChoicesData { - id: None, - title: title.into(), - options, - multi_select, - allow_custom_input, - }) - } - - /// 创建提供选择项消息(带 ID) - pub fn choices_with_id( - id: impl Into, - title: impl Into, - options: Vec, - multi_select: bool, - allow_custom_input: bool, - ) -> Self { - Self::Choices(ChoicesData { - id: Some(id.into()), - title: title.into(), - options, - multi_select, - allow_custom_input, - }) - } - /// 创建 ASR 结果消息 pub fn asr_result(text: impl Into) -> Self { Self::AsrResult(text.into()) } /// 创建状态栏消息 - pub fn status(text: impl Into) -> Self { - Self::Status(text.into()) + pub fn title(text: impl Into) -> Self { + Self::Title(text.into()) } } @@ -396,6 +304,7 @@ impl ServerMessage { #[cfg(test)] mod tests { + use super::*; #[test] fn test_client_pty_input_msgpack() { @@ -447,33 +356,6 @@ mod tests { } } - #[test] - fn test_client_choice_json() { - let msg = ClientMessage::choice(2); - let json = msg.to_json().unwrap(); - println!("JSON: {}", json); - let decoded = ClientMessage::from_json(&json).unwrap(); - match decoded { - ClientMessage::Choice { index } => { - assert_eq!(index, 2); - } - _ => panic!("Wrong message type"), - } - } - - #[test] - fn test_client_choice_msgpack() { - let msg = ClientMessage::choice(2); - let bytes = msg.to_msgpack().unwrap(); - let decoded = ClientMessage::from_msgpack(&bytes).unwrap(); - match decoded { - ClientMessage::Choice { index } => { - assert_eq!(index, 2); - } - _ => panic!("Wrong message type"), - } - } - #[test] fn test_client_input_msgpack() { let msg = ClientMessage::input("Hello, world!"); @@ -501,52 +383,13 @@ mod tests { } } - #[test] - fn test_server_get_input_msgpack() { - let msg = ServerMessage::get_input("请说话"); - let bytes = msg.to_msgpack().unwrap(); - let decoded = ServerMessage::from_msgpack(&bytes).unwrap(); - match decoded { - ServerMessage::GetInput(data) => { - assert_eq!(data.prompt, "请说话"); - } - _ => panic!("Wrong message type"), - } - } - - #[test] - fn test_server_choices_msgpack() { - let msg = ServerMessage::choices( - "请选择", - vec![ - "选项A".to_string(), - "选项B".to_string(), - "选项C".to_string(), - ], - false, - false, - ); - let bytes = msg.to_msgpack().unwrap(); - let decoded = ServerMessage::from_msgpack(&bytes).unwrap(); - match decoded { - ServerMessage::Choices(data) => { - assert_eq!(data.title, "请选择"); - assert_eq!(data.options.len(), 3); - assert_eq!(data.options[0], "选项A"); - assert_eq!(data.options[1], "选项B"); - assert_eq!(data.options[2], "选项C"); - } - _ => panic!("Wrong message type"), - } - } - #[test] fn test_server_status_msgpack() { - let msg = ServerMessage::status("Connected"); + let msg = ServerMessage::title("Connected"); let bytes = msg.to_msgpack().unwrap(); let decoded = ServerMessage::from_msgpack(&bytes).unwrap(); match decoded { - ServerMessage::Status(text) => { + ServerMessage::Title(text) => { assert_eq!(text, "Connected"); } _ => panic!("Wrong message type"), @@ -623,29 +466,14 @@ mod tests { } } - #[test] - fn test_server_choices_json() { - let msg = ServerMessage::choices("请选择", vec!["A".into(), "B".into()], false, false); - let json = msg.to_json().unwrap(); - println!("JSON: {}", json); - let decoded = ServerMessage::from_json(&json).unwrap(); - match decoded { - ServerMessage::Choices(data) => { - assert_eq!(data.title, "请选择"); - assert_eq!(data.options, vec!["A", "B"]); - } - _ => panic!("Wrong message type"), - } - } - #[test] fn test_server_status_json() { - let msg = ServerMessage::status("Ready"); + let msg = ServerMessage::title("Ready"); let json = msg.to_json().unwrap(); println!("JSON: {}", json); let decoded = ServerMessage::from_json(&json).unwrap(); match decoded { - ServerMessage::Status(text) => { + ServerMessage::Title(text) => { assert_eq!(text, "Ready"); } _ => panic!("Wrong message type"), diff --git a/src/ws.rs b/src/ws.rs index 6211bc1..75d5e7c 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -11,7 +11,7 @@ pub struct Server { impl Server { pub async fn new(uri: String) -> anyhow::Result { - let uri = format!("{}", uri); + let uri = format!("{}?pty=false&img=true&width=288&height=80", uri); let (ws, _resp) = tokio_websockets::ClientBuilder::new() .uri(&uri)? .connect() @@ -85,6 +85,8 @@ impl Server { } }; return Some(msg); + } else if msg.is_ping() { + // ignore ping frames } else { log::warn!("Received unsupported message type: {:?}", msg); continue;