diff --git a/Cargo.lock b/Cargo.lock index 1e51372..df7fd5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -400,15 +400,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "defmt" -version = "0.3.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0963443817029b2024136fc4dd07a5107eb8f977eaf18fcd1fdeb11306b64ad" -dependencies = [ - "defmt 1.0.1", -] - [[package]] name = "defmt" version = "1.0.1" @@ -455,15 +446,15 @@ checksum = "dc2d050bdc5c21e0862a89256ed8029ae6c290a93aecefc73084b3002cdebb01" [[package]] name = "embassy-sync" -version = "0.6.2" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d2c8cdff05a7a51ba0087489ea44b0b1d97a296ca6b1d6d1a33ea7423d34049" +checksum = "73974a3edbd0bd286759b3d483540f0ebef705919a5f56f4fc7709066f71689b" dependencies = [ "cfg-if", "critical-section", - "embedded-io-async", + "embedded-io-async 0.6.1", + "futures-core", "futures-sink", - "futures-util", "heapless 0.8.0", ] @@ -540,31 +531,46 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" +[[package]] +name = "embedded-io" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eb1aa714776b75c7e67e1da744b81a129b3ff919c8712b5e1b32252c1f07cc7" + [[package]] name = "embedded-io-async" version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ff09972d4073aa8c299395be75161d582e7629cd663171d62af73c8d50dba3f" dependencies = [ - "embedded-io", + "embedded-io 0.6.1", +] + +[[package]] +name = "embedded-io-async" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2564b9f813c544241430e147d8bc454815ef9ac998878d30cc3055449f7fd4c0" +dependencies = [ + "embedded-io 0.7.1", ] [[package]] name = "embedded-svc" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7770e30ab55cfbf954c00019522490d6ce26a3334bede05a732ba61010e98e0" +checksum = "8bfc6d05bac4af70b0795d1f1b6ddd44aa85ad04b05d34d64b1b9fce36de107e" dependencies = [ - "defmt 0.3.100", - "embedded-io", - "embedded-io-async", + "defmt", + "embedded-io 0.7.1", + "embedded-io-async 0.7.0", "enumset", - "heapless 0.8.0", + "heapless 0.9.3", "log", "num_enum", "serde", - "strum 0.25.0", - "strum_macros 0.25.3", + "strum 0.27.2", + "strum_macros 0.27.2", ] [[package]] @@ -653,9 +659,9 @@ dependencies = [ [[package]] name = "esp-idf-hal" -version = "0.45.2" +version = "0.46.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775ce25171dc4f615146a4a27ed3a64c6fd99ced77d7112062f2b19bf933f5db" +checksum = "2585c80f488be431ad14e199883a05ff31576adaa23014befa0f040fd91490b8" dependencies = [ "atomic-waker", "embassy-sync", @@ -664,22 +670,21 @@ dependencies = [ "embedded-hal 1.0.0", "embedded-hal-async", "embedded-hal-nb", - "embedded-io", - "embedded-io-async", + "embedded-io 0.7.1", + "embedded-io-async 0.7.0", "embuild", "enumset", "esp-idf-sys", - "heapless 0.8.0", + "heapless 0.9.3", "log", "nb 1.1.0", - "num_enum", ] [[package]] name = "esp-idf-svc" -version = "0.51.0" +version = "0.52.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bc07aaba257d28d54a96af005ca67d0b38876d8837f5d54a3e0547e100b219c" +checksum = "033432e951f49284a4b8874cdb2d7fb940a3b0181460f452ca7414ebe66babca" dependencies = [ "embassy-futures", "embedded-hal-async", @@ -688,7 +693,7 @@ dependencies = [ "enumset", "esp-idf-hal", "futures-io", - "heapless 0.8.0", + "heapless 0.9.3", "log", "num_enum", "uncased", @@ -696,9 +701,9 @@ dependencies = [ [[package]] name = "esp-idf-sys" -version = "0.36.1" +version = "0.37.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb77a3d02b579a60a811ed9be22b78c5e794bc492d833ee7fc44d3a0155885e1" +checksum = "f8df62993f242eb05d9ddb63e503624fb22f24a11d6c680e85cf0169ee14421f" dependencies = [ "anyhow", "build-time", @@ -716,9 +721,9 @@ dependencies = [ [[package]] name = "esp32-nimble" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053db9052cf401761b4b42af928e83bbe86542941c82a9ec4622813be53759b8" +checksum = "aabfc9f595c551d3270ab2a5feca5f5b961863bcc40bb5549a8f9b4154fd8997" dependencies = [ "anyhow", "bitflags 2.11.1", @@ -1019,7 +1024,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" dependencies = [ "hash32 0.3.1", - "serde", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4" +dependencies = [ + "hash32 0.3.1", + "serde_core", "stable_deref_trait", ] @@ -1758,11 +1773,11 @@ dependencies = [ [[package]] name = "strum" -version = "0.25.0" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros 0.25.3", + "strum_macros 0.27.2", ] [[package]] @@ -1780,14 +1795,13 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.25.3" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", - "rustversion", "syn 2.0.117", ] @@ -2036,7 +2050,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "vibekeys" -version = "0.2.3" +version = "0.3.0" dependencies = [ "ansi-parser 0.9.1", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 86ece1a..3f10574 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vibekeys" -version = "0.2.4" +version = "0.3.0" authors = ["csh <458761603@qq.com>"] edition = "2021" resolver = "2" @@ -26,6 +26,7 @@ opt-level = "z" [features] default = [] max2 = [] +i2c_oled = [] experimental = ["esp-idf-svc/experimental"] @@ -33,9 +34,9 @@ experimental = ["esp-idf-svc/experimental"] log = "0.4" anyhow = "1.0" -esp-idf-svc = "0.51" -esp32-nimble = "0.11.1" -embedded-svc = "0.28.1" +esp-idf-svc = "0.52.1" +esp32-nimble = "0.12.0" +embedded-svc = "0.29.0" zerocopy = "0.8.33" diff --git a/sdkconfig.defaults b/sdkconfig.defaults index 611ed62..1908cce 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -10,8 +10,13 @@ CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y CONFIG_ESP32S3_DEFAULT_CPU_FREQ_MHZ=240 # Workaround for https://github.com/espressif/esp-idf/issues/7631 -#CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=n -#CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL=n +CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y +CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL=y +CONFIG_MBEDTLS_TLS_VERSION_1_2=y +CONFIG_MBEDTLS_TLS_VERSION_1_3=y +CONFIG_MBEDTLS_EXTERNAL_MEM_ALLOC=y +CONFIG_MBEDTLS_DYNAMIC_BUFFER=y + CONFIG_I2C_SKIP_LEGACY_CONFLICT_CHECK=y diff --git a/src/audio.rs b/src/audio.rs index 2b3db54..9e410a9 100644 --- a/src/audio.rs +++ b/src/audio.rs @@ -1,11 +1,11 @@ use std::sync::Arc; use esp_idf_svc::hal::gpio::AnyIOPin; -use esp_idf_svc::hal::i2s::{config, I2sDriver, I2S0}; +use esp_idf_svc::hal::i2s::{config, I2sDriver, I2sRx, I2S0}; use esp_idf_svc::sys::esp_sr; -const SAMPLE_RATE: u32 = 16000; +pub const SAMPLE_RATE: u32 = 16000; pub static mut AFE_LINEAR_GAIN: f32 = 1.5; pub static mut AGC_TARGET_LEVEL_DBFS: i32 = 3; @@ -233,11 +233,11 @@ fn audio_task_run( } pub struct AudioWorker { - pub in_i2s: I2S0, - pub in_ws: AnyIOPin, - pub in_clk: AnyIOPin, - pub din: AnyIOPin, - pub in_mclk: Option, + pub in_i2s: I2S0<'static>, + pub in_ws: AnyIOPin<'static>, + pub in_clk: AnyIOPin<'static>, + pub din: AnyIOPin<'static>, + pub in_mclk: Option>, } impl AudioWorker { @@ -293,3 +293,222 @@ impl AudioWorker { audio_task_run(&mut fn_read, afe_handle) } } + +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)] +#[serde(tag = "platform")] +pub enum AsrConfig { + #[serde(alias = "whisper")] + Whisper { + uri: String, + api_key: String, + model: String, + }, +} + +impl AsrConfig { + pub fn from_json(json: &str) -> anyhow::Result { + let config = serde_json::from_str(json)?; + Ok(config) + } + + pub fn load_from_nvs(nvs: &esp_idf_svc::nvs::EspDefaultNvs) -> Option { + let asr_config_len = nvs.str_len("asr_config").ok()??; // Check if the key exists + if asr_config_len == 0 { + return None; // No config stored + } + + let mut buffer = vec![0u8; asr_config_len]; + + let json = nvs.get_str("asr_config", &mut buffer).ok()??; + + Self::from_json(&json).ok() + } + + pub fn save_to_nvs(&self, nvs: &esp_idf_svc::nvs::EspDefaultNvs) -> anyhow::Result<()> { + let json = serde_json::to_string(self)?; + nvs.set_str("asr_config", &json)?; + Ok(()) + } + + pub fn requires_tls(&self) -> bool { + match self { + AsrConfig::Whisper { uri, .. } => uri.starts_with("https://"), + } + } +} + +#[derive(Debug, serde::Deserialize)] +struct AsrResult { + #[serde(default)] + text: String, + #[serde(default)] + error: Option, +} + +impl AsrResult { + fn parse_text(&self) -> String { + if self.text.trim().starts_with("[") { + let mut texts = vec![]; + for line in self.text.lines() { + if let Some((_, t)) = line.split_once("] ") { + texts.push(t.to_string()); + } else { + texts.push(line.to_string()); + } + } + texts.join("\n") + } else { + self.text.clone() + } + } +} + +pub struct Driver(I2sDriver<'static, I2sRx>); + +impl Driver { + pub fn new(worker: AudioWorker) -> anyhow::Result { + let i2s_config = config::StdConfig::new( + config::Config::default() + .auto_clear(true) + .dma_buffer_count(2) + .frames_per_buffer(512), + config::StdClkConfig::from_sample_rate_hz(SAMPLE_RATE), + config::StdSlotConfig::philips_slot_default( + config::DataBitWidth::Bits16, + config::SlotMode::Mono, + ), + config::StdGpioConfig::default(), + ); + + let mut rx_driver = I2sDriver::new_std_rx( + worker.in_i2s, + &i2s_config, + worker.in_clk, + worker.din, + worker.in_mclk, + worker.in_ws, + ) + .map_err(|e| anyhow::anyhow!("Error create RX: {:?}", e))?; + rx_driver.rx_enable()?; + + Ok(Self(rx_driver)) + } + + pub fn read(&mut self, buffer: &mut [u8]) -> anyhow::Result { + let len = self + .0 + .read(buffer, esp_idf_svc::hal::delay::TickType::new_millis(100).0)?; + + Ok(len) + } + + pub fn start_whisper( + &mut self, + uri: &str, + api_key: &str, + model: &str, + mut on_start_listen: impl FnMut(), + is_stop: impl Fn() -> bool, + ) -> anyhow::Result { + let config = esp_idf_svc::http::client::Configuration { + crt_bundle_attach: Some(esp_idf_svc::sys::esp_crt_bundle_attach), + ..Default::default() + }; + let conn = esp_idf_svc::http::client::EspHttpConnection::new(&config)?; + let mut client = embedded_svc::http::client::Client::wrap(conn); + + // 手动构造 multipart + let boundary = "----WebKitFormBoundary7MA4YWxkTrZu0gW"; + let content_type = format!("multipart/form-data; boundary={}", boundary); + + let header_value = format!("Bearer {}", api_key); + + let headers = [ + ("Content-Type", content_type.as_str()), + ("Authorization", header_value.as_str()), + ]; + let mut req: esp_idf_svc::http::client::Request< + &mut esp_idf_svc::http::client::EspHttpConnection, + > = client.post( + uri, + if api_key.is_empty() { + &headers[..1] + } else { + &headers + }, + )?; + + // 写 multipart 头部 + let header = format!( + "--{}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\nContent-Type: audio/wav\r\n\r\n", + boundary + ); + req.write(header.as_bytes())?; + + let wav_header = crate::util::create_unlimited_wav_header(&crate::util::WavConfig { + sample_rate: SAMPLE_RATE, + channels: 1, + bits_per_sample: 16, + }); + req.write(&wav_header)?; + + on_start_listen(); + + // 边录边写音频数据 + let mut buffer = vec![0u8; 2 * SAMPLE_RATE as usize / 10]; + let max_chunks = 10 * 30; // 30s + + for _ in 0..max_chunks { + if is_stop() { + break; + } + let len = self.read(&mut buffer)?; + if len > 0 { + let n = req.write(&buffer[..len])?; + log::debug!("Wrote {} bytes of audio data", n); + } + } + + // 写 model 字段 + let model_field = format!( + "\r\n--{}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n{}\r\n", + boundary, model + ); + req.write(model_field.as_bytes())?; + + // 写结束标记 + let footer = format!("--{}--", boundary); + req.write(footer.as_bytes())?; + req.flush()?; + let mut resp = req.submit()?; + // buffer.clear(); + log::info!("resp code: {}", resp.status()); + let bytes_read = + embedded_svc::utils::io::try_read_full(&mut resp, &mut buffer).map_err(|e| e.0)?; + let resp_body = std::str::from_utf8(&buffer[0..bytes_read])?; + let asr_result: AsrResult = serde_json::from_str(resp_body)?; + log::info!("{asr_result:?}"); + if let Some(ref e) = asr_result.error { + log::error!("error: {}", serde_json::to_string(e).unwrap()) + } + + // let v = serde_json::from_str::(resp_body)?; + + Ok(asr_result.parse_text()) + } + + pub fn start_asr bool, F2: FnMut()>( + &mut self, + asr_config: &AsrConfig, + on_start_listen: F2, + is_stop: F, + ) -> anyhow::Result { + match asr_config { + AsrConfig::Whisper { + uri, + api_key, + model, + } => self.start_whisper(uri, api_key, model, on_start_listen, is_stop), + } + } +} diff --git a/src/bt_keyboard_mode.rs b/src/bt_keyboard_mode.rs index 5e47c43..86f0ef1 100644 --- a/src/bt_keyboard_mode.rs +++ b/src/bt_keyboard_mode.rs @@ -78,13 +78,12 @@ impl KeymapConfig { } pub fn load_from_nvs(nvs: &esp_idf_svc::nvs::EspDefaultNvs) -> anyhow::Result { - if !nvs.contains("keymap_config")? { - log::info!("No keymap config found in NVS, using default"); - return Ok(Self::default()); - } - let keymap_size = nvs.blob_len("keymap_config")?.unwrap_or_default(); log::info!("Keymap config size in NVS: {} bytes", keymap_size); + if keymap_size == 0 { + log::warn!("Keymap config blob is empty, using default"); + return Ok(Self::default()); + } let mut buf = vec![0; keymap_size]; nvs.get_blob("keymap_config", &mut buf)?; @@ -559,6 +558,7 @@ pub async fn key_event( pub async fn wait_key_event(key_pins: &mut KeysPin) -> ControllerCommand { tokio::select! { _ = key_pins.mic.wait_for_any_edge() => { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; // Debounce delay if key_pins.mic.is_low() { ControllerCommand::KeyboardPress(KeysPin::MIC) } else { @@ -566,6 +566,7 @@ pub async fn wait_key_event(key_pins: &mut KeysPin) -> ControllerCommand { } }, _ = key_pins.custom.wait_for_any_edge() => { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; // Debounce delay if key_pins.custom.is_low() { ControllerCommand::KeyboardPress(KeysPin::CUSTOM) } else { @@ -573,6 +574,7 @@ pub async fn wait_key_event(key_pins: &mut KeysPin) -> ControllerCommand { } } _ = key_pins.esc.wait_for_any_edge() => { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; // Debounce delay if key_pins.esc.is_low() { log::info!("ESC key pressed"); ControllerCommand::KeyboardPress(KeysPin::ESC) @@ -582,6 +584,7 @@ pub async fn wait_key_event(key_pins: &mut KeysPin) -> ControllerCommand { } }, _ = key_pins.next.wait_for_any_edge() => { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; // Debounce delay if key_pins.next.is_low() { ControllerCommand::KeyboardPress(KeysPin::NEXT) } else { @@ -589,6 +592,7 @@ pub async fn wait_key_event(key_pins: &mut KeysPin) -> ControllerCommand { } }, _ = key_pins.switch.wait_for_any_edge() => { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; // Debounce delay if key_pins.switch.is_low() { ControllerCommand::KeyboardPress(KeysPin::SWITCH) } else { @@ -596,6 +600,7 @@ pub async fn wait_key_event(key_pins: &mut KeysPin) -> ControllerCommand { } }, _ = key_pins.backspace.wait_for_any_edge() => { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; // Debounce delay if key_pins.backspace.is_low() { ControllerCommand::KeyboardPress(KeysPin::BACKSPACE) } else { @@ -603,6 +608,7 @@ pub async fn wait_key_event(key_pins: &mut KeysPin) -> ControllerCommand { } }, _ = key_pins.accept.wait_for_any_edge() => { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; // Debounce delay if key_pins.accept.is_low() { ControllerCommand::KeyboardPress(KeysPin::ACCEPT) } else { @@ -625,6 +631,7 @@ pub async fn wait_key_event(key_pins: &mut KeysPin) -> ControllerCommand { } }, _ = key_pins.rotate_button.wait_for_any_edge() => { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; // Debounce delay if key_pins.rotate_button.is_low() { ControllerCommand::KeyboardPress(KeysPin::ROTATE_BUTTON) } else { @@ -787,7 +794,7 @@ impl KeyboardAndMouse { hid.report_map(HID_REPORT_DISCRIPTOR); - hid.set_battery_level(battery_level); + // hid.set_battery_level(battery_level); let hid_service_id = hid.hid_service().lock().uuid(); @@ -933,9 +940,12 @@ const CONTROLLER_SERVICE_ID: BleUuid = super::bt_wifi_mode::SERVICE_ID; const KEYBOARD_DISPLAY_ID: BleUuid = uuid128!("cdaa6472-67a8-4241-93cf-145051608573"); const KEYBOARD_NOTIFY_ID: BleUuid = uuid128!("d4f7e1b3-3c4d-4f4e-8e2a-8f4e5c6d7e8f"); const KEYMAP_CONFIG_ID: BleUuid = uuid128!("6f2a291c-0e4d-4f0f-9446-50bcd0b73bb0"); +const KEYMAP_ASR_RESULT_ID: BleUuid = uuid128!("f67f3c25-c9f0-456e-955e-cd9d9dd91051"); +const KEYMAP_ASR_CONFIG_ID: BleUuid = uuid128!("faf9e22c-e8fc-421b-afef-8b5236813fb1"); pub struct ControllerService { pub notify_characteristic: Arc>, + pub paster_characteristic: Arc>, } impl ControllerService { @@ -945,6 +955,13 @@ impl ControllerService { .set_value(message.as_bytes()) .notify(); } + + pub fn notify_asr(&self, message: &str) { + self.paster_characteristic + .lock() + .set_value(message.as_bytes()) + .notify(); + } } #[derive(Debug)] @@ -955,6 +972,8 @@ pub enum ControllerCommand { RotateDown, RotateUp, KeymapConfig(String), + Paste(u8), + AsrConfig(String), } pub fn new_controller_service( @@ -975,23 +994,52 @@ pub fn new_controller_service( let _ = tx_.blocking_send(ControllerCommand::DisplayKeyboard(s)); }); + let paster_characteristic = service.create_characteristic( + KEYMAP_ASR_RESULT_ID, + NimbleProperties::WRITE | NimbleProperties::NOTIFY, + ); + + let tx_ = tx.clone(); + paster_characteristic.lock().on_write(move |args| { + let data = args.recv_data(); + if !data.is_empty() { + let _ = tx_.blocking_send(ControllerCommand::Paste(data[0])); + } + }); + let notify_characteristic = service.create_characteristic(KEYBOARD_NOTIFY_ID, NimbleProperties::NOTIFY); let keymap_config_characteristic = service.create_characteristic(KEYMAP_CONFIG_ID, NimbleProperties::WRITE); + let tx_ = tx.clone(); keymap_config_characteristic.lock().on_write(move |args| { log::info!("Wrote to keymap config characteristic"); let data = args.recv_data(); log::info!("Received keymap data: {:?}", data); let s = String::from_utf8_lossy(&data).to_string(); - let _ = tx.blocking_send(ControllerCommand::KeymapConfig(s)); + let _ = tx_.blocking_send(ControllerCommand::KeymapConfig(s)); }); + let keymap_asr_config_characteristic = + service.create_characteristic(KEYMAP_ASR_CONFIG_ID, NimbleProperties::WRITE); + + keymap_asr_config_characteristic + .lock() + .on_write(move |args| { + log::info!("Wrote to ASR config characteristic"); + let data = args.recv_data(); + log::info!("Received ASR config data: {:?}", data); + let s = String::from_utf8_lossy(&data).to_string(); + + let _ = tx.blocking_send(ControllerCommand::AsrConfig(s)); + }); + Ok(ControllerService { notify_characteristic, + paster_characteristic, }) } diff --git a/src/bt_wifi_mode.rs b/src/bt_wifi_mode.rs index be86c85..59fed82 100644 --- a/src/bt_wifi_mode.rs +++ b/src/bt_wifi_mode.rs @@ -64,14 +64,14 @@ impl Setting { .unwrap_or_default() .to_string(); - let background_png = if nvs.contains("background_png")? { - let background_png_size = nvs - .blob_len("background_png") - .map_err(|e| log::error!("Failed to get background_png size: {:?}", e)) - .ok() - .flatten() - .unwrap_or(1024 * 1024); + let background_png_size = nvs + .blob_len("background_png") + .map_err(|e| log::error!("Failed to get background_png size: {:?}", e)) + .ok() + .flatten() + .unwrap_or(0); + let background_png = if background_png_size != 0 { log::info!("Background PNG size in NVS: {} bytes", background_png_size); let mut png_buf = vec![0; background_png_size]; diff --git a/src/i2c/mod.rs b/src/i2c/mod.rs index 4b07eff..5fc7906 100644 --- a/src/i2c/mod.rs +++ b/src/i2c/mod.rs @@ -26,8 +26,8 @@ pub fn i2c_init(_i2c: I2C0, sda: Gpio48, scl: Gpio45) -> anyhow::Result<()> { clk_source: soc_periph_i2c_clk_src_t_I2C_CLK_SRC_DEFAULT, }, i2c_port: I2C0::port() as i32, - scl_io_num: scl.pin(), - sda_io_num: sda.pin(), + scl_io_num: scl.pin() as _, + sda_io_num: sda.pin() as _, glitch_ignore_cnt: 7, flags, intr_priority: 0, diff --git a/src/lcd.rs b/src/lcd.rs index 0a428f0..b6649da 100644 --- a/src/lcd.rs +++ b/src/lcd.rs @@ -39,12 +39,12 @@ pub fn init_spi(_spi: SPI3, mosi: Gpio21, clk: Gpio47) -> Result<(), EspError> { const GPIO_NUM_NC: i32 = -1; let mut buscfg = spi_bus_config_t::default(); - buscfg.__bindgen_anon_1.mosi_io_num = mosi.pin(); + buscfg.__bindgen_anon_1.mosi_io_num = mosi.pin() as _; buscfg.__bindgen_anon_2.miso_io_num = GPIO_NUM_NC; - buscfg.sclk_io_num = clk.pin(); + buscfg.sclk_io_num = clk.pin() as _; buscfg.__bindgen_anon_3.quadwp_io_num = GPIO_NUM_NC; buscfg.__bindgen_anon_4.quadhd_io_num = GPIO_NUM_NC; - buscfg.max_transfer_sz = 4096; + buscfg.max_transfer_sz = 1024 * 4; esp!(unsafe { spi_bus_initialize(SPI3::device(), &buscfg, spi_common_dma_t_SPI_DMA_CH_AUTO,) }) } @@ -54,10 +54,10 @@ pub fn init_lcd(cs: Gpio12, dc: Gpio13, rst: Gpio14) -> Result<(), EspError> { ::log::info!("Install panel IO"); let mut panel_io: esp_lcd_panel_io_handle_t = std::ptr::null_mut(); let mut io_config = esp_lcd_panel_io_spi_config_t::default(); - io_config.cs_gpio_num = cs.pin(); - io_config.dc_gpio_num = dc.pin(); + io_config.cs_gpio_num = cs.pin() as _; + io_config.dc_gpio_num = dc.pin() as _; io_config.spi_mode = 3; - io_config.pclk_hz = 40 * 1000 * 1000; + io_config.pclk_hz = 60 * 1000 * 1000; io_config.trans_queue_depth = 10; io_config.lcd_cmd_bits = 8; io_config.lcd_param_bits = 8; @@ -70,7 +70,7 @@ pub fn init_lcd(cs: Gpio12, dc: Gpio13, rst: Gpio14) -> Result<(), EspError> { let mut panel_config = esp_lcd_panel_dev_config_t::default(); let mut panel: esp_lcd_panel_handle_t = std::ptr::null_mut(); - panel_config.reset_gpio_num = rst.pin(); + panel_config.reset_gpio_num = rst.pin() as _; panel_config.data_endian = lcd_rgb_data_endian_t_LCD_RGB_DATA_ENDIAN_LITTLE; panel_config.__bindgen_anon_1.rgb_ele_order = lcd_rgb_element_order_t_LCD_RGB_ELEMENT_ORDER_RGB; panel_config.bits_per_pixel = 16; @@ -334,9 +334,27 @@ impl DisplayTargetDrive for FrameBuffer { let x_end = bounding_box.top_left.x + bounding_box.size.width as i32; let y_end = bounding_box.top_left.y + bounding_box.size.height as i32; - let e = flush_display(self.buffers.data(), x_start, y_start, x_end, y_end); - if e != 0 { - return Err(anyhow::anyhow!("Failed to flush display: error code {}", e)); + for i in 0..5 { + let e = flush_display(self.buffers.data(), x_start, y_start, x_end, y_end); + if e != 0 { + std::thread::sleep(std::time::Duration::from_millis(100)); + crate::log_heap(); + if i < 4 { + log::warn!( + "flush_display failed (attempt {}), retrying... error code: {}", + i + 1, + e + ); + } else { + log::error!( + "flush_display failed after {} attempts. error code: {}", + i + 1, + e + ); + anyhow::bail!("Failed to flush display after multiple attempts"); + } + continue; + } } self.buffers.clone_from(&self.background_buffers); diff --git a/src/main.rs b/src/main.rs index ec0409a..fdc0295 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,36 +10,38 @@ mod app; mod audio; mod bt_keyboard_mode; mod bt_wifi_mode; +#[cfg(feature = "i2c_oled")] mod i2c; mod lcd; mod protocol; +mod util; mod wifi; mod ws; -type AnyBtn = PinDriver<'static, esp_idf_svc::hal::gpio::AnyIOPin, esp_idf_svc::hal::gpio::Input>; +type AnyBtn = PinDriver<'static, esp_idf_svc::hal::gpio::Input>; fn new_btn( - pin: AnyIOPin, + pin: AnyIOPin<'static>, pull: esp_idf_svc::hal::gpio::Pull, interrupt: esp_idf_svc::hal::gpio::InterruptType, ) -> anyhow::Result { - let mut btn = PinDriver::input(pin)?; - btn.set_pull(pull)?; + let mut btn = PinDriver::input(pin, pull)?; btn.set_interrupt_type(interrupt)?; Ok(btn) } const DEFAULT_SNTP_SERVERS: [&str; 4] = [ - "pool.ntp.org", "time.apple.com", "time.windows.com", "time.google.com", + "pool.ntp.org", ]; pub fn sync_time(display_target: &mut lcd::FrameBuffer) -> anyhow::Result<()> { use esp_idf_svc::sntp::{EspSntp, OperatingMode, SntpConf, SyncMode, SyncStatus}; for i in 0..DEFAULT_SNTP_SERVERS.len() { + log_heap(); log::info!("SNTP sync time with server: {}", DEFAULT_SNTP_SERVERS[i]); lcd::display_text( display_target, @@ -54,10 +56,12 @@ pub fn sync_time(display_target: &mut lcd::FrameBuffer) -> anyhow::Result<()> { }; let ntp_client = EspSntp::new(&conf)?; - for _ in 0..30 { + for _ in 0..15 { let status = ntp_client.get_sync_status(); log::info!("sntp sync status {:?}", status); + log_heap(); if status == SyncStatus::Completed { + lcd::display_text(display_target, "Syncing time Completed", 0)?; return Ok(()); } std::thread::sleep(std::time::Duration::from_secs(1)); @@ -82,7 +86,7 @@ pub fn goto_next_firmware() -> anyhow::Result<()> { fn main() -> anyhow::Result<()> { esp_idf_svc::sys::link_patches(); esp_idf_svc::log::EspLogger::initialize_default(); - let peripherals = esp_idf_svc::hal::prelude::Peripherals::take().unwrap(); + let peripherals = esp_idf_svc::hal::peripherals::Peripherals::take().unwrap(); let sysloop = esp_idf_svc::eventloop::EspSystemEventLoop::take()?; let _fs = esp_idf_svc::io::vfs::MountedEventfs::mount(20)?; let partition = esp_idf_svc::nvs::EspDefaultNvsPartition::take()?; @@ -198,6 +202,7 @@ fn main() -> anyhow::Result<()> { // Load keymap config before moving nvs let mut keymap = bt_keyboard_mode::KeymapConfig::load_from_nvs(&nvs)?; log::info!("Loaded keymap config with {} keys", keymap.keys.len()); + let asr_config = audio::AsrConfig::load_from_nvs(&nvs); let mut wifi = esp_idf_svc::wifi::EspWifi::new(peripherals.modem, sysloop.clone(), None)?; let mac = wifi.sta_netif().get_mac().unwrap(); @@ -298,12 +303,12 @@ fn main() -> anyhow::Result<()> { let service = server.create_service(bt_wifi_mode::SERVICE_ID); let mut keyboard = bt_keyboard_mode::KeyboardAndMouse::new(ble_device, 100)?; - let service_id = { + let (controller, service_id) = { let mut lock = service.lock(); - bt_keyboard_mode::new_controller_service(&mut lock, tx)?; + let controller = bt_keyboard_mode::new_controller_service(&mut lock, tx)?; // Start setting service bt_wifi_mode::new_setting_service(&mut lock, setting_arc.clone(), Some(setting_tx))?; - lock.uuid() + (controller, lock.uuid()) }; let server = ble_device.get_server(); @@ -326,6 +331,54 @@ fn main() -> anyhow::Result<()> { rotate_button: pin18, }; + let mut driver: Option = None; + + let r = wifi::connect(&mut wifi, &setting.ssid, &setting.pass, sysloop.clone()); + if r.is_err() { + let e = r.err(); + log::error!("Failed to connect to WiFi: {:?}", e); + lcd::display_text( + &mut target, + &format!(" WiFi connection failed: {:?}\n", e), + 0, + )?; + std::thread::sleep(std::time::Duration::from_secs(3)); + } else { + log::info!("WiFi connected successfully"); + log::info!("ASR config loaded from NVS: {:?}", asr_config); + + if let Some(ref asr_config) = asr_config { + if asr_config.requires_tls() { + let r = sync_time(&mut target); + if r.is_err() { + log::error!("Failed to sync time: {:?}", r.err()); + let _ = lcd::display_text(&mut target, " Time sync failed\n", 0); + std::thread::sleep(std::time::Duration::from_secs(3)); + } else { + let worker = audio::AudioWorker { + in_i2s: peripherals.i2s0, + in_ws: peripherals.pins.gpio41.into(), + in_clk: peripherals.pins.gpio42.into(), + din: peripherals.pins.gpio40.into(), + in_mclk: None, + }; + let _ = driver.insert(audio::Driver::new(worker)?); + } + } else { + let worker = audio::AudioWorker { + in_i2s: peripherals.i2s0, + in_ws: peripherals.pins.gpio41.into(), + in_clk: peripherals.pins.gpio42.into(), + din: peripherals.pins.gpio40.into(), + in_mclk: None, + }; + let _ = driver.insert(audio::Driver::new(worker)?); + } + } + } + + log_heap(); + std::thread::sleep(std::time::Duration::from_millis(500)); lcd::display_text(&mut target, "Keyboard Mode", 0)?; runtime.block_on(keyboard_mode_main( @@ -337,6 +390,9 @@ fn main() -> anyhow::Result<()> { setting_rx, rx, &mut keymap, + driver, + asr_config, + controller, )); } @@ -517,6 +573,35 @@ fn handle_keymap_config( } } +fn handle_keymap_asr_config( + config: String, + nvs: &mut esp_idf_svc::nvs::EspDefaultNvs, +) -> anyhow::Result { + log::info!("Received keymap config: {}", config); + if config.is_empty() { + nvs.remove("asr_config").ok(); + return Ok("ASR config cleared".to_string()); + } + match audio::AsrConfig::from_json(&config) { + Ok(config) => match config.save_to_nvs(nvs) { + Ok(()) => { + log::info!("asr config merged and saved to NVS successfully"); + Ok(format!( + "ASR config updated: {}", + serde_json::to_string_pretty(&config) + .unwrap_or_else(|_| "Failed to serialize ASR config".to_string()) + )) + } + Err(e) => { + anyhow::bail!("Failed to save asr_config to NVS: {:?}", e); + } + }, + Err(e) => { + anyhow::bail!("Failed to parse asr_config JSON: {:?}", e); + } + } +} + async fn keyboard_mode_main( display: &mut lcd::FrameBuffer, ble_device: &mut esp32_nimble::BLEDevice, @@ -526,6 +611,9 @@ async fn keyboard_mode_main( mut setting_rx: tokio::sync::mpsc::Receiver, mut rx: tokio::sync::mpsc::Receiver, keymap: &mut bt_keyboard_mode::KeymapConfig, + mut driver: Option, + asr_config: Option, + controller: bt_keyboard_mode::ControllerService, ) -> ! { loop { let event = tokio::select! { @@ -533,6 +621,8 @@ async fn keyboard_mode_main( Some(bt_wifi_mode::BTevent::Reset) = setting_rx.recv() => { handle_reset_event(setting_arc); } + // Handle physical key events + key_evt = bt_keyboard_mode::wait_key_event(key_pins) => key_evt, // Handle controller commands from BLE Some(evt) = rx.recv() => { match evt { @@ -545,13 +635,46 @@ async fn keyboard_mode_main( lcd::display_text(display, "keymap updated!", 0).unwrap(); continue; } + bt_keyboard_mode::ControllerCommand::AsrConfig(config) => { + match handle_keymap_asr_config(config, &mut setting_arc.lock().unwrap().1) { + Ok(msg) => { + lcd::display_text(display, &msg, 0).unwrap(); + } + Err(e) => { + log::error!("Failed to update ASR config: {:?}", e); + lcd::display_text(display, &format!("Failed to update ASR config:\n{:?}", e), 0).unwrap(); + } + } + continue; + } controller_evt => controller_evt, } } - // Handle physical key events - key_evt = bt_keyboard_mode::wait_key_event(key_pins) => key_evt }; + if let (Some(driver), Some(asr_config)) = (driver.as_mut(), asr_config.as_ref()) { + if matches!( + event, + bt_keyboard_mode::ControllerCommand::KeyboardPress(bt_keyboard_mode::KeysPin::MIC) + ) { + match driver.start_asr( + asr_config, + || lcd::display_text(display, "start recording", 0).unwrap(), + || key_pins.mic.is_high(), + ) { + Ok(asr) => { + lcd::display_text(display, &format!("ASR:{asr}"), 0).unwrap(); + controller.notify_asr(&asr); + } + Err(e) => { + log::error!("ASR error: {:?}", e); + lcd::display_text(display, &format!("ASR error: {:?}", e), 0).unwrap(); + } + } + continue; + } + } + let _ = handle_key_event(display, ble_device, keyboard, event, keymap); } } @@ -607,6 +730,15 @@ pub fn handle_key_event( log::info!("Handling controller command: {:?}", event); use bt_keyboard_mode::KeysPin; match event { + bt_keyboard_mode::ControllerCommand::Paste(p) => { + if p == 0x01 { + keyboard.ctrl_press(b'v'); + keyboard.release(); + } else if p == 0x02 { + keyboard.gui_press(b'v'); + keyboard.release(); + } + } bt_keyboard_mode::ControllerCommand::DisplayKeyboard(text) => { lcd::display_text(display, &text, 0)?; } @@ -671,6 +803,9 @@ pub fn handle_key_event( bt_keyboard_mode::ControllerCommand::KeymapConfig(_) => { // KeymapConfig is handled separately in keyboard_mode_main } + bt_keyboard_mode::ControllerCommand::AsrConfig(_) => { + // AsrConfig is handled separately in keyboard_mode_main + } } Ok(()) diff --git a/src/ota.rs b/src/ota.rs index 5ddf180..f9597ce 100644 --- a/src/ota.rs +++ b/src/ota.rs @@ -6,15 +6,14 @@ use esp_idf_svc::{ mod wifi; -type AnyBtn = PinDriver<'static, esp_idf_svc::hal::gpio::AnyIOPin, esp_idf_svc::hal::gpio::Input>; +type AnyBtn = PinDriver<'static, esp_idf_svc::hal::gpio::Input>; fn new_btn( - pin: AnyIOPin, + pin: AnyIOPin<'static>, pull: esp_idf_svc::hal::gpio::Pull, interrupt: esp_idf_svc::hal::gpio::InterruptType, ) -> anyhow::Result { - let mut btn = PinDriver::input(pin)?; - btn.set_pull(pull)?; + let mut btn = PinDriver::input(pin, pull)?; btn.set_interrupt_type(interrupt)?; Ok(btn) } @@ -74,7 +73,7 @@ fn main() -> anyhow::Result<()> { let partition = esp_idf_svc::nvs::EspDefaultNvsPartition::take()?; let nvs = esp_idf_svc::nvs::EspDefaultNvs::new(partition, "setting", true)?; - let peripherals = esp_idf_svc::hal::prelude::Peripherals::take().unwrap(); + let peripherals = esp_idf_svc::hal::peripherals::Peripherals::take().unwrap(); let sysloop = esp_idf_svc::eventloop::EspSystemEventLoop::take()?; let mut bl = esp_idf_svc::hal::gpio::PinDriver::output(peripherals.pins.gpio11)?; @@ -326,9 +325,9 @@ mod lcd { const GPIO_NUM_NC: i32 = -1; let mut buscfg = spi_bus_config_t::default(); - buscfg.__bindgen_anon_1.mosi_io_num = mosi.pin(); + buscfg.__bindgen_anon_1.mosi_io_num = mosi.pin() as _; buscfg.__bindgen_anon_2.miso_io_num = GPIO_NUM_NC; - buscfg.sclk_io_num = clk.pin(); + buscfg.sclk_io_num = clk.pin() as _; buscfg.__bindgen_anon_3.quadwp_io_num = GPIO_NUM_NC; buscfg.__bindgen_anon_4.quadhd_io_num = GPIO_NUM_NC; buscfg.max_transfer_sz = @@ -344,8 +343,8 @@ mod lcd { ::log::info!("Install panel IO"); let mut panel_io: esp_lcd_panel_io_handle_t = std::ptr::null_mut(); let mut io_config = esp_lcd_panel_io_spi_config_t::default(); - io_config.cs_gpio_num = cs.pin(); - io_config.dc_gpio_num = dc.pin(); + io_config.cs_gpio_num = cs.pin() as _; + io_config.dc_gpio_num = dc.pin() as _; io_config.spi_mode = 3; io_config.pclk_hz = 40 * 1000 * 1000; io_config.trans_queue_depth = 10; @@ -360,7 +359,7 @@ mod lcd { let mut panel_config = esp_lcd_panel_dev_config_t::default(); let mut panel: esp_lcd_panel_handle_t = std::ptr::null_mut(); - panel_config.reset_gpio_num = rst.pin(); + panel_config.reset_gpio_num = rst.pin() as _; panel_config.data_endian = lcd_rgb_data_endian_t_LCD_RGB_DATA_ENDIAN_LITTLE; panel_config.__bindgen_anon_1.rgb_ele_order = lcd_rgb_element_order_t_LCD_RGB_ELEMENT_ORDER_RGB; diff --git a/src/util.rs b/src/util.rs new file mode 100644 index 0000000..9b67ea1 --- /dev/null +++ b/src/util.rs @@ -0,0 +1,54 @@ +use std::io::Write; + +/// WAV 音频参数结构体 +#[derive(Debug, Clone)] +pub struct WavConfig { + pub sample_rate: u32, // 采样率 (Hz) + pub channels: u16, // 声道数 + pub bits_per_sample: u16, // 位深度 +} + +impl Default for WavConfig { + fn default() -> Self { + Self { + sample_rate: crate::audio::SAMPLE_RATE, + channels: 1, // 单声道 + bits_per_sample: 16, // 16-bit + } + } +} + +pub fn create_unlimited_wav_header(config: &WavConfig) -> Vec { + let mut wav_data = Vec::new(); + let mut cursor = std::io::Cursor::new(&mut wav_data); + + let bytes_per_sample = config.bits_per_sample / 8; + let byte_rate = config.sample_rate * config.channels as u32 * bytes_per_sample as u32; + let block_align = config.channels * bytes_per_sample; + // let data_size = 0u32; + // let file_size = 0u32; + let data_size = 0xFFFFFFFFu32; // unknown data size + let file_size = 0x7FFFFFFFu32; + + cursor.write_all(b"RIFF").unwrap(); // ChunkID + cursor.write_all(&file_size.to_le_bytes()).unwrap(); // ChunkSize (little-endian) + cursor.write_all(b"WAVE").unwrap(); // Format + + // fmt subchunk + cursor.write_all(b"fmt ").unwrap(); // Subchunk1ID + cursor.write_all(&16u32.to_le_bytes()).unwrap(); // Subchunk1Size (PCM = 16) + cursor.write_all(&1u16.to_le_bytes()).unwrap(); // AudioFormat (PCM = 1) + cursor.write_all(&config.channels.to_le_bytes()).unwrap(); // NumChannels + cursor.write_all(&config.sample_rate.to_le_bytes()).unwrap(); // SampleRate + cursor.write_all(&byte_rate.to_le_bytes()).unwrap(); // ByteRate + cursor.write_all(&block_align.to_le_bytes()).unwrap(); // BlockAlign + cursor + .write_all(&config.bits_per_sample.to_le_bytes()) + .unwrap(); // BitsPerSample + + // data subchunk + cursor.write_all(b"data").unwrap(); // Subchunk2ID + cursor.write_all(&data_size.to_le_bytes()).unwrap(); // Subchunk2Size + + wav_data +}