diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index a324768..0a5502b 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -54,7 +54,7 @@ async-trait = "0.1" # Date/time chrono = { version = "0.4", features = ["serde"] } -# Audio capture (microphone via CoreAudio) +# Audio capture (microphone via CPAL) cpal = "0.15" # Whisper transcription (wraps whisper.cpp — requires cmake at build time) @@ -68,6 +68,14 @@ local-whisper = ["whisper-rs"] [target.'cfg(target_os = "macos")'.dependencies] objc2 = "0.6" +# Windows: WASAPI for system audio loopback +[target.'cfg(target_os = "windows")'.dependencies] +windows = { version = "0.58", features = [ + "Win32_Foundation", + "Win32_Media_Audio", + "Win32_System_Com", +] } + [profile.release] opt-level = 3 lto = true diff --git a/src-tauri/src/audio/loopback_windows.rs b/src-tauri/src/audio/loopback_windows.rs new file mode 100644 index 0000000..3490020 --- /dev/null +++ b/src-tauri/src/audio/loopback_windows.rs @@ -0,0 +1,202 @@ +//! System audio loopback capture on Windows via WASAPI. +//! +//! Uses WASAPI's loopback flag on the default render (output) device so that +//! whatever the user hears is captured — no special OS permission is required. +//! +//! The public API mirrors the macOS `loopback` module: `start()`, `drain()`, `stop()`. + +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::{thread, time::Duration}; + +// --- Global sample buffer -------------------------------------------------- + +struct LoopbackState { + samples: Vec, + source_rate: u32, +} + +static STATE: Mutex = Mutex::new(LoopbackState { + samples: Vec::new(), + source_rate: 44100, +}); + +static RUNNING: AtomicBool = AtomicBool::new(false); + +// --- Public API ----------------------------------------------------------- + +/// Start loopback capture. Spawns a background WASAPI capture thread. +/// Always returns `true`; errors are logged from the background thread. +pub fn start() -> bool { + { + let mut s = STATE + .lock() + .unwrap_or_else(|e| panic!("[aura] Loopback lock poisoned: {e}")); + s.samples.clear(); + s.source_rate = 44100; + } + RUNNING.store(true, Ordering::SeqCst); + thread::spawn(capture_loop); + true +} + +/// Drain accumulated samples without stopping capture. +/// Returns `(samples, source_rate_hz)`. +pub fn drain() -> (Vec, u32) { + let mut s = STATE + .lock() + .unwrap_or_else(|e| panic!("[aura] Loopback lock poisoned: {e}")); + let samples = std::mem::take(&mut s.samples); + let rate = s.source_rate; + (samples, rate) +} + +/// Stop loopback capture and return `(samples, source_rate_hz)`. +pub fn stop() -> (Vec, u32) { + RUNNING.store(false, Ordering::SeqCst); + // Give the background thread a moment to flush its last buffer. + thread::sleep(Duration::from_millis(150)); + let mut s = STATE + .lock() + .unwrap_or_else(|e| panic!("[aura] Loopback lock poisoned: {e}")); + let samples = std::mem::take(&mut s.samples); + let rate = s.source_rate; + (samples, rate) +} + +// --- Background capture thread -------------------------------------------- + +fn capture_loop() { + if let Err(e) = unsafe { wasapi_capture() } { + tracing::error!("[aura] WASAPI loopback error: {e}"); + } +} + +/// WASAPI loopback capture. +/// +/// Steps: +/// 1. CoInitializeEx on this thread (WASAPI is COM-based). +/// 2. Enumerate the default *render* (output) device — loopback captures +/// whatever that device is playing. +/// 3. Activate an IAudioClient, initialize it with AUDCLNT_STREAMFLAGS_LOOPBACK. +/// 4. Loop: poll `GetNextPacketSize`, then `GetBuffer` / `ReleaseBuffer`. +/// 5. Convert raw PCM to f32 and push into STATE. +unsafe fn wasapi_capture() -> windows::core::Result<()> { + use windows::Win32::{ + Media::Audio::{ + eConsole, eRender, IAudioCaptureClient, IAudioClient, IMMDeviceEnumerator, + MMDeviceEnumerator, AUDCLNT_SHAREMODE_SHARED, AUDCLNT_STREAMFLAGS_LOOPBACK, + }, + System::Com::{ + CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_ALL, COINIT_MULTITHREADED, + }, + }; + + CoInitializeEx(None, COINIT_MULTITHREADED).ok()?; + + let enumerator: IMMDeviceEnumerator = + CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)?; + + // Get the default *output* device — loopback taps the render pipeline. + let device = enumerator.GetDefaultAudioEndpoint(eRender, eConsole)?; + let client: IAudioClient = device.Activate(CLSCTX_ALL, None)?; + + // Query the device's native mix format (usually 32-bit float, stereo). + let fmt_ptr = client.GetMixFormat()?; + let fmt = &*fmt_ptr; + let sample_rate = fmt.nSamplesPerSec; + let channels = fmt.nChannels as usize; + let bits_per_sample = fmt.wBitsPerSample; + + tracing::info!( + "[aura] WASAPI loopback: {}Hz, {} ch, {} bps", + sample_rate, + channels, + bits_per_sample + ); + + { + let mut s = STATE + .lock() + .unwrap_or_else(|e| panic!("[aura] Loopback lock poisoned: {e}")); + s.source_rate = sample_rate; + } + + // Initialize for shared-mode loopback capture. + client.Initialize( + AUDCLNT_SHAREMODE_SHARED, + AUDCLNT_STREAMFLAGS_LOOPBACK, + 0, // buffer duration (0 = use default) + 0, // periodicity (must be 0 for shared mode) + fmt_ptr, + None, + )?; + + let capture: IAudioCaptureClient = client.GetService()?; + client.Start()?; + + while RUNNING.load(Ordering::SeqCst) { + // Poll for available packets. + let mut packet_size = 0u32; + capture.GetNextPacketSize(&mut packet_size)?; + + if packet_size == 0 { + thread::sleep(Duration::from_millis(10)); + continue; + } + + let mut data: *mut u8 = std::ptr::null_mut(); + let mut frames_available = 0u32; + let mut flags = 0u32; + + capture.GetBuffer(&mut data, &mut frames_available, &mut flags, None, None)?; + + if frames_available > 0 && !data.is_null() { + let floats = pcm_to_f32(data, frames_available as usize, channels, bits_per_sample); + STATE + .lock() + .unwrap_or_else(|e| panic!("[aura] Loopback lock poisoned: {e}")) + .samples + .extend_from_slice(&floats); + } + + capture.ReleaseBuffer(frames_available)?; + } + + client.Stop()?; + CoUninitialize(); + Ok(()) +} + +// --- PCM → f32 conversion ------------------------------------------------- + +/// Convert raw WASAPI PCM bytes to a flat f32 slice (interleaved channels). +/// +/// WASAPI shared mode most commonly delivers IEEE 32-bit float; 16-bit integer +/// is also handled. Unsupported depths output silence and log a warning. +fn pcm_to_f32(data: *const u8, num_frames: usize, channels: usize, bits_per_sample: u16) -> Vec { + let num_samples = num_frames * channels; + let mut out = Vec::with_capacity(num_samples); + + match bits_per_sample { + 32 => { + // IEEE float — the default for WASAPI shared mode. + let floats = + unsafe { std::slice::from_raw_parts(data as *const f32, num_samples) }; + out.extend_from_slice(floats); + } + 16 => { + let shorts = + unsafe { std::slice::from_raw_parts(data as *const i16, num_samples) }; + out.extend(shorts.iter().map(|&s| s as f32 / i16::MAX as f32)); + } + _ => { + tracing::warn!( + "[aura] Unsupported WASAPI bit depth: {bits_per_sample}, substituting silence" + ); + out.resize(num_samples, 0.0); + } + } + + out + } diff --git a/src-tauri/src/audio/mic.rs b/src-tauri/src/audio/mic.rs index eff4674..9980bc8 100644 --- a/src-tauri/src/audio/mic.rs +++ b/src-tauri/src/audio/mic.rs @@ -1,4 +1,175 @@ -//! Microphone capture via cpal (CoreAudio on macOS). +//! Microphone capture via cpal (CoreAudio on macOS, WASAPI on Windows). + +use anyhow::{Context, Result}; +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +use cpal::{SampleFormat, SupportedStreamConfig}; +use std::sync::{Arc, Mutex}; + +pub struct MicHandle { + _stream: SendStream, + pub samples: Arc>>, + pub sample_rate: u32, + pub channels: usize, +} + +#[allow(dead_code)] +struct SendStream(cpal::Stream); +// SAFETY: cpal's stream handle is internally thread-safe on all supported platforms. +unsafe impl Send for SendStream {} + +impl MicHandle { + pub fn stop(self) -> (Vec, u32, usize) { + drop(self._stream); // stops the audio callback + let samples = std::mem::take( + &mut *self + .samples + .lock() + .unwrap_or_else(|e| panic!("[aura] Mic samples lock poisoned: {e}")), + ); + (samples, self.sample_rate, self.channels) + } +} + +// --- Platform-specific permission handling -------------------------------- + +#[cfg(target_os = "macos")] +extern "C" { + fn aura_mic_auth_status() -> i32; + fn aura_request_mic_access() -> bool; +} + +/// Ensure microphone permission is granted before opening a cpal stream. +/// +/// On macOS this checks/requests access via the Swift AVFoundation layer so +/// that the system permission dialog appears before cpal touches CoreAudio. +/// +/// On Windows, WASAPI surfaces the permission prompt automatically when the +/// stream is opened, so no pre-check is needed. +#[cfg(target_os = "macos")] +fn ensure_mic_permission() -> Result<()> { + let status = unsafe { aura_mic_auth_status() }; + match status { + 1 => Ok(()), // already authorized + 2 => anyhow::bail!( + "Microphone access denied. Grant permission in System Settings → Privacy & Security → Microphone." + ), + 3 => anyhow::bail!("Microphone access restricted by system policy."), + _ => { + // Not determined — request now (blocks until user responds) + tracing::info!("Requesting microphone permission…"); + let granted = unsafe { aura_request_mic_access() }; + if granted { + Ok(()) + } else { + anyhow::bail!( + "Microphone access denied. Grant permission in System Settings → Privacy & Security → Microphone." + ) + } + } + } +} + +#[cfg(not(target_os = "macos"))] +fn ensure_mic_permission() -> Result<()> { + // On Windows (and other platforms) WASAPI / the OS handles permission + // prompts automatically when the audio stream is opened. + Ok(()) +} + +// --- Stream setup --------------------------------------------------------- + +pub fn start() -> Result { + ensure_mic_permission()?; + + let host = cpal::default_host(); + let device = host + .default_input_device() + .context("No default audio input device")?; + + tracing::info!("Mic device: {}", device.name().unwrap_or_default()); + + let (config, sample_format) = preferred_config(&device)?; + let sample_rate = config.sample_rate().0; + let channels = config.channels() as usize; + + tracing::info!( + "Mic config: {}Hz {} ch {:?}", + sample_rate, + channels, + sample_format + ); + + let samples: Arc>> = Arc::new(Mutex::new(Vec::new())); + let samples_cb = samples.clone(); + + let stream = match sample_format { + SampleFormat::F32 => build_stream::(&device, &config.into(), samples_cb)?, + SampleFormat::I16 => build_stream::(&device, &config.into(), samples_cb)?, + SampleFormat::U16 => build_stream::(&device, &config.into(), samples_cb)?, + _ => anyhow::bail!("Unsupported mic sample format: {:?}", sample_format), + }; + stream.play().context("Failed to start mic stream")?; + + Ok(MicHandle { + _stream: SendStream(stream), + samples, + sample_rate, + channels, + }) +} + +fn preferred_config(device: &cpal::Device) -> Result<(SupportedStreamConfig, SampleFormat)> { + let mut configs: Vec<_> = device.supported_input_configs()?.collect(); + configs.sort_by_key(|c| std::cmp::Reverse(c.max_sample_rate().0)); + for fmt in [SampleFormat::F32, SampleFormat::I16, SampleFormat::U16] { + if let Some(c) = configs.iter().find(|c| c.sample_format() == fmt) { + return Ok(((*c).with_max_sample_rate(), fmt)); + } + } + anyhow::bail!("No supported mic input config") +} + +fn build_stream( + device: &cpal::Device, + config: &cpal::StreamConfig, + samples: Arc>>, +) -> Result +where + T: cpal::Sample + cpal::SizedSample + ToF32, +{ + let stream = device.build_input_stream( + config, + move |data: &[T], _: &cpal::InputCallbackInfo| { + let floats: Vec = data.iter().map(|s| s.to_f32()).collect(); + samples + .lock() + .unwrap_or_else(|e| panic!("[aura] Mic callback lock poisoned: {e}")) + .extend(floats); + }, + |err| tracing::error!("Mic stream error: {}", err), + None, + )?; + Ok(stream) +} + +pub trait ToF32: Copy { + fn to_f32(self) -> f32; +} +impl ToF32 for f32 { + fn to_f32(self) -> f32 { + self + } +} +impl ToF32 for i16 { + fn to_f32(self) -> f32 { + self as f32 / i16::MAX as f32 + } +} +impl ToF32 for u16 { + fn to_f32(self) -> f32 { + (self as f32 / u16::MAX as f32) * 2.0 - 1.0 + } +}//! Microphone capture via cpal (CoreAudio on macOS). use anyhow::{Context, Result}; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; diff --git a/src-tauri/src/audio/mod.rs b/src-tauri/src/audio/mod.rs index 6e47e8c..010c80a 100644 --- a/src-tauri/src/audio/mod.rs +++ b/src-tauri/src/audio/mod.rs @@ -1,4 +1,155 @@ -//! Audio capture pipeline — mic (cpal) + system loopback (ScreenCaptureKit). +//! Audio capture pipeline — mic (cpal) + system loopback (ScreenCaptureKit on macOS, +//! WASAPI on Windows). +//! +//! Both streams are started together. On stop, the samples are mixed and +//! resampled to 16 kHz mono — the format whisper.cpp expects. + +// Platform-specific loopback backends, both exposed as the `loopback` module +// so the rest of this file stays platform-agnostic. +#[cfg(target_os = "macos")] +#[path = "loopback.rs"] +pub mod loopback; + +#[cfg(target_os = "windows")] +#[path = "loopback_windows.rs"] +pub mod loopback; + +pub mod mic; +pub mod resample; +pub mod vad; + +use anyhow::Result; +use resample::to_whisper_format; + +/// An active recording session. Drop or call `stop()` to end it. +pub struct CaptureHandle { + pub mic: Option, + pub loopback_active: bool, +} + +/// Start capturing audio according to the configured capture mode. +/// `system_audio` = true means also capture loopback audio. +pub fn start_capture(system_audio: bool) -> Result { + let mic = match mic::start() { + Ok(h) => { + tracing::info!("Mic capture started"); + Some(h) + } + Err(e) => { + tracing::warn!("Mic unavailable: {}. Continuing without mic.", e); + None + } + }; + + let loopback_active = if system_audio { + let ok = loopback::start(); + if ok { + tracing::info!("Loopback capture initiated"); + } else { + tracing::warn!("Loopback capture failed to start"); + } + ok + } else { + false + }; + + if system_audio && !loopback_active { + if let Some(m) = mic { + m.stop(); // Clean up mic if loopback failed + } + #[cfg(target_os = "macos")] + anyhow::bail!("Screen Recording access denied or invalid. If you just granted it, you MUST completely Quit (Cmd+Q) and reopen the app to apply the fix."); + #[cfg(not(target_os = "macos"))] + anyhow::bail!("System audio loopback capture failed to start."); + } + + if mic.is_none() && !loopback_active { + anyhow::bail!("No audio source available. Grant Microphone permission in System Settings."); + } + + Ok(CaptureHandle { + mic, + loopback_active, + }) +} + +impl CaptureHandle { + /// Drain audio accumulated since the last drain (or start), mixed and resampled to 16 kHz mono. + /// Does NOT stop the capture streams. Returns empty vec if no audio yet. + pub fn drain(&self) -> Vec { + let mic_16k = self.mic.as_ref().and_then(|h| { + let raw: Vec = std::mem::take( + &mut *h + .samples + .lock() + .unwrap_or_else(|e| panic!("[aura] Mic samples lock poisoned: {e}")), + ); + if raw.is_empty() { + return None; + } + Some(to_whisper_format(&raw, h.sample_rate, h.channels)) + }); + + let lb_16k = if self.loopback_active { + let (raw, rate) = loopback::drain(); + if raw.is_empty() { + None + } else { + Some(to_whisper_format(&raw, rate, 1)) + } + } else { + None + }; + + match (mic_16k, lb_16k) { + (Some(m), Some(l)) => mix(&m, &l), + (Some(m), None) => m, + (None, Some(l)) => l, + (None, None) => vec![], + } + } + + /// Stop all capture, drain remaining samples, mix streams, and return 16 kHz mono f32 samples. + pub fn stop(self) -> Vec { + // Stop mic and resample + let mic_16k = self.mic.map(|h| { + let (raw, rate, ch) = h.stop(); + tracing::info!("Mic: {} samples @ {}Hz {} ch", raw.len(), rate, ch); + to_whisper_format(&raw, rate, ch) + }); + + // Stop loopback and resample + let lb_16k = if self.loopback_active { + let (raw, rate) = loopback::stop(); + tracing::info!("Loopback: {} samples @ {}Hz mono", raw.len(), rate); + Some(to_whisper_format(&raw, rate, 1)) + } else { + None + }; + + // Mix the two streams + match (mic_16k, lb_16k) { + (Some(mic), Some(lb)) => mix(&mic, &lb), + (Some(mic), None) => mic, + (None, Some(lb)) => lb, + (None, None) => vec![], + } + } +} + +/// Element-wise mix of two 16 kHz mono streams. +/// Pads the shorter stream with silence. Levels chosen so neither clips. +pub fn mix(a: &[f32], b: &[f32]) -> Vec { + let len = a.len().max(b.len()); + let mut out = vec![0f32; len]; + for (i, &s) in a.iter().enumerate() { + out[i] += s * 0.6; + } + for (i, &s) in b.iter().enumerate() { + out[i] += s * 0.6; + } + out +}//! Audio capture pipeline — mic (cpal) + system loopback (ScreenCaptureKit). //! //! Both streams are started together. On stop, the samples are mixed and //! resampled to 16 kHz mono — the format whisper.cpp expects. diff --git a/src-tauri/src/calendar/mod.rs b/src-tauri/src/calendar/mod.rs index c2bedfe..e02be99 100644 --- a/src-tauri/src/calendar/mod.rs +++ b/src-tauri/src/calendar/mod.rs @@ -11,6 +11,97 @@ pub struct CalendarEvent { pub calendar_name: String, } +// --- macOS: EventKit via Swift FFI ---------------------------------------- + +#[cfg(target_os = "macos")] +extern "C" { + fn aura_calendar_events() -> *const std::ffi::c_char; + fn aura_calendar_events_free(ptr: *mut std::ffi::c_char); +} + +/// Fetch upcoming calendar events (next 24 hours). +/// +/// On macOS this reads from EventKit via the Swift layer. +/// On other platforms calendar integration is not yet implemented and an +/// empty list is returned. +/// +/// Blocks on a semaphore internally on macOS — call via `spawn_blocking`. +#[cfg(target_os = "macos")] +pub fn list_upcoming_events() -> Result, String> { + let json_ptr = unsafe { aura_calendar_events() }; + if json_ptr.is_null() { + return Err("Calendar access denied. Grant permission in System Settings → Privacy & Security → Calendars.".into()); + } + + let c_str = unsafe { std::ffi::CStr::from_ptr(json_ptr) }; + let json_str = c_str.to_str().map_err(|e| e.to_string())?; + let events: Vec = serde_json::from_str(json_str) + .map_err(|e| format!("Failed to parse calendar data: {e}"))?; + + unsafe { aura_calendar_events_free(json_ptr as *mut _) }; + Ok(events) +} + +/// Calendar integration is not yet implemented on this platform. +#[cfg(not(target_os = "macos"))] +pub fn list_upcoming_events() -> Result, String> { + Ok(vec![]) +} + +// --- Tests ---------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn calendar_event_serializes_to_camel_case() { + let event = CalendarEvent { + title: "Stand-up".into(), + start_date: "2026-03-13T10:00:00Z".into(), + end_date: "2026-03-13T10:30:00Z".into(), + meeting_url: Some("https://meet.google.com/abc-defg-hij".into()), + attendees: vec!["Alice".into(), "Bob".into()], + calendar_name: "Work".into(), + }; + let json = serde_json::to_value(&event).unwrap(); + assert!(json.get("startDate").is_some()); + assert!(json.get("meetingUrl").is_some()); + assert!(json.get("calendarName").is_some()); + // Must not have snake_case keys + assert!(json.get("start_date").is_none()); + assert!(json.get("meeting_url").is_none()); + } + + #[test] + fn calendar_event_round_trips() { + let original = CalendarEvent { + title: "1:1".into(), + start_date: "2026-03-13T14:00:00Z".into(), + end_date: "2026-03-13T14:30:00Z".into(), + meeting_url: None, + attendees: vec![], + calendar_name: "Personal".into(), + }; + let json = serde_json::to_string(&original).unwrap(); + let parsed: CalendarEvent = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed.title, "1:1"); + assert!(parsed.meeting_url.is_none()); + assert!(parsed.attendees.is_empty()); + } +}use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CalendarEvent { + pub title: String, + pub start_date: String, + pub end_date: String, + pub meeting_url: Option, + pub attendees: Vec, + pub calendar_name: String, +} + extern "C" { fn aura_calendar_events() -> *const std::ffi::c_char; fn aura_calendar_events_free(ptr: *mut std::ffi::c_char); diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index c5a80f4..16eb538 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,4 +1,55 @@ { + "$schema": "https://schema.tauri.app/config/2", + "productName": "Aura", + "version": "0.1.0", + "identifier": "app.aura.aura", + "build": { + "frontendDist": "../dist", + "devUrl": "http://localhost:1420", + "beforeDevCommand": "vite --port 1420", + "beforeBuildCommand": "vite build" + }, + "app": { + "withGlobalTauri": false, + "windows": [ + { + "title": "Aura", + "width": 380, + "height": 600, + "resizable": false, + "visible": false, + "decorations": false, + "transparent": true, + "alwaysOnTop": false, + "label": "main" + } + ], + "trayIcon": { + "id": "main", + "iconPath": "icons/tray.png", + "iconAsTemplate": true, + "menuOnLeftClick": false, + "tooltip": "Aura" + }, + "security": { + "csp": "default-src 'self'; style-src 'self' 'unsafe-inline'; connect-src ipc: http://localhost:*" + } + }, + "bundle": { + "active": true, + "targets": ["dmg", "app", "nsis"], + "icon": ["icons/icon.icns", "icons/icon.ico", "icons/icon.png"], + "macOS": { + "entitlements": "entitlements.plist", + "signingIdentity": "-", + "minimumSystemVersion": "13.0", + "infoPlist": "Info.plist" + }, + "windows": { + "nsis": {} + } + } +}{ "$schema": "https://schema.tauri.app/config/2", "productName": "Aura", "version": "0.1.0",