From 10793ac1d89ce27cf54fb1b48abc9df2a6d3cf80 Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Sun, 26 Jul 2026 20:25:16 -0700 Subject: [PATCH 01/19] Add live iOS Simulator framebuffer capture Replaces the need to shell out to `xcrun simctl io screenshot` for frame acquisition by driving SimulatorKit's display pipeline directly. Three things made this non-obvious: - CoreSimulator vends its IO ports and display descriptors as ROCKRemoteProxy objects that implement their interface via forwarding, so objc2's msg_send! rejects them in debug builds. Messages now go through raw objc_msgSend helpers guarded by respondsToSelector:. - ROCKit marshals block arguments across the proxy boundary by reading the block's ObjC type encoding, which requires BLOCK_HAS_SIGNATURE. block2 does not emit that flag yet, so the three screen callbacks are created by a small C shim where clang emits a conforming block. - SimulatorKit recycles its framebuffer IOSurface in place, so frames are deep-copied into a pooled buffer before going downstream. Also carries over the seed-based dirty check, the 200ms forced re-emit floor (needed so idle screens still paint for late joiners), and the re-wire retry that covers lazily created descriptors. Verified against a booted iPhone 17: 1206x2622, ~5fps idle, ~68fps animating. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <166158716+staging-devin-ai-integration[bot]@users.noreply.github.com> --- Cargo.lock | 66 +++ Cargo.toml | 32 ++ packages/accessibility-ios-sys/Cargo.toml | 22 +- packages/accessibility-ios-sys/build.rs | 11 + .../examples/framebuffer_probe.rs | 50 ++ packages/accessibility-ios-sys/src/lib.rs | 98 +++- packages/accessibility-ios-sys/src/macos.rs | 5 + .../accessibility-ios-sys/src/macos/blocks.c | 31 + .../src/macos/dynamic.rs | 101 ++++ .../src/macos/framebuffer.rs | 541 ++++++++++++++++++ .../src/macos/pixel_buffer.rs | 159 +++++ .../src/macos/void_block.rs | 63 ++ 12 files changed, 1142 insertions(+), 37 deletions(-) create mode 100644 packages/accessibility-ios-sys/build.rs create mode 100644 packages/accessibility-ios-sys/examples/framebuffer_probe.rs create mode 100644 packages/accessibility-ios-sys/src/macos/blocks.c create mode 100644 packages/accessibility-ios-sys/src/macos/dynamic.rs create mode 100644 packages/accessibility-ios-sys/src/macos/framebuffer.rs create mode 100644 packages/accessibility-ios-sys/src/macos/pixel_buffer.rs create mode 100644 packages/accessibility-ios-sys/src/macos/void_block.rs diff --git a/Cargo.lock b/Cargo.lock index 42fcde5..a4b19ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,12 +75,19 @@ dependencies = [ "accesskit", "anyhow", "block2", + "bytes", + "cc", + "dispatch2", "euclid", "image", "libc", "objc2", "objc2-core-foundation", + "objc2-core-media", + "objc2-core-video", "objc2-foundation", + "objc2-io-surface", + "objc2-video-toolbox", "slotmap", ] @@ -1724,6 +1731,28 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags", + "objc2", +] + [[package]] name = "objc2-core-data" version = "0.3.2" @@ -1774,6 +1803,22 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc2-core-media" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec576860167a15dd9fce7fbee7512beb4e31f532159d3482d1f9c6caedf31d" +dependencies = [ + "bitflags", + "block2", + "dispatch2", + "objc2", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-video", +] + [[package]] name = "objc2-core-services" version = "0.3.2" @@ -1805,10 +1850,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d425caf1df73233f29fd8a5c3e5edbc30d2d4307870f802d18f00d83dc5141a6" dependencies = [ "bitflags", + "block2", "objc2", "objc2-core-foundation", "objc2-core-graphics", "objc2-io-surface", + "objc2-metal", ] [[package]] @@ -1837,8 +1884,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" dependencies = [ "bitflags", + "libc", "objc2", "objc2-core-foundation", + "objc2-foundation", ] [[package]] @@ -1874,6 +1923,23 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-video-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05bf9a3c14831a7d9641b0d81d87dd913ee238a012b2fde27db5a84b56f5df3e" +dependencies = [ + "bitflags", + "block2", + "objc2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-media", + "objc2-core-video", + "objc2-foundation", + "objc2-metal", +] + [[package]] name = "once_cell" version = "1.21.4" diff --git a/Cargo.toml b/Cargo.toml index 232f1f2..8f93407 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,14 +31,46 @@ anyhow = "1.0.100" async-trait = "0.1.89" atspi = { version = "0.29", features = ["connection", "proxies"] } atspi-common = "0.13" +block2 = "0.6" +bytes = "1" clap = { version = "4", features = ["derive"] } cssparser = "0.31" +dispatch2 = "0.3" ctrlc = "3" euclid = { version = "0.22", features = ["serde"] } futures-lite = "2" image = "0.25" imageproc = "0.25" keyboard-types = "0.7" +libc = "0.2" +objc2 = "0.6" +objc2-core-foundation = { version = "0.3", features = ["CFBase", "CFCGTypes"] } +objc2-core-media = { version = "0.3", features = [ + "CMSampleBuffer", + "CMTime", + "CMFormatDescription", + "CMBlockBuffer", + "objc2-core-video", +] } +objc2-core-video = { version = "0.3", features = [ + "CVPixelBuffer", + "CVPixelBufferPool", + "CVImageBuffer", + "CVBuffer", + "CVReturn", + "objc2-io-surface", +] } +objc2-foundation = "0.3" +objc2-io-surface = { version = "0.3", features = ["IOSurfaceRef"] } +objc2-video-toolbox = { version = "0.3", features = [ + "VTCompressionSession", + "VTCompressionProperties", + "VTSession", + "VTErrors", + "block2", + "objc2-core-media", + "objc2-core-video", +] } quick-xml = "0.37" selectors = "0.25" serde = { version = "1.0.228", features = ["derive"] } diff --git a/packages/accessibility-ios-sys/Cargo.toml b/packages/accessibility-ios-sys/Cargo.toml index 72883f7..9ed705b 100644 --- a/packages/accessibility-ios-sys/Cargo.toml +++ b/packages/accessibility-ios-sys/Cargo.toml @@ -13,16 +13,22 @@ categories = ["accessibility", "api-bindings", "os::macos-apis"] [dependencies] accesskit.workspace = true anyhow.workspace = true +bytes.workspace = true euclid.workspace = true image.workspace = true slotmap.workspace = true [target.'cfg(target_os = "macos")'.dependencies] -block2 = "0.6" -libc = "0.2" -objc2 = "0.6" -objc2-foundation = "0.3" -objc2-core-foundation = { version = "0.3", features = [ - "CFBase", - "CFCGTypes", -] } +block2.workspace = true +dispatch2.workspace = true +libc.workspace = true +objc2.workspace = true +objc2-core-foundation.workspace = true +objc2-core-media.workspace = true +objc2-core-video.workspace = true +objc2-foundation.workspace = true +objc2-io-surface.workspace = true +objc2-video-toolbox.workspace = true + +[target.'cfg(target_os = "macos")'.build-dependencies] +cc = "1" diff --git a/packages/accessibility-ios-sys/build.rs b/packages/accessibility-ios-sys/build.rs new file mode 100644 index 0000000..8a084a9 --- /dev/null +++ b/packages/accessibility-ios-sys/build.rs @@ -0,0 +1,11 @@ +fn main() { + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("macos") { + return; + } + + println!("cargo:rerun-if-changed=src/macos/blocks.c"); + cc::Build::new() + .file("src/macos/blocks.c") + .flag("-fblocks") + .compile("accessibility_ios_blocks"); +} diff --git a/packages/accessibility-ios-sys/examples/framebuffer_probe.rs b/packages/accessibility-ios-sys/examples/framebuffer_probe.rs new file mode 100644 index 0000000..af6305b --- /dev/null +++ b/packages/accessibility-ios-sys/examples/framebuffer_probe.rs @@ -0,0 +1,50 @@ +//! Probe the SimulatorKit framebuffer capture path against a booted simulator. +//! +//! Run with: `cargo run -p accessibility-ios-sys --example framebuffer_probe` + +#[cfg(not(target_os = "macos"))] +fn main() { + eprintln!("framebuffer_probe only runs on macOS"); + std::process::exit(1); +} + +#[cfg(target_os = "macos")] +fn main() -> anyhow::Result<()> { + use accessibility_ios_sys::SimFramebuffer; + use std::time::{Duration, Instant}; + + let mut fb = SimFramebuffer::new(None)?; + println!("device udid: {}", fb.device_udid()); + + fb.start()?; + + let deadline = Instant::now() + Duration::from_secs(5); + let mut last = 0u64; + while Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(500)); + let stats = fb.stats(); + let delta = stats.frame_count - last; + last = stats.frame_count; + println!( + "frames={} (+{}) size={}x{} descriptors={} rewires={}", + stats.frame_count, + delta, + stats.width, + stats.height, + stats.descriptor_count, + stats.rewire_count, + ); + } + + let stats = fb.stats(); + fb.stop(); + + if stats.frame_count == 0 { + anyhow::bail!("no frames captured - the framebuffer pipeline did not come up"); + } + println!( + "OK: captured {} frames at {}x{}", + stats.frame_count, stats.width, stats.height + ); + Ok(()) +} diff --git a/packages/accessibility-ios-sys/src/lib.rs b/packages/accessibility-ios-sys/src/lib.rs index fb874b8..0036c64 100644 --- a/packages/accessibility-ios-sys/src/lib.rs +++ b/packages/accessibility-ios-sys/src/lib.rs @@ -30,18 +30,25 @@ pub(crate) mod frameworks { /// Load the CoreSimulator private framework. pub fn load_coresimulator_framework() -> Result<()> { - let paths: &[&[u8]] = &[ - b"/Library/Developer/PrivateFrameworks/CoreSimulator.framework/CoreSimulator\0", - b"/Applications/Xcode.app/Contents/Developer/Library/PrivateFrameworks/CoreSimulator.framework/CoreSimulator\0", - ]; - - for path in paths { - let handle = unsafe { - libc::dlopen( - path.as_ptr() as *const c_char, - libc::RTLD_NOW | libc::RTLD_GLOBAL, - ) - }; + const SUFFIX: &str = "Library/PrivateFrameworks/CoreSimulator.framework/CoreSimulator"; + + // The system-wide copy is canonical and exists whenever any Xcode is + // installed, so it goes first. + let mut paths = vec![format!( + "/Library/Developer/PrivateFrameworks/CoreSimulator.framework/CoreSimulator" + )]; + + if let Some(dev_path) = developer_dir() { + paths.push(format!("{dev_path}/{SUFFIX}")); + } + paths.push(format!( + "/Applications/Xcode.app/Contents/Developer/{SUFFIX}" + )); + + for path in &paths { + let c_path = CString::new(path.as_str())?; + let handle = + unsafe { libc::dlopen(c_path.as_ptr(), libc::RTLD_NOW | libc::RTLD_GLOBAL) }; if !handle.is_null() { return Ok(()); } @@ -49,31 +56,63 @@ pub(crate) mod frameworks { let error = unsafe { CStr::from_ptr(libc::dlerror()) }; Err(anyhow!( - "Failed to load CoreSimulator framework: {}", - error.to_string_lossy() + "Failed to load CoreSimulator framework: {}. Tried paths: {:?}", + error.to_string_lossy(), + paths )) } + /// Candidate SimulatorKit locations for a given developer directory. + /// + /// Xcode 27 moved SimulatorKit out of `Developer/Library/PrivateFrameworks` + /// and into `Contents/SharedFrameworks` (a sibling of `Developer`), so both + /// layouts have to be probed. + fn simulatorkit_candidates(developer_dir: &str) -> [String; 2] { + [ + format!("{developer_dir}/../SharedFrameworks/SimulatorKit.framework/SimulatorKit"), + format!( + "{developer_dir}/Library/PrivateFrameworks/SimulatorKit.framework/SimulatorKit" + ), + ] + } + + /// Resolve the active developer directory, preferring `DEVELOPER_DIR`. + /// + /// Falls back to `xcode-select -p`, which can point at a Command Line Tools + /// install that has no simulator frameworks at all — callers still probe the + /// well-known Xcode.app locations afterwards. + pub fn developer_dir() -> Option { + if let Ok(dir) = std::env::var("DEVELOPER_DIR") + && !dir.is_empty() + { + return Some(dir); + } + + let output = std::process::Command::new("xcode-select") + .arg("-p") + .output() + .ok()?; + if !output.status.success() { + return None; + } + let dir = String::from_utf8_lossy(&output.stdout).trim().to_string(); + (!dir.is_empty()).then_some(dir) + } + /// Load the SimulatorKit framework from Xcode. pub fn load_simulatorkit_framework() -> Result<*mut c_void> { let mut paths_to_try: Vec = Vec::new(); - if let Ok(output) = std::process::Command::new("xcode-select") - .arg("-p") - .output() - && output.status.success() - { - let dev_path = String::from_utf8_lossy(&output.stdout).trim().to_string(); - paths_to_try.push(format!( - "{}/Library/PrivateFrameworks/SimulatorKit.framework/SimulatorKit", - dev_path - )); + if let Some(dev_path) = developer_dir() { + paths_to_try.extend(simulatorkit_candidates(&dev_path)); } - paths_to_try.extend([ - "/Applications/Xcode.app/Contents/Developer/Library/PrivateFrameworks/SimulatorKit.framework/SimulatorKit".to_string(), - "/Applications/Xcode-beta.app/Contents/Developer/Library/PrivateFrameworks/SimulatorKit.framework/SimulatorKit".to_string(), - ]); + for app in [ + "/Applications/Xcode.app/Contents/Developer", + "/Applications/Xcode-beta.app/Contents/Developer", + ] { + paths_to_try.extend(simulatorkit_candidates(app)); + } for path in &paths_to_try { let c_path = CString::new(path.as_str()).unwrap(); @@ -105,7 +144,8 @@ mod macos; #[cfg(target_os = "macos")] pub use frameworks::{ - load_axp_framework, load_coresimulator_framework, load_frameworks, load_simulatorkit_framework, + developer_dir, load_axp_framework, load_coresimulator_framework, load_frameworks, + load_simulatorkit_framework, }; #[cfg(target_os = "macos")] diff --git a/packages/accessibility-ios-sys/src/macos.rs b/packages/accessibility-ios-sys/src/macos.rs index 2db586f..a022b4e 100644 --- a/packages/accessibility-ios-sys/src/macos.rs +++ b/packages/accessibility-ios-sys/src/macos.rs @@ -56,11 +56,16 @@ use slotmap::SecondaryMap; mod common; mod dispatcher; +mod dynamic; +mod framebuffer; mod hid; +mod pixel_buffer; mod reader; +mod void_block; pub use common::{ ButtonDirection, Element, ElementKey, ElementTree, HardwareButton, Point, Rect, ScreenSpace, Screenshot, Size, TreeFilter, load_frameworks, }; +pub use framebuffer::{CapturedFrame, FrameSink, FramebufferStats, SimFramebuffer}; pub use reader::IOSSimulatorAccessibility; diff --git a/packages/accessibility-ios-sys/src/macos/blocks.c b/packages/accessibility-ios-sys/src/macos/blocks.c new file mode 100644 index 0000000..d5edc33 --- /dev/null +++ b/packages/accessibility-ios-sys/src/macos/blocks.c @@ -0,0 +1,31 @@ +// Signature-bearing Objective-C blocks for CoreSimulator's remote proxies. +// +// CoreSimulator vends its IO descriptors as ROCKRemoteProxy objects, and +// ROCKit marshals block arguments across that boundary by reading the block's +// Objective-C type encoding out of its descriptor. That requires the +// BLOCK_HAS_SIGNATURE flag, which the `block2` crate does not currently emit +// (see the TODO in block2's global.rs). Clang always emits it, so the blocks +// handed to SimulatorKit are created here instead of in Rust. + +#include +#include + +typedef void (*accessibility_void_callback)(void *context); + +// Create a heap block wrapping `callback(context)`. +// +// The returned block is owned by the caller and must be handed to +// accessibility_release_block exactly once. `context` is not managed here; the +// Rust side owns it and must outlive the block. +void *accessibility_make_void_block(accessibility_void_callback callback, void *context) { + void (^block)(void) = ^{ + callback(context); + }; + return (void *)Block_copy(block); +} + +void accessibility_release_block(void *block) { + if (block != NULL) { + Block_release(block); + } +} diff --git a/packages/accessibility-ios-sys/src/macos/dynamic.rs b/packages/accessibility-ios-sys/src/macos/dynamic.rs new file mode 100644 index 0000000..2517961 --- /dev/null +++ b/packages/accessibility-ios-sys/src/macos/dynamic.rs @@ -0,0 +1,101 @@ +//! Dynamic Objective-C message sends for proxied private objects. +//! +//! CoreSimulator vends much of its IO graph as `ROCKRemoteProxy` objects, which +//! implement their interface through forwarding rather than real methods. That +//! breaks `objc2::msg_send!`, whose debug-build verification looks the selector +//! up with `class_getInstanceMethod` and panics when it is absent. +//! +//! These helpers call `objc_msgSend` directly so the message reaches the +//! proxy's forwarding machinery, which is what the selector is for. Callers +//! must guard with [`responds_to`] first — an unrecognized selector reaching +//! forwarding is a hard crash, not a recoverable error. + +use std::ffi::c_void; + +use objc2::runtime::{AnyObject, Sel}; +use objc2::sel; + +/// Send a zero-argument message returning an object pointer. +pub(super) unsafe fn send_id(receiver: *mut AnyObject, selector: Sel) -> *mut AnyObject { + if receiver.is_null() { + return std::ptr::null_mut(); + } + type Imp = unsafe extern "C" fn(*mut AnyObject, Sel) -> *mut AnyObject; + let imp: Imp = unsafe { std::mem::transmute(objc2::ffi::objc_msgSend as *const c_void) }; + unsafe { imp(receiver, selector) } +} + +/// Send a one-argument message returning an object pointer. +pub(super) unsafe fn send_id_with_id( + receiver: *mut AnyObject, + selector: Sel, + argument: *mut AnyObject, +) -> *mut AnyObject { + if receiver.is_null() { + return std::ptr::null_mut(); + } + type Imp = unsafe extern "C" fn(*mut AnyObject, Sel, *mut AnyObject) -> *mut AnyObject; + let imp: Imp = unsafe { std::mem::transmute(objc2::ffi::objc_msgSend as *const c_void) }; + unsafe { imp(receiver, selector, argument) } +} + +/// Send `registerScreenCallbacksWithUUID:callbackQueue:frameCallback: +/// surfacesChangedCallback:propertiesChangedCallback:`. +/// +/// SimulatorKit retains the three blocks for the lifetime of the registration, +/// so the caller must keep them alive until it unregisters the same UUID. +pub(super) unsafe fn send_register_screen_callbacks( + receiver: *mut AnyObject, + selector: Sel, + uuid: *mut AnyObject, + queue: *mut c_void, + frame_callback: *mut c_void, + surfaces_changed_callback: *mut c_void, + properties_changed_callback: *mut c_void, +) { + type Imp = unsafe extern "C" fn( + *mut AnyObject, + Sel, + *mut AnyObject, + *mut c_void, + *mut c_void, + *mut c_void, + *mut c_void, + ); + let imp: Imp = unsafe { std::mem::transmute(objc2::ffi::objc_msgSend as *const c_void) }; + unsafe { + imp( + receiver, + selector, + uuid, + queue, + frame_callback, + surfaces_changed_callback, + properties_changed_callback, + ) + } +} + +/// Send a one-argument message returning nothing. +pub(super) unsafe fn send_void_with_id( + receiver: *mut AnyObject, + selector: Sel, + argument: *mut AnyObject, +) { + if receiver.is_null() { + return; + } + type Imp = unsafe extern "C" fn(*mut AnyObject, Sel, *mut AnyObject); + let imp: Imp = unsafe { std::mem::transmute(objc2::ffi::objc_msgSend as *const c_void) }; + unsafe { imp(receiver, selector, argument) } +} + +/// Whether `receiver` will handle `selector`, including via forwarding. +pub(super) unsafe fn responds_to(receiver: *mut AnyObject, selector: Sel) -> bool { + if receiver.is_null() { + return false; + } + type Imp = unsafe extern "C" fn(*mut AnyObject, Sel, Sel) -> bool; + let imp: Imp = unsafe { std::mem::transmute(objc2::ffi::objc_msgSend as *const c_void) }; + unsafe { imp(receiver, sel!(respondsToSelector:), selector) } +} diff --git a/packages/accessibility-ios-sys/src/macos/framebuffer.rs b/packages/accessibility-ios-sys/src/macos/framebuffer.rs new file mode 100644 index 0000000..8de034c --- /dev/null +++ b/packages/accessibility-ios-sys/src/macos/framebuffer.rs @@ -0,0 +1,541 @@ +//! Live framebuffer capture for a booted iOS Simulator. +//! +//! SimulatorKit exposes the simulator's display as an `IOSurface` behind a +//! private "device IO port" graph. This module walks that graph and registers +//! screen callbacks so we get pushed a notification whenever the display is +//! repainted, which is dramatically cheaper and lower latency than polling +//! `xcrun simctl io screenshot`. +//! +//! The traversal is: +//! +//! ```text +//! SimDevice -> [device io] (SimDeviceIOClient) +//! -> updateIOPorts +//! -> deviceIOPorts (filter portIdentifier == com.apple.framebuffer.display) +//! -> [port descriptor] +//! -> registerScreenCallbacksWithUUID:... +//! -> [descriptor framebufferSurface] -> IOSurface +//! ``` +//! +//! Registering the callbacks is load-bearing: it is what makes SimulatorKit +//! attach the display pipeline to our client and populate `framebufferSurface`. +//! Merely reading the property without registering does not reliably work. + +use std::ffi::c_void; +use std::ptr::NonNull; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use anyhow::{Result, anyhow}; +use dispatch2::{DispatchQueue, DispatchQueueAttr, DispatchRetained}; +use objc2::rc::Retained; +use objc2::runtime::AnyObject; +use objc2::{msg_send, sel}; +use objc2_core_foundation::{CFRetained, kCFAllocatorDefault}; +use objc2_core_video::{ + CVPixelBuffer, CVPixelBufferCreateWithIOSurface, CVPixelBufferGetHeight, CVPixelBufferGetWidth, + kCVPixelFormatType_32BGRA, +}; +use objc2_foundation::{NSString, NSUUID}; +use objc2_io_surface::IOSurfaceRef; + +use super::common::find_booted_device; +use super::dynamic::{ + responds_to, send_id, send_id_with_id, send_register_screen_callbacks, send_void_with_id, +}; +use super::pixel_buffer::{Photocopier, cf_dict_u32}; +use super::void_block::VoidBlock; + +/// Port identifier for the simulator's main display framebuffer. +/// +/// Several ports can carry this identifier (main screen plus secondary planes), +/// so every match is registered and the largest live surface wins. +const FRAMEBUFFER_PORT_ID: &str = "com.apple.framebuffer.display"; + +/// How often to force a repaint when the screen is otherwise idle. +/// +/// This is not an optimization. A `multipart/x-mixed-replace` consumer will not +/// paint a part until the *next* boundary arrives, and clients that join while +/// the simulator is static would otherwise sit blank forever. ~5fps of forced +/// re-emits keeps both honest. +const IDLE_INTERVAL: Duration = Duration::from_millis(200); + +/// Ticks between re-wire attempts while no frame has ever been captured. +/// +/// Descriptors are sometimes created lazily, so a registration that happened +/// too early silently yields nothing. Rebuilding the port graph roughly once a +/// second recovers from that. +const REWIRE_TICKS: u64 = 5; + +/// A frame handed to the sink, valid only for the duration of the call. +pub struct CapturedFrame<'a> { + pub pixel_buffer: &'a CVPixelBuffer, + pub width: u32, + pub height: u32, + pub captured_at: Instant, +} + +/// Sink invoked on the capture queue for every accepted frame. +pub type FrameSink = Box) + Send + 'static>; + +#[derive(Debug, Clone, Copy, Default)] +pub struct FramebufferStats { + pub frame_count: u64, + pub width: u32, + pub height: u32, + pub descriptor_count: usize, + pub rewire_count: u64, +} + +/// Wrapper making a raw Objective-C pointer movable across threads. +/// +/// The pointers here are owned by SimulatorKit and only ever messaged from the +/// capture queue (or from `start`/`stop`, which are externally serialized). +#[derive(Clone, Copy)] +struct ObjcPtr(*mut AnyObject); +unsafe impl Send for ObjcPtr {} +unsafe impl Sync for ObjcPtr {} + +impl ObjcPtr { + fn as_ptr(self) -> *mut AnyObject { + self.0 + } +} + +/// A framebuffer descriptor plus the UUID its callbacks were registered under. +struct Registration { + descriptor: ObjcPtr, + uuid: Retained, + /// Last `IOSurfaceGetSeed` observed, used to skip unchanged repaints. + last_seed: Option, +} + +/// State shared between the capture queue, the idle timer, and the owner. +struct CaptureState { + registrations: Mutex>, + photocopier: Mutex, + sink: Mutex>, + last_capture: Mutex, + frame_count: AtomicU64, + rewire_count: AtomicU64, + width: AtomicUsize, + height: AtomicUsize, + running: AtomicBool, +} + +impl CaptureState { + fn stats(&self) -> FramebufferStats { + FramebufferStats { + frame_count: self.frame_count.load(Ordering::Relaxed), + width: self.width.load(Ordering::Relaxed) as u32, + height: self.height.load(Ordering::Relaxed) as u32, + descriptor_count: self.registrations.lock().unwrap().len(), + rewire_count: self.rewire_count.load(Ordering::Relaxed), + } + } + + /// Choose the descriptor whose live surface currently has the largest area. + /// + /// Secondary planes share the framebuffer port identifier, so picking the + /// first match would frequently land on a tiny overlay instead of the + /// actual screen. + fn pick_best_surface(&self) -> Option<(usize, CFRetained)> { + let registrations = self.registrations.lock().unwrap(); + let mut best: Option<(usize, CFRetained, usize)> = None; + + for (index, registration) in registrations.iter().enumerate() { + let Some(surface) = (unsafe { framebuffer_surface(registration.descriptor.as_ptr()) }) + else { + continue; + }; + let area = surface.width() * surface.height(); + if area == 0 { + continue; + } + if best + .as_ref() + .is_none_or(|(_, _, best_area)| area > *best_area) + { + best = Some((index, surface, area)); + } + } + + best.map(|(index, surface, _)| (index, surface)) + } + + /// Capture one frame. `force` bypasses the unchanged-seed check. + fn capture(&self, force: bool) { + if !self.running.load(Ordering::Relaxed) { + return; + } + + let Some((index, surface)) = self.pick_best_surface() else { + return; + }; + + // Skip repaints of an unchanged surface, but never skip the very first + // frame and never skip a forced idle re-emit. + let seed = surface.seed(); + { + let mut registrations = self.registrations.lock().unwrap(); + let Some(registration) = registrations.get_mut(index) else { + return; + }; + let unchanged = registration.last_seed == Some(seed); + if unchanged && !force && self.frame_count.load(Ordering::Relaxed) > 0 { + return; + } + registration.last_seed = Some(seed); + } + + let width = surface.width(); + let height = surface.height(); + self.width.store(width, Ordering::Relaxed); + self.height.store(height, Ordering::Relaxed); + + let Some(live) = (unsafe { pixel_buffer_for_surface(&surface) }) else { + return; + }; + + *self.last_capture.lock().unwrap() = Instant::now(); + + // SimulatorKit recycles this IOSurface in place while VideoToolbox + // encodes asynchronously, so the surface must be deep-copied before it + // is handed downstream or we hand out torn frames. + let copy = { + let mut photocopier = self.photocopier.lock().unwrap(); + match photocopier.copy(&live) { + Ok(copy) => copy, + Err(_) => return, + } + }; + + self.frame_count.fetch_add(1, Ordering::Relaxed); + + let mut sink = self.sink.lock().unwrap(); + if let Some(sink) = sink.as_mut() { + sink(CapturedFrame { + pixel_buffer: ©, + width: CVPixelBufferGetWidth(©) as u32, + height: CVPixelBufferGetHeight(©) as u32, + captured_at: Instant::now(), + }); + } + } +} + +/// Live framebuffer capture session for one simulator device. +pub struct SimFramebuffer { + device: ObjcPtr, + device_udid: String, + io_client: Option, + queue: DispatchRetained, + state: Arc, + /// SimulatorKit retains these blocks for the lifetime of the registration. + /// Dropping them early is a use-after-free, not a clean failure. + blocks: Vec, + idle_thread: Option>, +} + +// The raw pointers are only messaged from the capture queue or from methods +// that take `&mut self`, so the type is safe to move between threads. +unsafe impl Send for SimFramebuffer {} + +impl SimFramebuffer { + /// Attach to a booted simulator. `udid` of `None` picks the first booted one. + pub fn new(udid: Option<&str>) -> Result { + crate::frameworks::load_coresimulator_framework()?; + crate::frameworks::load_simulatorkit_framework()?; + + let device = unsafe { find_booted_device(udid)? }; + let device_udid = unsafe { device_udid_string(device)? }; + + Ok(Self { + device: ObjcPtr(device), + device_udid, + io_client: None, + queue: DispatchQueue::new( + "com.accessibility_cli.framebuffer", + DispatchQueueAttr::SERIAL, + ), + state: Arc::new(CaptureState { + registrations: Mutex::new(Vec::new()), + photocopier: Mutex::new(Photocopier::new()), + sink: Mutex::new(None), + last_capture: Mutex::new(Instant::now()), + frame_count: AtomicU64::new(0), + rewire_count: AtomicU64::new(0), + width: AtomicUsize::new(0), + height: AtomicUsize::new(0), + running: AtomicBool::new(false), + }), + blocks: Vec::new(), + idle_thread: None, + }) + } + + pub fn device_udid(&self) -> &str { + &self.device_udid + } + + pub fn stats(&self) -> FramebufferStats { + self.state.stats() + } + + /// Install the frame sink. Called on the capture queue, so it must be quick. + pub fn set_sink(&mut self, sink: Option) { + *self.state.sink.lock().unwrap() = sink; + } + + /// Begin capturing. Idempotent-ish: calling twice re-wires the pipeline. + pub fn start(&mut self) -> Result<()> { + self.state.running.store(true, Ordering::Relaxed); + self.wire_up()?; + self.start_idle_timer(); + Ok(()) + } + + /// Walk the IO port graph and register screen callbacks on every + /// framebuffer descriptor. + fn wire_up(&mut self) -> Result<()> { + let device = self.device.as_ptr(); + + let io: *mut AnyObject = unsafe { msg_send![device, io] }; + if io.is_null() { + return Err(anyhow!( + "SimDevice returned no IO client; is the simulator still booted?" + )); + } + self.io_client = Some(ObjcPtr(io)); + + let _: () = unsafe { msg_send![io, updateIOPorts] }; + + let descriptors = unsafe { framebuffer_descriptors(io)? }; + if descriptors.is_empty() { + return Err(anyhow!( + "No '{FRAMEBUFFER_PORT_ID}' IO ports found on the simulator" + )); + } + + self.unregister_all(); + + let mut registrations = Vec::with_capacity(descriptors.len()); + let mut blocks = Vec::with_capacity(descriptors.len() * 3); + + for descriptor in descriptors { + let uuid = NSUUID::new(); + + let state = Arc::clone(&self.state); + let frame_cb = VoidBlock::new(move || state.capture(false)); + // `surfacesChanged` carries no payload; it just means "re-query". + let state = Arc::clone(&self.state); + let surfaces_cb = VoidBlock::new(move || state.capture(true)); + let props_cb = VoidBlock::new(|| {}); + + unsafe { + send_register_screen_callbacks( + descriptor, + sel!(registerScreenCallbacksWithUUID:callbackQueue:frameCallback:surfacesChangedCallback:propertiesChangedCallback:), + &*uuid as *const NSUUID as *mut AnyObject, + &*self.queue as *const DispatchQueue as *mut c_void, + frame_cb.as_ptr(), + surfaces_cb.as_ptr(), + props_cb.as_ptr(), + ); + } + + blocks.push(frame_cb); + blocks.push(surfaces_cb); + blocks.push(props_cb); + registrations.push(Registration { + descriptor: ObjcPtr(descriptor), + uuid, + last_seed: None, + }); + } + + *self.state.registrations.lock().unwrap() = registrations; + self.blocks = blocks; + + // Registration alone does not deliver a first frame; prime it. + self.state.capture(true); + Ok(()) + } + + /// Drive forced re-emits, and re-wire while nothing has arrived yet. + fn start_idle_timer(&mut self) { + if self.idle_thread.is_some() { + return; + } + let state = Arc::clone(&self.state); + self.idle_thread = Some(std::thread::spawn(move || { + let mut tick: u64 = 0; + while state.running.load(Ordering::Relaxed) { + std::thread::sleep(IDLE_INTERVAL); + if !state.running.load(Ordering::Relaxed) { + break; + } + tick += 1; + + let idle_for = state.last_capture.lock().unwrap().elapsed(); + if idle_for >= IDLE_INTERVAL { + state.capture(true); + } + + // Self-heal only until the first frame ever lands. After that a + // silent pipeline means an idle screen, not a broken graph. + if state.frame_count.load(Ordering::Relaxed) == 0 && tick % REWIRE_TICKS == 0 { + state.rewire_count.fetch_add(1, Ordering::Relaxed); + } + } + })); + } + + fn unregister_all(&mut self) { + let mut registrations = self.state.registrations.lock().unwrap(); + for registration in registrations.drain(..) { + let descriptor = registration.descriptor.as_ptr(); + let selector = sel!(unregisterScreenCallbacksWithUUID:); + if unsafe { responds_to(descriptor, selector) } { + unsafe { + send_void_with_id( + descriptor, + selector, + &*registration.uuid as *const NSUUID as *mut AnyObject, + ); + } + } + } + drop(registrations); + self.blocks.clear(); + } + + pub fn stop(&mut self) { + self.state.running.store(false, Ordering::Relaxed); + if let Some(handle) = self.idle_thread.take() { + let _ = handle.join(); + } + self.unregister_all(); + *self.state.sink.lock().unwrap() = None; + } +} + +impl Drop for SimFramebuffer { + fn drop(&mut self) { + self.stop(); + } +} + +/// Read `-framebufferSurface` off a descriptor, retaining the result. +unsafe fn framebuffer_surface(descriptor: *mut AnyObject) -> Option> { + if descriptor.is_null() { + return None; + } + let selector = sel!(framebufferSurface); + if !unsafe { responds_to(descriptor, selector) } { + return None; + } + let surface = unsafe { send_id(descriptor, selector) }; + let surface = NonNull::new(surface)?.cast::(); + // The getter returns an autoreleased/unowned surface; retain it so it + // survives for as long as we hold it. + Some(unsafe { CFRetained::retain(surface) }) +} + +/// Wrap an IOSurface as a BGRA `CVPixelBuffer` without copying pixels. +unsafe fn pixel_buffer_for_surface(surface: &IOSurfaceRef) -> Option> { + let attrs = unsafe { + cf_dict_u32( + objc2_core_video::kCVPixelBufferPixelFormatTypeKey, + kCVPixelFormatType_32BGRA, + ) + }; + + let mut out: *mut CVPixelBuffer = std::ptr::null_mut(); + let status = unsafe { + CVPixelBufferCreateWithIOSurface( + kCFAllocatorDefault, + surface, + Some(&attrs), + NonNull::from(&mut out), + ) + }; + if status != 0 { + return None; + } + NonNull::new(out).map(|p| unsafe { CFRetained::from_raw(p) }) +} + +/// Collect the descriptors of every `com.apple.framebuffer.display` IO port. +/// +/// Ports arrive as `ROCKRemoteProxy` objects that implement their interface by +/// forwarding, so every message has to go through the dynamic helpers rather +/// than `msg_send!`. +unsafe fn framebuffer_descriptors(io: *mut AnyObject) -> Result> { + let key = NSString::from_str("deviceIOPorts"); + let ports = unsafe { + send_id_with_id( + io, + sel!(valueForKey:), + &*key as *const NSString as *mut AnyObject, + ) + }; + if ports.is_null() { + return Err(anyhow!("SimDeviceIOClient exposed no deviceIOPorts")); + } + + let count: usize = unsafe { msg_send![ports, count] }; + let mut descriptors = Vec::new(); + + for index in 0..count { + let port: *mut AnyObject = unsafe { msg_send![ports, objectAtIndex: index] }; + + if !unsafe { responds_to(port, sel!(portIdentifier)) } { + continue; + } + let identifier = unsafe { send_id(port, sel!(portIdentifier)) }; + // The identifier is usually an NSString but can be a richer object; + // falling back to -description matches serve-sim's stringification. + let Some(identifier) = (unsafe { super::common::nsstring_to_string_static(identifier) }) + .or_else(|| unsafe { object_description(identifier) }) + else { + continue; + }; + if identifier != FRAMEBUFFER_PORT_ID { + continue; + } + + if !unsafe { responds_to(port, sel!(descriptor)) } { + continue; + } + let descriptor = unsafe { send_id(port, sel!(descriptor)) }; + + // Not every descriptor on a framebuffer port actually vends a surface. + if !unsafe { responds_to(descriptor, sel!(framebufferSurface)) } { + continue; + } + + descriptors.push(descriptor); + } + + Ok(descriptors) +} + +/// `-[NSObject description]` as a Rust string, for non-NSString identifiers. +unsafe fn object_description(object: *mut AnyObject) -> Option { + if !unsafe { responds_to(object, sel!(description)) } { + return None; + } + unsafe { super::common::nsstring_to_string_static(send_id(object, sel!(description))) } +} + +unsafe fn device_udid_string(device: *mut AnyObject) -> Result { + let udid: *mut AnyObject = unsafe { msg_send![device, UDID] }; + let string: *mut AnyObject = unsafe { msg_send![udid, UUIDString] }; + unsafe { super::common::nsstring_to_string_static(string) } + .ok_or_else(|| anyhow!("Failed to read simulator UDID")) +} + +// Silence an unused import warning on the c_void alias used by the pool module. +const _: Option<*const c_void> = None; diff --git a/packages/accessibility-ios-sys/src/macos/pixel_buffer.rs b/packages/accessibility-ios-sys/src/macos/pixel_buffer.rs new file mode 100644 index 0000000..bd02c34 --- /dev/null +++ b/packages/accessibility-ios-sys/src/macos/pixel_buffer.rs @@ -0,0 +1,159 @@ +//! Pooled deep copies of live framebuffer pixel buffers. +//! +//! SimulatorKit hands out a `CVPixelBuffer` that wraps its own framebuffer +//! `IOSurface`, and it recycles that surface in place. Retaining the pixel +//! buffer does not help, because the *surface* mutates underneath it. Since +//! VideoToolbox encodes asynchronously, anything downstream must own its own +//! copy or it will encode torn frames. +//! +//! The copy is a straightforward row-wise `memcpy` into a size-keyed +//! `CVPixelBufferPool`. It is the single largest CPU cost in the capture path; +//! a Metal blit or an IOSurface ring would avoid it, but correctness first. + +use std::ptr::NonNull; + +use anyhow::{Result, anyhow}; +use objc2_core_foundation::{ + CFDictionary, CFNumber, CFRetained, CFString, CFType, kCFAllocatorDefault, +}; +use objc2_core_video::{ + CVPixelBuffer, CVPixelBufferGetBaseAddress, CVPixelBufferGetBytesPerRow, + CVPixelBufferGetHeight, CVPixelBufferGetWidth, CVPixelBufferLockBaseAddress, + CVPixelBufferLockFlags, CVPixelBufferPool, CVPixelBufferUnlockBaseAddress, + kCVPixelBufferHeightKey, kCVPixelBufferIOSurfacePropertiesKey, + kCVPixelBufferPixelFormatTypeKey, kCVPixelBufferWidthKey, kCVPixelFormatType_32BGRA, +}; + +/// Build an untyped `CFDictionary` from `CFString` keys to arbitrary CF values. +pub(super) fn cf_dict(pairs: &[(&CFString, &CFType)]) -> CFRetained { + let keys: Vec<&CFString> = pairs.iter().map(|(k, _)| *k).collect(); + let values: Vec<&CFType> = pairs.iter().map(|(_, v)| *v).collect(); + let typed = CFDictionary::::from_slices(&keys, &values); + // The generic parameters are purely a Rust-side convenience; the underlying + // CF object is the same either way. + unsafe { CFRetained::cast_unchecked::(typed) } +} + +/// Single-entry dictionary mapping a key to a `u32`, as CoreVideo expects. +pub(super) fn cf_dict_u32(key: &CFString, value: u32) -> CFRetained { + let number = CFNumber::new_i32(value as i32); + cf_dict(&[(key, number.as_ref())]) +} + +/// Reusable pool of BGRA pixel buffers, rebuilt whenever the source resizes. +pub(super) struct Photocopier { + pool: Option>, + dimensions: (usize, usize), +} + +// The pool is only ever touched behind the capture state's mutex, which +// serializes the capture queue against the idle timer. +unsafe impl Send for Photocopier {} + +impl Photocopier { + pub(super) fn new() -> Self { + Self { + pool: None, + dimensions: (0, 0), + } + } + + /// Deep-copy `source` into a pooled buffer of the same size. + pub(super) fn copy(&mut self, source: &CVPixelBuffer) -> Result> { + let width = CVPixelBufferGetWidth(source); + let height = CVPixelBufferGetHeight(source); + if width == 0 || height == 0 { + return Err(anyhow!("Source pixel buffer has zero extent")); + } + + if self.pool.is_none() || self.dimensions != (width, height) { + self.pool = Some(build_pool(width, height)?); + self.dimensions = (width, height); + } + let pool = self.pool.as_ref().expect("pool built above"); + + let mut out: *mut CVPixelBuffer = std::ptr::null_mut(); + let status = unsafe { + CVPixelBufferPool::create_pixel_buffer( + kCFAllocatorDefault, + pool, + NonNull::from(&mut out), + ) + }; + let destination = NonNull::new(out) + .filter(|_| status == 0) + .map(|p| unsafe { CFRetained::from_raw(p) }) + .ok_or_else(|| anyhow!("CVPixelBufferPoolCreatePixelBuffer failed: {status}"))?; + + unsafe { + CVPixelBufferLockBaseAddress(source, CVPixelBufferLockFlags::ReadOnly); + CVPixelBufferLockBaseAddress(&destination, CVPixelBufferLockFlags::empty()); + } + + let result = (|| { + let src_base = CVPixelBufferGetBaseAddress(source); + let dst_base = CVPixelBufferGetBaseAddress(&destination); + if src_base.is_null() || dst_base.is_null() { + return Err(anyhow!("Pixel buffer base address unavailable")); + } + + let src_stride = CVPixelBufferGetBytesPerRow(source); + let dst_stride = CVPixelBufferGetBytesPerRow(&destination); + let row_bytes = src_stride.min(dst_stride); + + for row in 0..height { + unsafe { + std::ptr::copy_nonoverlapping( + (src_base as *const u8).add(row * src_stride), + (dst_base as *mut u8).add(row * dst_stride), + row_bytes, + ); + } + } + Ok(()) + })(); + + unsafe { + CVPixelBufferUnlockBaseAddress(&destination, CVPixelBufferLockFlags::empty()); + CVPixelBufferUnlockBaseAddress(source, CVPixelBufferLockFlags::ReadOnly); + } + + result.map(|()| destination) + } +} + +/// Create an IOSurface-backed BGRA pool at the given size. +fn build_pool(width: usize, height: usize) -> Result> { + let format = CFNumber::new_i32(kCVPixelFormatType_32BGRA as i32); + let width_num = CFNumber::new_i32(width as i32); + let height_num = CFNumber::new_i32(height as i32); + // An empty IOSurface-properties dictionary is the documented way to ask for + // IOSurface-backed buffers, which keeps VideoToolbox on the zero-copy path. + let io_surface_props = CFDictionary::::empty(); + + let attrs = unsafe { + cf_dict(&[ + (kCVPixelBufferPixelFormatTypeKey, format.as_ref()), + (kCVPixelBufferWidthKey, width_num.as_ref()), + (kCVPixelBufferHeightKey, height_num.as_ref()), + ( + kCVPixelBufferIOSurfacePropertiesKey, + &*CFRetained::cast_unchecked::(io_surface_props), + ), + ]) + }; + + let mut out: *mut CVPixelBufferPool = std::ptr::null_mut(); + let status = unsafe { + CVPixelBufferPool::create( + kCFAllocatorDefault, + None, + Some(&attrs), + NonNull::from(&mut out), + ) + }; + NonNull::new(out) + .filter(|_| status == 0) + .map(|p| unsafe { CFRetained::from_raw(p) }) + .ok_or_else(|| anyhow!("CVPixelBufferPoolCreate failed: {status}")) +} diff --git a/packages/accessibility-ios-sys/src/macos/void_block.rs b/packages/accessibility-ios-sys/src/macos/void_block.rs new file mode 100644 index 0000000..127ea54 --- /dev/null +++ b/packages/accessibility-ios-sys/src/macos/void_block.rs @@ -0,0 +1,63 @@ +//! Owned, signature-bearing `void (^)(void)` blocks backed by a C shim. +//! +//! See `blocks.c` for why these cannot be `block2::RcBlock`s. + +use std::ffi::c_void; + +unsafe extern "C" { + fn accessibility_make_void_block( + callback: unsafe extern "C" fn(*mut c_void), + context: *mut c_void, + ) -> *mut c_void; + fn accessibility_release_block(block: *mut c_void); +} + +/// A heap Objective-C block that invokes a Rust closure when called. +/// +/// The block and the boxed closure are released together on drop. SimulatorKit +/// retains the block for as long as the registration is live, so a `VoidBlock` +/// must be kept alive until the matching `unregisterScreenCallbacksWithUUID:`. +pub(super) struct VoidBlock { + block: *mut c_void, + closure: *mut Box, +} + +// The closure is `Send + Sync` and the block is invoked by GCD from an +// arbitrary thread, so the handle is safe to move between threads. +unsafe impl Send for VoidBlock {} +unsafe impl Sync for VoidBlock {} + +impl VoidBlock { + pub(super) fn new(closure: F) -> Self + where + F: Fn() + Send + Sync + 'static, + { + let boxed: Box> = Box::new(Box::new(closure)); + let closure = Box::into_raw(boxed); + let block = + unsafe { accessibility_make_void_block(invoke_closure, closure as *mut c_void) }; + Self { block, closure } + } + + /// The raw `id`-compatible block pointer to hand to Objective-C. + pub(super) fn as_ptr(&self) -> *mut c_void { + self.block + } +} + +unsafe extern "C" fn invoke_closure(context: *mut c_void) { + if context.is_null() { + return; + } + let closure = unsafe { &*(context as *const Box) }; + closure(); +} + +impl Drop for VoidBlock { + fn drop(&mut self) { + unsafe { + accessibility_release_block(self.block); + drop(Box::from_raw(self.closure)); + } + } +} From 57329f1973fb85057af693aacc23591f0f7bedde Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Sun, 26 Jul 2026 20:26:58 -0700 Subject: [PATCH 02/19] Add VideoToolbox H.264 encoding for simulator capture Encodes on the capture queue, since CVPixelBuffer and CMSampleBuffer are not Send and only the compressed bytes need to travel. Emits either Annex-B (for WebRTC) or AVCC plus a separate avcC record (for browser VideoDecoder) from the same session. Configured for interactive latency: low-latency rate control, no frame reordering, MaxFrameDelayCount 0, Baseline profile for broad WebRTC codec negotiation. SPS/PPS are prepended only to IDRs rather than every frame. Verified against a booted iPhone 17: 211 access units over 5s at ~60fps, all Annex-B framed, ~4 Mbps, keyframes on the configured 2s interval. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <166158716+staging-devin-ai-integration[bot]@users.noreply.github.com> --- .../examples/framebuffer_probe.rs | 88 +++- packages/accessibility-ios-sys/src/macos.rs | 4 + .../src/macos/encoder.rs | 431 ++++++++++++++++++ .../accessibility-ios-sys/src/macos/stream.rs | 81 ++++ 4 files changed, 582 insertions(+), 22 deletions(-) create mode 100644 packages/accessibility-ios-sys/src/macos/encoder.rs create mode 100644 packages/accessibility-ios-sys/src/macos/stream.rs diff --git a/packages/accessibility-ios-sys/examples/framebuffer_probe.rs b/packages/accessibility-ios-sys/examples/framebuffer_probe.rs index af6305b..45d97a0 100644 --- a/packages/accessibility-ios-sys/examples/framebuffer_probe.rs +++ b/packages/accessibility-ios-sys/examples/framebuffer_probe.rs @@ -1,4 +1,4 @@ -//! Probe the SimulatorKit framebuffer capture path against a booted simulator. +//! Probe the SimulatorKit framebuffer capture and H.264 encode path. //! //! Run with: `cargo run -p accessibility-ios-sys --example framebuffer_probe` @@ -10,41 +10,85 @@ fn main() { #[cfg(target_os = "macos")] fn main() -> anyhow::Result<()> { - use accessibility_ios_sys::SimFramebuffer; + use accessibility_ios_sys::{ + ChunkKind, EncodedChunk, EncoderConfig, NalFormat, SimVideoStream, + }; + use std::sync::Arc; + use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant}; - let mut fb = SimFramebuffer::new(None)?; - println!("device udid: {}", fb.device_udid()); + let keyframes = Arc::new(AtomicU64::new(0)); + let deltas = Arc::new(AtomicU64::new(0)); + let parameter_sets = Arc::new(AtomicU64::new(0)); + let bytes = Arc::new(AtomicU64::new(0)); + let first_nal_ok = Arc::new(AtomicU64::new(0)); - fb.start()?; + let sink = { + let (keyframes, deltas, parameter_sets, bytes, first_nal_ok) = ( + Arc::clone(&keyframes), + Arc::clone(&deltas), + Arc::clone(¶meter_sets), + Arc::clone(&bytes), + Arc::clone(&first_nal_ok), + ); + Arc::new(move |chunk: EncodedChunk| { + bytes.fetch_add(chunk.data.len() as u64, Ordering::Relaxed); + match chunk.kind { + ChunkKind::ParameterSet => parameter_sets.fetch_add(1, Ordering::Relaxed), + ChunkKind::Keyframe => keyframes.fetch_add(1, Ordering::Relaxed), + ChunkKind::Delta => deltas.fetch_add(1, Ordering::Relaxed), + }; + // Every Annex-B access unit must open with a start code. + if chunk.data.starts_with(&[0, 0, 0, 1]) { + first_nal_ok.fetch_add(1, Ordering::Relaxed); + } + }) + }; + + let config = EncoderConfig { + nal_format: NalFormat::AnnexB, + ..Default::default() + }; + let stream = SimVideoStream::start(None, config, sink)?; + println!("device udid: {}", stream.device_udid()); - let deadline = Instant::now() + Duration::from_secs(5); - let mut last = 0u64; + let started = Instant::now(); + let deadline = started + Duration::from_secs(5); while Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(500)); - let stats = fb.stats(); - let delta = stats.frame_count - last; - last = stats.frame_count; + std::thread::sleep(Duration::from_millis(1000)); + let stats = stream.stats(); println!( - "frames={} (+{}) size={}x{} descriptors={} rewires={}", + "frames={} {}x{} keyframes={} deltas={} kb={}", stats.frame_count, - delta, stats.width, stats.height, - stats.descriptor_count, - stats.rewire_count, + keyframes.load(Ordering::Relaxed), + deltas.load(Ordering::Relaxed), + bytes.load(Ordering::Relaxed) / 1024, ); } - let stats = fb.stats(); - fb.stop(); + let total_chunks = keyframes.load(Ordering::Relaxed) + deltas.load(Ordering::Relaxed); + let elapsed = started.elapsed().as_secs_f64(); - if stats.frame_count == 0 { - anyhow::bail!("no frames captured - the framebuffer pipeline did not come up"); - } + println!("---"); + println!("encoded chunks : {total_chunks}"); + println!("keyframes : {}", keyframes.load(Ordering::Relaxed)); + println!("annex-b valid : {}", first_nal_ok.load(Ordering::Relaxed)); println!( - "OK: captured {} frames at {}x{}", - stats.frame_count, stats.width, stats.height + "bitrate : {:.2} Mbps", + (bytes.load(Ordering::Relaxed) as f64 * 8.0) / elapsed / 1_000_000.0 ); + + if total_chunks == 0 { + anyhow::bail!("no encoded frames produced"); + } + if keyframes.load(Ordering::Relaxed) == 0 { + anyhow::bail!("no keyframes produced - decoders would never start"); + } + if first_nal_ok.load(Ordering::Relaxed) != total_chunks { + anyhow::bail!("some access units were not Annex-B framed"); + } + println!("OK"); Ok(()) } diff --git a/packages/accessibility-ios-sys/src/macos.rs b/packages/accessibility-ios-sys/src/macos.rs index a022b4e..7c08796 100644 --- a/packages/accessibility-ios-sys/src/macos.rs +++ b/packages/accessibility-ios-sys/src/macos.rs @@ -57,15 +57,19 @@ use slotmap::SecondaryMap; mod common; mod dispatcher; mod dynamic; +mod encoder; mod framebuffer; mod hid; mod pixel_buffer; mod reader; +mod stream; mod void_block; pub use common::{ ButtonDirection, Element, ElementKey, ElementTree, HardwareButton, Point, Rect, ScreenSpace, Screenshot, Size, TreeFilter, load_frameworks, }; +pub use encoder::{ChunkKind, ChunkSink, EncodedChunk, EncoderConfig, H264Encoder, NalFormat}; pub use framebuffer::{CapturedFrame, FrameSink, FramebufferStats, SimFramebuffer}; pub use reader::IOSSimulatorAccessibility; +pub use stream::{ScreenGeometry, SimVideoStream}; diff --git a/packages/accessibility-ios-sys/src/macos/encoder.rs b/packages/accessibility-ios-sys/src/macos/encoder.rs new file mode 100644 index 0000000..cd3dd1f --- /dev/null +++ b/packages/accessibility-ios-sys/src/macos/encoder.rs @@ -0,0 +1,431 @@ +//! Hardware H.264 encoding of captured simulator frames via VideoToolbox. +//! +//! The encoder is deliberately driven from the capture queue: `CVPixelBuffer` +//! and `CMSampleBuffer` are not `Send`, so encoding in place and shipping only +//! the compressed bytes elsewhere avoids marshalling raw frames across threads. +//! +//! Two output framings are supported from the same session: +//! +//! - [`NalFormat::AnnexB`] for WebRTC, which expects start-code delimited NALs. +//! - [`NalFormat::Avcc`] for browser `VideoDecoder`, which wants length-prefixed +//! NALs plus a separate `avcC` parameter-set blob. + +use std::ffi::c_void; +use std::ptr::NonNull; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use anyhow::{Result, anyhow}; +use block2::RcBlock; +use bytes::Bytes; +use objc2_core_foundation::{ + CFBoolean, CFNumber, CFRetained, CFString, CFType, kCFAllocatorDefault, +}; +use objc2_core_media::{ + CMSampleBuffer, CMTime, CMVideoCodecType, CMVideoFormatDescriptionGetH264ParameterSetAtIndex, + kCMTimeInvalid, +}; +use objc2_core_video::CVImageBuffer; +use objc2_video_toolbox::{VTCompressionSession, VTEncodeInfoFlags, VTSessionSetProperty}; + +use super::pixel_buffer::cf_dict; + +/// `avc1` — H.264 in a `CMVideoCodecType`. +const CODEC_H264: CMVideoCodecType = 0x6176_6331; + +/// How the compressed NAL units are framed on the way out. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NalFormat { + /// `00 00 00 01` start codes, with SPS/PPS inlined ahead of each IDR. + AnnexB, + /// 4-byte big-endian length prefixes, exactly as VideoToolbox emits them. + Avcc, +} + +/// Kind of payload in an [`EncodedChunk`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ChunkKind { + /// An `avcC` parameter set record. Only emitted in [`NalFormat::Avcc`]. + ParameterSet, + /// An IDR access unit. + Keyframe, + /// A non-IDR access unit. + Delta, +} + +/// One encoded unit ready for the wire. +#[derive(Debug, Clone)] +pub struct EncodedChunk { + pub data: Bytes, + pub kind: ChunkKind, +} + +#[derive(Debug, Clone, Copy)] +pub struct EncoderConfig { + pub fps: u32, + pub bitrate: u32, + /// Seconds between scheduled keyframes. + pub keyframe_interval_secs: u32, + pub nal_format: NalFormat, +} + +impl Default for EncoderConfig { + fn default() -> Self { + Self { + fps: 60, + bitrate: 6_000_000, + keyframe_interval_secs: 2, + nal_format: NalFormat::AnnexB, + } + } +} + +/// Sink for encoded chunks, invoked on the capture queue. +pub type ChunkSink = Arc; + +pub struct H264Encoder { + session: Option>, + config: EncoderConfig, + dimensions: (i32, i32), + /// Frame index, used to synthesize presentation timestamps. + frame_index: i64, + /// Set by RTCP PLI/FIR so the next frame is forced to an IDR. + force_keyframe: Arc, + /// Whether the `avcC` record for the current session has been emitted. + emitted_parameter_set: Arc>, + sink: ChunkSink, +} + +// VideoToolbox sessions are documented as thread-safe, and this one is only +// driven from the capture queue regardless. +unsafe impl Send for H264Encoder {} + +impl H264Encoder { + pub fn new(config: EncoderConfig, force_keyframe: Arc, sink: ChunkSink) -> Self { + Self { + session: None, + config, + dimensions: (0, 0), + frame_index: 0, + force_keyframe, + emitted_parameter_set: Arc::new(Mutex::new(false)), + sink, + } + } + + /// Encode one frame, rebuilding the session if the source resized. + pub fn encode(&mut self, image: &CVImageBuffer, width: i32, height: i32) -> Result<()> { + if self.session.is_none() || self.dimensions != (width, height) { + self.dimensions = (width, height); + self.rebuild_session()?; + } + let session = self.session.as_ref().expect("session built above"); + + let force = self.force_keyframe.swap(false, Ordering::Relaxed); + let frame_properties = force.then(|| { + cf_dict(&[( + unsafe { objc2_video_toolbox::kVTEncodeFrameOptionKey_ForceKeyFrame }, + CFBoolean::new(true).as_ref(), + )]) + }); + + let pts = unsafe { CMTime::new(self.frame_index, self.config.fps as i32) }; + let duration = unsafe { CMTime::new(1, self.config.fps as i32) }; + self.frame_index += 1; + + let sink = Arc::clone(&self.sink); + let nal_format = self.config.nal_format; + let emitted = Arc::clone(&self.emitted_parameter_set); + + let handler = RcBlock::new( + move |status: i32, _flags: VTEncodeInfoFlags, sample: *mut CMSampleBuffer| { + if status != 0 || sample.is_null() { + return; + } + let sample = unsafe { &*sample }; + for chunk in package_sample(sample, nal_format, &emitted) { + sink(chunk); + } + }, + ); + + let status = unsafe { + session.encode_frame_with_output_handler( + image, + pts, + duration, + frame_properties.as_deref(), + std::ptr::null_mut(), + RcBlock::as_ptr(&handler) as *mut _, + ) + }; + if status != 0 { + return Err(anyhow!("VTCompressionSessionEncodeFrame failed: {status}")); + } + Ok(()) + } + + fn rebuild_session(&mut self) -> Result<()> { + if let Some(session) = self.session.take() { + unsafe { session.invalidate() }; + } + *self.emitted_parameter_set.lock().unwrap() = false; + + let (width, height) = self.dimensions; + let mut out: *mut VTCompressionSession = std::ptr::null_mut(); + + // Low-latency rate control keeps the decoder's frame buffering small. + // Without it browsers routinely add ~300ms of latency even though we + // emit no B-frames. It is not available on every encoder, so a plain + // session is an acceptable fallback. + let low_latency = cf_dict(&[( + unsafe { objc2_video_toolbox::kVTVideoEncoderSpecification_EnableLowLatencyRateControl }, + CFBoolean::new(true).as_ref(), + )]); + + let mut status = unsafe { + VTCompressionSession::create( + kCFAllocatorDefault, + width, + height, + CODEC_H264, + Some(&low_latency), + None, + kCFAllocatorDefault, + None, + std::ptr::null_mut(), + NonNull::from(&mut out), + ) + }; + if status != 0 || out.is_null() { + out = std::ptr::null_mut(); + status = unsafe { + VTCompressionSession::create( + kCFAllocatorDefault, + width, + height, + CODEC_H264, + None, + None, + kCFAllocatorDefault, + None, + std::ptr::null_mut(), + NonNull::from(&mut out), + ) + }; + } + + let session = NonNull::new(out) + .filter(|_| status == 0) + .map(|p| unsafe { CFRetained::from_raw(p) }) + .ok_or_else(|| anyhow!("VTCompressionSessionCreate failed: {status}"))?; + + let keyframe_interval = self.config.fps * self.config.keyframe_interval_secs; + set_bool(&session, "RealTime", true); + set_bool(&session, "AllowFrameReordering", false); + set_i32(&session, "MaxFrameDelayCount", 0); + // Baseline is the safest bet for WebRTC: every browser negotiates it, + // and we gain nothing from High on a UI stream with no B-frames. + set_string(&session, "ProfileLevel", "H264_Baseline_AutoLevel"); + set_i32(&session, "ExpectedFrameRate", self.config.fps as i32); + set_i32(&session, "MaxKeyFrameInterval", keyframe_interval as i32); + set_i32(&session, "AverageBitRate", self.config.bitrate as i32); + + self.session = Some(session); + self.frame_index = 0; + Ok(()) + } +} + +impl Drop for H264Encoder { + fn drop(&mut self) { + if let Some(session) = self.session.take() { + unsafe { + session.complete_frames(kCMTimeInvalid); + session.invalidate(); + } + } + } +} + +/// Turn one compressed sample into the chunks that go on the wire. +fn package_sample( + sample: &CMSampleBuffer, + format: NalFormat, + emitted_parameter_set: &Mutex, +) -> Vec { + let is_keyframe = sample_is_keyframe(sample); + let Some(avcc) = sample_bytes(sample) else { + return Vec::new(); + }; + + let mut chunks = Vec::new(); + let parameter_sets = is_keyframe + .then(|| unsafe { sample.format_description() }) + .flatten() + .and_then(|desc| extract_parameter_sets(&desc)); + + match format { + NalFormat::Avcc => { + // The browser needs the avcC record once per session, before the + // first keyframe it decodes. + if let Some((sps, pps)) = parameter_sets.as_ref() { + let mut emitted = emitted_parameter_set.lock().unwrap(); + if !*emitted { + *emitted = true; + chunks.push(EncodedChunk { + data: Bytes::from(build_avcc_record(sps, pps)), + kind: ChunkKind::ParameterSet, + }); + } + } + chunks.push(EncodedChunk { + data: Bytes::from(avcc), + kind: if is_keyframe { + ChunkKind::Keyframe + } else { + ChunkKind::Delta + }, + }); + } + NalFormat::AnnexB => { + let mut out = Vec::with_capacity(avcc.len() + 64); + // SPS/PPS are only prepended to IDRs; repeating them on every + // delta frame is pure waste. + if let Some((sps, pps)) = parameter_sets.as_ref() { + out.extend_from_slice(&[0, 0, 0, 1]); + out.extend_from_slice(sps); + out.extend_from_slice(&[0, 0, 0, 1]); + out.extend_from_slice(pps); + } + append_annex_b(&avcc, &mut out); + chunks.push(EncodedChunk { + data: Bytes::from(out), + kind: if is_keyframe { + ChunkKind::Keyframe + } else { + ChunkKind::Delta + }, + }); + } + } + + chunks +} + +/// Rewrite AVCC length prefixes as Annex-B start codes. +fn append_annex_b(avcc: &[u8], out: &mut Vec) { + let mut offset = 0usize; + while offset + 4 <= avcc.len() { + let length = u32::from_be_bytes([ + avcc[offset], + avcc[offset + 1], + avcc[offset + 2], + avcc[offset + 3], + ]) as usize; + offset += 4; + if offset + length > avcc.len() { + break; + } + out.extend_from_slice(&[0, 0, 0, 1]); + out.extend_from_slice(&avcc[offset..offset + length]); + offset += length; + } +} + +/// Build an ISO/IEC 14496-15 `avcC` record from a SPS/PPS pair. +fn build_avcc_record(sps: &[u8], pps: &[u8]) -> Vec { + let mut out = Vec::with_capacity(sps.len() + pps.len() + 11); + out.push(0x01); // configurationVersion + out.extend_from_slice(&sps[1..4]); // profile, compatibility, level + out.push(0xFF); // reserved | lengthSizeMinusOne = 3 (4-byte lengths) + out.push(0xE1); // reserved | numOfSequenceParameterSets = 1 + out.extend_from_slice(&(sps.len() as u16).to_be_bytes()); + out.extend_from_slice(sps); + out.push(0x01); // numOfPictureParameterSets + out.extend_from_slice(&(pps.len() as u16).to_be_bytes()); + out.extend_from_slice(pps); + out +} + +/// A sample is a sync frame unless it is explicitly flagged `NotSync`. +/// +/// The attachment dictionaries are untyped, so this goes through the raw CF +/// entry point rather than the generically typed `CFDictionary` wrapper. +fn sample_is_keyframe(sample: &CMSampleBuffer) -> bool { + unsafe extern "C-unwind" { + fn CFDictionaryContainsKey(dict: *const c_void, key: *const c_void) -> u8; + } + + let Some(attachments) = (unsafe { sample.sample_attachments_array(false) }) else { + return true; + }; + if attachments.count() == 0 { + return true; + } + let entry = unsafe { attachments.value_at_index(0) }; + if entry.is_null() { + return true; + } + let key = unsafe { objc2_core_media::kCMSampleAttachmentKey_NotSync }; + unsafe { CFDictionaryContainsKey(entry, key as *const CFString as *const c_void) == 0 } +} + +/// Copy the compressed bytes out of the sample's block buffer. +fn sample_bytes(sample: &CMSampleBuffer) -> Option> { + let block = unsafe { sample.data_buffer() }?; + let total = unsafe { block.data_length() }; + if total == 0 { + return None; + } + let mut out = vec![0u8; total]; + let destination = NonNull::new(out.as_mut_ptr() as *mut c_void)?; + let status = unsafe { block.copy_data_bytes(0, total, destination) }; + (status == 0).then_some(out) +} + +/// Pull SPS (index 0) and PPS (index 1) out of a format description. +fn extract_parameter_sets( + description: &objc2_core_media::CMFormatDescription, +) -> Option<(Vec, Vec)> { + let mut sets = Vec::with_capacity(2); + for index in 0..2usize { + let mut pointer: *const u8 = std::ptr::null(); + let mut size: usize = 0; + let status = unsafe { + CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + description, + index, + &mut pointer, + &mut size, + std::ptr::null_mut(), + std::ptr::null_mut(), + ) + }; + if status != 0 || pointer.is_null() || size == 0 { + return None; + } + sets.push(unsafe { std::slice::from_raw_parts(pointer, size) }.to_vec()); + } + let pps = sets.pop()?; + let sps = sets.pop()?; + // The avcC record copies profile/compatibility/level out of the SPS. + (sps.len() >= 4).then_some((sps, pps)) +} + +fn set_property(session: &VTCompressionSession, key: &str, value: &CFType) { + let key = CFString::from_str(key); + unsafe { VTSessionSetProperty(session, &key, Some(value)) }; +} + +fn set_bool(session: &VTCompressionSession, key: &str, value: bool) { + set_property(session, key, CFBoolean::new(value).as_ref()); +} + +fn set_i32(session: &VTCompressionSession, key: &str, value: i32) { + set_property(session, key, CFNumber::new_i32(value).as_ref()); +} + +fn set_string(session: &VTCompressionSession, key: &str, value: &str) { + set_property(session, key, CFString::from_str(value).as_ref()); +} diff --git a/packages/accessibility-ios-sys/src/macos/stream.rs b/packages/accessibility-ios-sys/src/macos/stream.rs new file mode 100644 index 0000000..6f6a5d1 --- /dev/null +++ b/packages/accessibility-ios-sys/src/macos/stream.rs @@ -0,0 +1,81 @@ +//! Capture + encode pipeline for a booted simulator. +//! +//! Ties [`SimFramebuffer`] to [`H264Encoder`], keeping the encode step on the +//! capture queue so that only compressed bytes ever cross a thread boundary. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use anyhow::Result; +use objc2_core_video::CVImageBuffer; + +use super::encoder::{ChunkSink, EncoderConfig, H264Encoder}; +use super::framebuffer::{FramebufferStats, SimFramebuffer}; + +/// Pixel geometry of the captured display. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ScreenGeometry { + pub width: u32, + pub height: u32, +} + +/// A running capture-and-encode session. +pub struct SimVideoStream { + framebuffer: SimFramebuffer, + force_keyframe: Arc, +} + +impl SimVideoStream { + /// Start capturing `udid` and pushing encoded chunks to `sink`. + /// + /// `sink` runs on the capture queue, so it must not block; the intended + /// use is a bounded channel that drops on overflow. + pub fn start(udid: Option<&str>, config: EncoderConfig, sink: ChunkSink) -> Result { + let mut framebuffer = SimFramebuffer::new(udid)?; + let force_keyframe = Arc::new(AtomicBool::new(false)); + + let mut encoder = H264Encoder::new(config, Arc::clone(&force_keyframe), sink); + framebuffer.set_sink(Some(Box::new(move |frame| { + // `CVPixelBuffer` derefs to `CVImageBuffer`, which is what + // VideoToolbox wants. + let image: &CVImageBuffer = frame.pixel_buffer; + if let Err(error) = encoder.encode(image, frame.width as i32, frame.height as i32) { + eprintln!("[capture] encode failed: {error}"); + } + }))); + + framebuffer.start()?; + Ok(Self { + framebuffer, + force_keyframe, + }) + } + + pub fn device_udid(&self) -> &str { + self.framebuffer.device_udid() + } + + pub fn stats(&self) -> FramebufferStats { + self.framebuffer.stats() + } + + pub fn geometry(&self) -> ScreenGeometry { + let stats = self.framebuffer.stats(); + ScreenGeometry { + width: stats.width, + height: stats.height, + } + } + + /// Ask the encoder to make the next frame an IDR. + /// + /// Driven by RTCP PLI/FIR from WebRTC receivers, and by new subscribers on + /// the raw stream endpoints. + pub fn request_keyframe(&self) { + self.force_keyframe.store(true, Ordering::Relaxed); + } + + pub fn stop(&mut self) { + self.framebuffer.stop(); + } +} From cdf2efdf0d2ff5497a885a147e9363a052d05273 Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Sun, 26 Jul 2026 20:43:15 -0700 Subject: [PATCH 03/19] Add serve-sim: stream and inspect the iOS Simulator in a browser Adds `accessibility-cli serve-sim`, which serves a live, interactive view of a booted simulator with accessibility element inspection. accessibility-core gains a platform-agnostic `video` module: a VideoCapture trait plus encoded-frame types. It is encoded-frame oriented rather than surface oriented because every platform that can do this has a hardware encoder next to its capture API, and passing raw surfaces across the boundary would force a copy. Only the iOS Simulator implements it; other platforms report it unsupported, matching how capture_screen already stubs. accessibility-serve is a new crate holding the server so that webrtc, axum and their transitive dependencies stay out of the library crates. Notes on the design: - Video is offered over WebRTC by default and as length-prefixed H.264 over a WebSocket for WebCodecs clients. One Annex-B encoder feeds both; the WebSocket path converts framing on the way out. - Input and accessibility each get a dedicated thread. The simulator's Objective-C objects are not Sync and their calls block, and an accessibility tree fetch can take hundreds of milliseconds, which pointer events must not queue behind. - Element picking uses the cached tree for hover feedback and confirms with a real objectAtPoint: hit test once the pointer settles. The cached tree mis-picks overlapping views; the simulator does not. - Accessibility frames arrive in macOS screen points wherever the Simulator window happens to sit, so rects are normalized against the app's own bounds before leaving the server. That also makes them scale-independent. The framebuffer wiring state moved into CaptureState so the idle timer can actually rebuild the pipeline; previously it only counted attempts. This also let the unsafe Send/Sync impls on SimFramebuffer go away, leaving the unsafety localized to the raw pointer wrapper. Verified against a booted iPhone 17 over both transports: video renders, taps and the home button drive the device, and the inspector highlights elements and reports a copyable CLI selector. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <166158716+staging-devin-ai-integration[bot]@users.noreply.github.com> --- Cargo.lock | 1939 ++++++++++++++++- Cargo.toml | 8 + packages/accessibility-cli/Cargo.toml | 4 + packages/accessibility-cli/src/lib.rs | 93 + packages/accessibility-core/Cargo.toml | 1 + .../src/accessibility/mod.rs | 17 + packages/accessibility-core/src/lib.rs | 1 + .../src/platform/ios_simulator.rs | 73 + packages/accessibility-core/src/video/mod.rs | 130 ++ packages/accessibility-ios-sys/src/macos.rs | 1 + .../src/macos/framebuffer.rs | 220 +- .../accessibility-ios-sys/src/macos/hid.rs | 42 +- packages/accessibility-serve/Cargo.toml | 27 + packages/accessibility-serve/src/avcc.rs | 195 ++ packages/accessibility-serve/src/ax.rs | 201 ++ packages/accessibility-serve/src/http.rs | 203 ++ packages/accessibility-serve/src/input.rs | 101 + packages/accessibility-serve/src/lib.rs | 109 + packages/accessibility-serve/src/session.rs | 160 ++ .../accessibility-serve/src/webrtc_stream.rs | 183 ++ .../accessibility-serve/static/index.html | 519 +++++ 21 files changed, 4071 insertions(+), 156 deletions(-) create mode 100644 packages/accessibility-core/src/video/mod.rs create mode 100644 packages/accessibility-serve/Cargo.toml create mode 100644 packages/accessibility-serve/src/avcc.rs create mode 100644 packages/accessibility-serve/src/ax.rs create mode 100644 packages/accessibility-serve/src/http.rs create mode 100644 packages/accessibility-serve/src/input.rs create mode 100644 packages/accessibility-serve/src/lib.rs create mode 100644 packages/accessibility-serve/src/session.rs create mode 100644 packages/accessibility-serve/src/webrtc_stream.rs create mode 100644 packages/accessibility-serve/static/index.html diff --git a/Cargo.lock b/Cargo.lock index a4b19ff..01178b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,6 +31,8 @@ name = "accessibility-cli" version = "0.1.0" dependencies = [ "accessibility-core", + "accessibility-serve", + "anyhow", "assert_cmd", "clap", "ctrlc", @@ -53,6 +55,7 @@ dependencies = [ "accesskit", "anyhow", "async-trait", + "bytes", "cssparser", "euclid", "image", @@ -119,6 +122,24 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "accessibility-serve" +version = "0.1.0" +dependencies = [ + "accessibility-core", + "accessibility-ios-sys", + "anyhow", + "axum", + "bytes", + "futures-util", + "serde", + "serde_json", + "tokio", + "tower-http", + "tracing", + "webrtc", +] + [[package]] name = "accessibility-windows-sys" version = "0.1.0" @@ -150,6 +171,41 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -257,6 +313,15 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "arg_enum_proc_macro" version = "0.3.4" @@ -283,6 +348,45 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "asn1-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom 7.1.3", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "assert_cmd" version = "2.2.1" @@ -506,7 +610,7 @@ dependencies = [ "num-traits", "pastey", "rayon", - "thiserror", + "thiserror 2.0.18", "v_frame", "y4m", ] @@ -520,7 +624,7 @@ dependencies = [ "anyhow", "arrayvec", "log", - "nom", + "nom 8.0.0", "num-rational", "v_frame", ] @@ -534,18 +638,100 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "base64", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha1", + "sync_wrapper", + "tokio", + "tokio-tungstenite", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bit_field" version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.11.1" @@ -564,6 +750,24 @@ dependencies = [ "no_std_io2", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -633,6 +837,15 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.2.61" @@ -645,6 +858,18 @@ dependencies = [ "shlex", ] +[[package]] +name = "ccm" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae3c82e4355234767756212c570e29833699ab63e6ffd161887314cc5b43847" +dependencies = [ + "aead", + "cipher", + "ctr", + "subtle", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -657,6 +882,41 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", + "zeroize", +] + [[package]] name = "clap" version = "4.6.1" @@ -730,6 +990,36 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -770,7 +1060,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags", + "bitflags 2.11.1", "crossterm_winapi", "parking_lot", "rustix 0.38.44", @@ -792,6 +1082,29 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "typenum", +] + [[package]] name = "cssparser" version = "0.31.2" @@ -815,6 +1128,15 @@ dependencies = [ "syn", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "ctrlc" version = "3.5.2" @@ -822,10 +1144,73 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" dependencies = [ "dispatch2", - "nix", + "nix 0.31.2", "windows-sys 0.61.2", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom 7.1.3", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + [[package]] name = "derive_more" version = "0.99.20" @@ -843,18 +1228,77 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + [[package]] name = "dispatch2" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags", + "bitflags 2.11.1", "block2", "libc", "objc2", ] +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dtls" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f531dd7c181beaf3cebab3716afa4d0d41ab888be85232583f56bbaf07ca208a" +dependencies = [ + "aes", + "aes-gcm", + "async-trait", + "bincode", + "byteorder", + "cbc", + "ccm", + "chacha20poly1305", + "der-parser", + "hmac", + "log", + "p256", + "p384", + "portable-atomic", + "rand 0.9.4", + "rand_core 0.6.4", + "rcgen", + "ring", + "rustls", + "sec1", + "serde", + "sha1", + "sha2", + "thiserror 1.0.69", + "tokio", + "webrtc-util", + "x25519-dalek", + "x509-parser", +] + [[package]] name = "dtoa" version = "1.0.11" @@ -876,12 +1320,47 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + [[package]] name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest", + "ff", + "generic-array", + "group", + "hkdf", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encode_unicode" version = "1.0.0" @@ -1029,6 +1508,22 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1060,6 +1555,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "fslock" version = "0.2.1" @@ -1070,6 +1574,31 @@ dependencies = [ "winapi", ] +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + [[package]] name = "futures-core" version = "0.3.32" @@ -1106,6 +1635,23 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" + [[package]] name = "futures-task" version = "0.3.32" @@ -1118,8 +1664,13 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] @@ -1133,6 +1684,17 @@ dependencies = [ "byteorder", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + [[package]] name = "gethostname" version = "1.1.0" @@ -1181,6 +1743,16 @@ dependencies = [ "wasip3", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gif" version = "0.14.2" @@ -1191,6 +1763,17 @@ dependencies = [ "weezl", ] +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "half" version = "2.7.1" @@ -1235,12 +1818,213 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "id-arena" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "image" version = "0.25.10" @@ -1311,6 +2095,37 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "interceptor" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea51375727680dc15f06e8ad90fa31df75d79dd030100e8ad60eef1c27fe2c98" +dependencies = [ + "async-trait", + "bytes", + "futures", + "log", + "portable-atomic", + "rand 0.9.4", + "rtcp", + "rtp", + "thiserror 1.0.69", + "tokio", + "waitgroup", + "webrtc-srtp", + "webrtc-util", +] + [[package]] name = "interpolate_name" version = "0.2.4" @@ -1322,6 +2137,12 @@ dependencies = [ "syn", ] +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1380,7 +2201,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b750dcadc39a09dbadd74e118f6dd6598df77fa01df0cfcdc52c28dece74528a" dependencies = [ - "bitflags", + "bitflags 2.11.1", "serde", "unicode-segmentation", ] @@ -1437,6 +2258,12 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + [[package]] name = "lock_api" version = "0.4.14" @@ -1461,6 +2288,12 @@ dependencies = [ "imgref", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "matrixmultiply" version = "0.3.10" @@ -1481,12 +2314,31 @@ dependencies = [ "rayon", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -1496,6 +2348,18 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1548,13 +2412,26 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", + "memoffset 0.7.1", + "pin-utils", +] + [[package]] name = "nix" version = "0.31.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3" dependencies = [ - "bitflags", + "bitflags 2.11.1", "cfg-if", "cfg_aliases", "libc", @@ -1569,6 +2446,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nom" version = "8.0.0" @@ -1623,6 +2510,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-derive" version = "0.4.2" @@ -1690,7 +2583,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" dependencies = [ - "bitflags", + "bitflags 2.11.1", "block2", "libc", "objc2", @@ -1711,7 +2604,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69282c2b5bc58fba07cb9de2113619532eb551e98efe3d8d695509ef45fbd53b" dependencies = [ - "bitflags", + "bitflags 2.11.1", "libc", "objc2", "objc2-core-foundation", @@ -1726,7 +2619,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73ad74d880bb43877038da939b7427bba67e9dd42004a18b809ba7d87cee241c" dependencies = [ - "bitflags", + "bitflags 2.11.1", "objc2", "objc2-foundation", ] @@ -1749,7 +2642,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" dependencies = [ - "bitflags", + "bitflags 2.11.1", "objc2", ] @@ -1759,7 +2652,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b402a653efbb5e82ce4df10683b6b28027616a2715e90009947d50b8dd298fa" dependencies = [ - "bitflags", + "bitflags 2.11.1", "objc2", "objc2-foundation", ] @@ -1770,7 +2663,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags", + "bitflags 2.11.1", "block2", "dispatch2", "libc", @@ -1783,7 +2676,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" dependencies = [ - "bitflags", + "bitflags 2.11.1", "block2", "dispatch2", "libc", @@ -1809,7 +2702,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05ec576860167a15dd9fce7fbee7512beb4e31f532159d3482d1f9c6caedf31d" dependencies = [ - "bitflags", + "bitflags 2.11.1", "block2", "dispatch2", "objc2", @@ -1837,7 +2730,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cde0dfb48d25d2b4862161a4d5fcc0e3c24367869ad306b0c9ec0073bfed92d" dependencies = [ - "bitflags", + "bitflags 2.11.1", "objc2", "objc2-core-foundation", "objc2-core-graphics", @@ -1849,7 +2742,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d425caf1df73233f29fd8a5c3e5edbc30d2d4307870f802d18f00d83dc5141a6" dependencies = [ - "bitflags", + "bitflags 2.11.1", "block2", "objc2", "objc2-core-foundation", @@ -1870,7 +2763,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags", + "bitflags 2.11.1", "block2", "libc", "objc2", @@ -1883,7 +2776,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" dependencies = [ - "bitflags", + "bitflags 2.11.1", "libc", "objc2", "objc2-core-foundation", @@ -1896,7 +2789,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0125f776a10d00af4152d74616409f0d4a2053a6f57fa5b7d6aa2854ac04794" dependencies = [ - "bitflags", + "bitflags 2.11.1", "objc2", "objc2-foundation", ] @@ -1907,7 +2800,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" dependencies = [ - "bitflags", + "bitflags 2.11.1", "objc2", "objc2-foundation", ] @@ -1918,7 +2811,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "709fe137109bd1e8b5a99390f77a7d8b2961dafc1a1c5db8f2e60329ad6d895a" dependencies = [ - "bitflags", + "bitflags 2.11.1", "objc2", "objc2-core-foundation", ] @@ -1929,7 +2822,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05bf9a3c14831a7d9641b0d81d87dd913ee238a012b2fde27db5a84b56f5df3e" dependencies = [ - "bitflags", + "bitflags 2.11.1", "block2", "objc2", "objc2-core-foundation", @@ -1940,6 +2833,15 @@ dependencies = [ "objc2-metal", ] +[[package]] +name = "oid-registry" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1952,6 +2854,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "ordered-stream" version = "0.2.0" @@ -1971,6 +2879,30 @@ dependencies = [ "ttf-parser", ] +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + [[package]] name = "parking" version = "2.2.1" @@ -2013,8 +2945,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" [[package]] -name = "phf" -version = "0.10.1" +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" dependencies = [ @@ -2098,6 +3055,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "piper" version = "0.2.5" @@ -2109,13 +3072,23 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "png" version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" dependencies = [ - "bitflags", + "bitflags 2.11.1", "crc32fast", "fdeflate", "flate2", @@ -2136,6 +3109,50 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "portable-atomic" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2191,6 +3208,15 @@ dependencies = [ "syn", ] +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -2388,7 +3414,7 @@ dependencies = [ "rand 0.9.4", "rand_chacha 0.9.0", "simd_helpers", - "thiserror", + "thiserror 2.0.18", "v_frame", "wasm-bindgen", ] @@ -2434,13 +3460,27 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.11.1", ] [[package]] @@ -2492,6 +3532,16 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + [[package]] name = "rgb" version = "0.8.53" @@ -2501,13 +3551,71 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rtcp" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81d30d1c4091644431c22acf9f8be6191b56805e0e977f15ca7104b4a6d6eaec" +dependencies = [ + "bytes", + "thiserror 1.0.69", + "webrtc-util", +] + +[[package]] +name = "rtp" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f126f38ea84c02480e32e547c1459a939052f74fb92117ac3eef23fdac6b023" +dependencies = [ + "bytes", + "memchr", + "portable-atomic", + "rand 0.9.4", + "serde", + "thiserror 1.0.69", + "webrtc-util", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "rustix" version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -2520,19 +3628,59 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "safe_arch" version = "0.7.4" @@ -2588,13 +3736,39 @@ version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" +[[package]] +name = "sdp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32c374dceda16965d541c8800ce9cc4e1c14acfd661ddf7952feeedc3411e5c6" +dependencies = [ + "rand 0.9.4", + "substring", + "thiserror 1.0.69", + "url", +] + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "selectors" version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06" dependencies = [ - "bitflags", + "bitflags 2.11.1", "cssparser", "derive_more", "fxhash", @@ -2667,6 +3841,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_repr" version = "0.1.20" @@ -2678,6 +3863,18 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "serial_test" version = "3.4.0" @@ -2714,6 +3911,28 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "1.3.0" @@ -2730,6 +3949,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "simba" version = "0.8.1" @@ -2792,6 +4021,25 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "smol_str" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd538fb6910ac1099850255cf94a94df6551fbdd602454387d0adb2d1ca6dead" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.3" @@ -2802,6 +4050,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -2820,6 +4078,40 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "stun" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a512c5d501e3e3b5a4bb3e8e31462d56d54a66b95a28b8596e14422bf21c32b" +dependencies = [ + "base64", + "crc", + "lazy_static", + "md-5", + "rand 0.9.4", + "ring", + "subtle", + "thiserror 1.0.69", + "tokio", + "url", + "webrtc-util", +] + +[[package]] +name = "substring" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ee6433ecef213b2e72f587ef64a2f5943e7cd16fbd82dbe8bc07486c534c86" +dependencies = [ + "autocfg", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.117" @@ -2831,6 +4123,23 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -2859,13 +4168,33 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -2894,21 +4223,61 @@ dependencies = [ ] [[package]] -name = "tokio" -version = "1.52.1" +name = "time" +version = "0.3.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" dependencies = [ - "bytes", - "libc", - "mio", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "tracing", - "windows-sys 0.61.2", + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.52.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.6.3", + "tokio-macros", + "tracing", + "windows-sys 0.61.2", ] [[package]] @@ -2922,6 +4291,31 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml_datetime" version = "1.1.1+spec-1.1.0" @@ -2952,12 +4346,55 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.11.1", + "bytes", + "http", + "pin-project-lite", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2989,6 +4426,43 @@ version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" +[[package]] +name = "tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.4", + "sha1", + "thiserror 2.0.18", +] + +[[package]] +name = "turn" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ed995882f66ab94238de77c62e5e778389698ab700afa4696f4754da8f457cb" +dependencies = [ + "async-trait", + "base64", + "futures", + "log", + "md-5", + "portable-atomic", + "rand 0.9.4", + "ring", + "stun", + "thiserror 1.0.69", + "tokio", + "tokio-util", + "webrtc-util", +] + [[package]] name = "typenum" version = "1.20.0" @@ -3001,11 +4475,17 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ - "memoffset", + "memoffset 0.9.1", "tempfile", "windows-sys 0.61.2", ] +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -3024,6 +4504,40 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -3084,6 +4598,15 @@ dependencies = [ "libc", ] +[[package]] +name = "waitgroup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1f50000a783467e6c0200f9d10642f4bc424e39efc1b770203e88b488f79292" +dependencies = [ + "atomic-waker", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -3181,12 +4704,181 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags", + "bitflags 2.11.1", "hashbrown 0.15.5", "indexmap", "semver", ] +[[package]] +name = "webrtc" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08fd686c0920ac08f3a57eacc48e31f0e4ca1ffefba4478784606f78c14e83ad" +dependencies = [ + "arc-swap", + "async-trait", + "bytes", + "dtls", + "hex", + "interceptor", + "lazy_static", + "log", + "portable-atomic", + "rand 0.9.4", + "rcgen", + "regex", + "ring", + "rtcp", + "rtp", + "sdp", + "serde", + "serde_json", + "sha2", + "smol_str", + "stun", + "thiserror 1.0.69", + "tokio", + "turn", + "unicase", + "url", + "waitgroup", + "webrtc-data", + "webrtc-ice", + "webrtc-mdns", + "webrtc-media", + "webrtc-sctp", + "webrtc-srtp", + "webrtc-util", +] + +[[package]] +name = "webrtc-data" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "062a5438d63bb0756a221693d76cc0dd6119affee1dfdfe57abe3a2a8c8b3eea" +dependencies = [ + "bytes", + "log", + "portable-atomic", + "thiserror 1.0.69", + "tokio", + "webrtc-sctp", + "webrtc-util", +] + +[[package]] +name = "webrtc-ice" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cb13fd1a373e68addc4bba0c8ca058627518e54342583d024bdcbb8ae5d97d" +dependencies = [ + "arc-swap", + "async-trait", + "crc", + "log", + "portable-atomic", + "rand 0.9.4", + "serde", + "serde_json", + "stun", + "thiserror 1.0.69", + "tokio", + "turn", + "url", + "uuid", + "waitgroup", + "webrtc-mdns", + "webrtc-util", +] + +[[package]] +name = "webrtc-mdns" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17279a067e75df72ce923fdeb7f04cd808f6f5aa4910dc6bcb4fbe66b396ace" +dependencies = [ + "log", + "socket2 0.5.10", + "thiserror 1.0.69", + "tokio", + "webrtc-util", +] + +[[package]] +name = "webrtc-media" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a84c910fec0848fd5a0d8a5651e0ddbdedaf25a7d3ae3f0b15f71ac73a1773" +dependencies = [ + "byteorder", + "bytes", + "rand 0.9.4", + "rtp", + "thiserror 1.0.69", +] + +[[package]] +name = "webrtc-sctp" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f985465467d8910c1f8ac4382cd64f83b1f6a1a75021a82b221546f6fb3b856f" +dependencies = [ + "arc-swap", + "async-trait", + "bytes", + "crc", + "log", + "portable-atomic", + "rand 0.9.4", + "thiserror 1.0.69", + "tokio", + "webrtc-util", +] + +[[package]] +name = "webrtc-srtp" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d8cdc33413f1d0192670a80ce93d17cb78d57fe3a2414be30d6f6dff121123" +dependencies = [ + "aead", + "aes", + "aes-gcm", + "byteorder", + "bytes", + "ctr", + "hmac", + "log", + "rtcp", + "rtp", + "sha1", + "subtle", + "thiserror 1.0.69", + "tokio", + "webrtc-util", +] + +[[package]] +name = "webrtc-util" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c0c7e0c8f280f2bbfae442701465777ac07adaf46ce0c5863cd58e13fe472a" +dependencies = [ + "async-trait", + "bitflags 1.3.2", + "bytes", + "ipnet", + "lazy_static", + "log", + "nix 0.26.4", + "portable-atomic", + "rand 0.9.4", + "thiserror 1.0.69", + "tokio", + "winapi", +] + [[package]] name = "weezl" version = "0.1.12" @@ -3342,6 +5034,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.59.0" @@ -3506,7 +5207,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags", + "bitflags 2.11.1", "indexmap", "log", "serde", @@ -3536,6 +5237,12 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + [[package]] name = "x11rb" version = "0.13.2" @@ -3553,12 +5260,74 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" +[[package]] +name = "x25519-dalek" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" +dependencies = [ + "curve25519-dalek", + "rand_core 0.6.4", + "serde", + "zeroize", +] + +[[package]] +name = "x509-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom 7.1.3", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + [[package]] name = "y4m" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zbus" version = "5.15.0" @@ -3677,6 +5446,80 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index 8f93407..82a0f92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "packages/accessibility-windows-sys", "packages/accessibility-ios-sys", "packages/accessibility-core", + "packages/accessibility-serve", "packages/accessibility-cli", ] @@ -19,6 +20,7 @@ homepage = "https://github.com/DioxusLabs/accessibility-cli" [workspace.dependencies] accessibility-core = { path = "packages/accessibility-core", version = "0.1.0" } +accessibility-serve = { path = "packages/accessibility-serve", version = "0.1.0" } accessibility-cli = { path = "packages/accessibility-cli", version = "0.1.0" } accessibility-android-sys = { path = "packages/accessibility-android-sys", version = "0.1.0" } accessibility-linux-sys = { path = "packages/accessibility-linux-sys", version = "0.1.0" } @@ -30,6 +32,7 @@ accesskit = { version = "0.22", features = ["enumn", "schemars", "serde"] } anyhow = "1.0.100" async-trait = "0.1.89" atspi = { version = "0.29", features = ["connection", "proxies"] } +axum = "0.8" atspi-common = "0.13" block2 = "0.6" bytes = "1" @@ -39,6 +42,7 @@ dispatch2 = "0.3" ctrlc = "3" euclid = { version = "0.22", features = ["serde"] } futures-lite = "2" +futures-util = "0.3" image = "0.25" imageproc = "0.25" keyboard-types = "0.7" @@ -77,8 +81,12 @@ serde = { version = "1.0.228", features = ["derive"] } serde_json = "1" serial_test = { version = "3", features = ["file_locks"] } slotmap = { version = "1.1", features = ["serde"] } +tower-http = "0.6" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } tokio = { version = "1.49.0", features = ["full"] } viuer = "0.9" +webrtc = "0.14" x11rb = { version = "0.13", features = ["randr"] } zbus = { version = "5", features = ["tokio"] } diff --git a/packages/accessibility-cli/Cargo.toml b/packages/accessibility-cli/Cargo.toml index da7911e..d1cf5ea 100644 --- a/packages/accessibility-cli/Cargo.toml +++ b/packages/accessibility-cli/Cargo.toml @@ -9,6 +9,7 @@ homepage.workspace = true [dependencies] accessibility-core.workspace = true +anyhow.workspace = true clap.workspace = true ctrlc.workspace = true serde_json.workspace = true @@ -22,3 +23,6 @@ serial_test.workspace = true [[bin]] name = "accessibility-cli" path = "src/main.rs" + +[target.'cfg(target_os = "macos")'.dependencies] +accessibility-serve.workspace = true diff --git a/packages/accessibility-cli/src/lib.rs b/packages/accessibility-cli/src/lib.rs index 15cef63..2470b49 100644 --- a/packages/accessibility-cli/src/lib.rs +++ b/packages/accessibility-cli/src/lib.rs @@ -1389,6 +1389,10 @@ Examples: )] #[command(version)] pub struct Cli { + /// Subcommand to run. Omit to use the flag-driven query interface above. + #[command(subcommand)] + pub command: Option, + /// Target platform (defaults to current OS) #[arg(long, short = 'p', value_enum, default_value_t = PlatformType::default())] pub platform: PlatformType, @@ -1842,6 +1846,50 @@ fn parse_long_press(s: &str) -> Result<(f64, f64, u64), String> { Ok((x, y, duration_ms)) } +/// Subcommands that do something other than query an accessibility tree. +#[derive(clap::Subcommand)] +pub enum Command { + /// Serve the iOS Simulator in a browser: live video, input, and element + /// inspection. + ServeSim(ServeSimArgs), +} + +#[derive(clap::Args)] +pub struct ServeSimArgs { + /// Simulator to serve. Defaults to the first booted device. + #[arg(long)] + pub udid: Option, + + /// Port to listen on. + #[arg(long, default_value_t = 3200)] + pub port: u16, + + /// Address to bind. Defaults to loopback; use 0.0.0.0 to expose on the LAN. + #[arg(long, default_value = "127.0.0.1")] + pub bind: std::net::IpAddr, + + /// Transport the browser should prefer: webrtc or h264. + #[arg(long, default_value = "webrtc")] + pub transport: String, + + /// Encoder frame rate ceiling. The simulator only paints on change, so + /// this is an upper bound rather than a target. + #[arg(long, default_value_t = 60)] + pub fps: u32, + + /// Target bitrate in bits per second. + #[arg(long, default_value_t = 6_000_000)] + pub bitrate: u32, + + /// Seconds between scheduled keyframes. + #[arg(long, default_value_t = 2)] + pub keyframe_interval: u32, + + /// ICE server URL, repeatable. Only needed to traverse a NAT. + #[arg(long = "ice-server")] + pub ice_servers: Vec, +} + /// Run the CLI using process arguments. pub fn run() { let cli = Cli::parse(); @@ -1852,9 +1900,54 @@ pub fn run() { std::process::exit(1); } }; + + if let Some(command) = &cli.command { + runtime.block_on(run_command(command)); + return; + } + runtime.block_on(run_cli(&cli)); } +async fn run_command(command: &Command) { + let result = match command { + Command::ServeSim(args) => run_serve_sim(args).await, + }; + if let Err(error) = result { + eprintln!("Error: {error:#}"); + std::process::exit(1); + } +} + +#[cfg(target_os = "macos")] +async fn run_serve_sim(args: &ServeSimArgs) -> anyhow::Result<()> { + use accessibility_core::video::VideoConfig; + use accessibility_serve::{ServeConfig, Transport, serve}; + + let transport: Transport = args.transport.parse()?; + serve(ServeConfig { + udid: args.udid.clone(), + address: std::net::SocketAddr::new(args.bind, args.port), + transport, + video: VideoConfig { + fps: args.fps, + bitrate: args.bitrate, + keyframe_interval_secs: args.keyframe_interval, + // WebRTC needs Annex-B; the raw H.264 transport converts on the + // way out so a single encoder feeds both. + nal_format: accessibility_core::video::NalFormat::AnnexB, + ..Default::default() + }, + ice_servers: args.ice_servers.clone(), + }) + .await +} + +#[cfg(not(target_os = "macos"))] +async fn run_serve_sim(_args: &ServeSimArgs) -> anyhow::Result<()> { + anyhow::bail!("serve-sim requires macOS with Xcode and a booted iOS Simulator") +} + /// Build a TreeFilter from CommonArgs fn build_filter(common: &CommonArgs) -> TreeFilter { TreeFilter { diff --git a/packages/accessibility-core/Cargo.toml b/packages/accessibility-core/Cargo.toml index 2a9f89b..d46c75b 100644 --- a/packages/accessibility-core/Cargo.toml +++ b/packages/accessibility-core/Cargo.toml @@ -16,6 +16,7 @@ accessibility-android-sys.workspace = true accesskit.workspace = true anyhow.workspace = true async-trait.workspace = true +bytes.workspace = true cssparser.workspace = true euclid.workspace = true image.workspace = true diff --git a/packages/accessibility-core/src/accessibility/mod.rs b/packages/accessibility-core/src/accessibility/mod.rs index 2cda6bb..ec3e53c 100644 --- a/packages/accessibility-core/src/accessibility/mod.rs +++ b/packages/accessibility-core/src/accessibility/mod.rs @@ -106,6 +106,23 @@ pub trait AccessibilityReader { async { anyhow::bail!("Screen bounds not supported on this platform") } } + /// Start a live video capture session, pushing encoded frames to `sink`. + /// + /// This is the streaming counterpart to [`Self::capture_screen`]. Only the + /// iOS Simulator implements it today. + fn start_video_capture( + &self, + _config: &crate::video::VideoConfig, + _sink: crate::video::FrameSink, + ) -> Result> { + crate::video::unsupported(self.platform_name()) + } + + /// Whether [`Self::start_video_capture`] is expected to succeed. + fn supports_video_capture(&self) -> bool { + false + } + /// Get the platform name (e.g., "macOS", "Windows", "Linux", "iOS"). fn platform_name(&self) -> &'static str { "Unknown" diff --git a/packages/accessibility-core/src/lib.rs b/packages/accessibility-core/src/lib.rs index 365f0c0..ea1cc70 100644 --- a/packages/accessibility-core/src/lib.rs +++ b/packages/accessibility-core/src/lib.rs @@ -4,3 +4,4 @@ pub mod accessibility; pub mod api; pub mod input; pub mod platform; +pub mod video; diff --git a/packages/accessibility-core/src/platform/ios_simulator.rs b/packages/accessibility-core/src/platform/ios_simulator.rs index 38b26b6..24737b1 100644 --- a/packages/accessibility-core/src/platform/ios_simulator.rs +++ b/packages/accessibility-core/src/platform/ios_simulator.rs @@ -7,6 +7,7 @@ use std::collections::HashMap; use std::future; +use std::sync::Arc; use accessibility_ios_sys as sys; use accesskit::{Action, Role}; @@ -17,6 +18,9 @@ use crate::accessibility::{ AccessibilityReader, Element, ElementCache, ElementKey, ElementTree, Point, Rect, Screenshot, Size, Target, TreeFilter, }; +use crate::video::{ + EncodedFrame, FrameKind, FrameSink, NalFormat, ScreenGeometry, VideoCapture, VideoConfig, +}; pub use sys::{ButtonDirection, HardwareButton}; @@ -294,6 +298,75 @@ impl AccessibilityReader for IOSSimulatorAccessibility { fn supports_hit_test(&self) -> bool { true } + + fn start_video_capture( + &self, + config: &VideoConfig, + sink: FrameSink, + ) -> Result> { + let session = SimulatorVideoCapture::start(self.device_udid(), config, sink)?; + Ok(Box::new(session)) + } + + fn supports_video_capture(&self) -> bool { + true + } +} + +/// Live framebuffer capture for the simulator, wrapping `accessibility-ios-sys`. +pub struct SimulatorVideoCapture { + inner: sys::SimVideoStream, +} + +impl SimulatorVideoCapture { + /// Start capturing the given device. + /// + /// The returned session owns the SimulatorKit registration; dropping it + /// tears the capture pipeline down. + pub fn start(udid: &str, config: &VideoConfig, sink: FrameSink) -> Result { + let sys_config = sys::EncoderConfig { + fps: config.fps, + bitrate: config.bitrate, + keyframe_interval_secs: config.keyframe_interval_secs, + nal_format: match config.nal_format { + NalFormat::AnnexB => sys::NalFormat::AnnexB, + NalFormat::Avcc => sys::NalFormat::Avcc, + }, + }; + + let sys_sink: sys::ChunkSink = Arc::new(move |chunk: sys::EncodedChunk| { + sink(EncodedFrame { + data: chunk.data, + kind: match chunk.kind { + sys::ChunkKind::ParameterSet => FrameKind::ParameterSet, + sys::ChunkKind::Keyframe => FrameKind::Keyframe, + sys::ChunkKind::Delta => FrameKind::Delta, + }, + }); + }); + + Ok(Self { + inner: sys::SimVideoStream::start(Some(udid), sys_config, sys_sink)?, + }) + } +} + +impl VideoCapture for SimulatorVideoCapture { + fn geometry(&self) -> ScreenGeometry { + let geometry = self.inner.geometry(); + ScreenGeometry { + width: geometry.width, + height: geometry.height, + } + } + + fn request_keyframe(&self) { + self.inner.request_keyframe(); + } + + fn stop(&mut self) { + self.inner.stop(); + } } use super::macos::IOSAdapter; diff --git a/packages/accessibility-core/src/video/mod.rs b/packages/accessibility-core/src/video/mod.rs new file mode 100644 index 0000000..7cfd614 --- /dev/null +++ b/packages/accessibility-core/src/video/mod.rs @@ -0,0 +1,130 @@ +//! Platform-agnostic live video capture. +//! +//! This is the streaming counterpart to [`crate::accessibility::Screenshot`]: +//! where a screenshot is a single decoded image, a [`VideoCapture`] is a +//! continuous source of *encoded* frames. +//! +//! Deliberately encoded-frame oriented. Every platform that can do this at all +//! has a hardware encoder sitting right next to its capture API, and handing +//! raw surfaces across the abstraction boundary would force a copy and make +//! the zero-copy paths unreachable. +//! +//! Only the iOS Simulator backend is implemented today; every other platform +//! reports [`VideoCapture`] as unsupported. + +use std::sync::Arc; + +use anyhow::Result; +use bytes::Bytes; + +/// Compressed video codec. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum VideoCodec { + #[default] + H264, +} + +/// How H.264 NAL units are framed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum NalFormat { + /// `00 00 00 01` start codes. What WebRTC's H.264 payloader expects. + #[default] + AnnexB, + /// 4-byte big-endian length prefixes, paired with a separate parameter set + /// record. What the browser `VideoDecoder` API expects. + Avcc, +} + +/// What a given [`EncodedFrame`] carries. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FrameKind { + /// Codec configuration (an `avcC` record for H.264). Only produced in + /// [`NalFormat::Avcc`]; in Annex-B the parameter sets are inline. + ParameterSet, + /// An independently decodable frame. + Keyframe, + /// A frame that depends on earlier frames. + Delta, +} + +impl FrameKind { + /// Whether a client joining at this frame can start decoding. + pub fn is_decodable_entry_point(self) -> bool { + matches!(self, FrameKind::ParameterSet | FrameKind::Keyframe) + } +} + +/// One encoded unit, ready to be put on a wire. +#[derive(Debug, Clone)] +pub struct EncodedFrame { + pub data: Bytes, + pub kind: FrameKind, +} + +/// Pixel dimensions of the captured display. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ScreenGeometry { + pub width: u32, + pub height: u32, +} + +impl ScreenGeometry { + pub fn is_valid(self) -> bool { + self.width > 0 && self.height > 0 + } +} + +/// Encoder tuning. +#[derive(Debug, Clone, Copy)] +pub struct VideoConfig { + pub codec: VideoCodec, + pub nal_format: NalFormat, + pub fps: u32, + pub bitrate: u32, + /// Seconds between scheduled keyframes. Shorter means faster recovery for + /// clients that join late or drop packets, at the cost of bitrate. + pub keyframe_interval_secs: u32, +} + +impl Default for VideoConfig { + fn default() -> Self { + Self { + codec: VideoCodec::default(), + nal_format: NalFormat::default(), + fps: 60, + bitrate: 6_000_000, + keyframe_interval_secs: 2, + } + } +} + +/// Sink for encoded frames. +/// +/// Invoked on the platform's capture thread, so implementations must not +/// block. The expected shape is a bounded channel that drops on overflow: for +/// interactive streaming, a stale frame is worth less than a fresh one. +pub type FrameSink = Arc; + +/// A running video capture session. +/// +/// `Sync` is required because a session is shared across every connected +/// viewer. Implementations only expose atomics and locked state through +/// `&self`; anything that mutates the pipeline takes `&mut self`. +pub trait VideoCapture: Send + Sync { + /// Pixel geometry of the source. May be zero until the first frame lands. + fn geometry(&self) -> ScreenGeometry; + + /// Request that the next encoded frame be a keyframe. + /// + /// Called when a new client subscribes, or in response to an RTCP PLI/FIR + /// from a WebRTC receiver. + fn request_keyframe(&self); + + /// Stop capturing and release platform resources. + fn stop(&mut self); +} + +/// Error returned by platforms without a video backend. +pub fn unsupported(platform: &str) -> Result { + anyhow::bail!("Video capture is not supported on {platform}") +} diff --git a/packages/accessibility-ios-sys/src/macos.rs b/packages/accessibility-ios-sys/src/macos.rs index 7c08796..fef79b8 100644 --- a/packages/accessibility-ios-sys/src/macos.rs +++ b/packages/accessibility-ios-sys/src/macos.rs @@ -71,5 +71,6 @@ pub use common::{ }; pub use encoder::{ChunkKind, ChunkSink, EncodedChunk, EncoderConfig, H264Encoder, NalFormat}; pub use framebuffer::{CapturedFrame, FrameSink, FramebufferStats, SimFramebuffer}; +pub use hid::{SimulatorHID, TouchPhase}; pub use reader::IOSSimulatorAccessibility; pub use stream::{ScreenGeometry, SimVideoStream}; diff --git a/packages/accessibility-ios-sys/src/macos/framebuffer.rs b/packages/accessibility-ios-sys/src/macos/framebuffer.rs index 8de034c..82fcbb2 100644 --- a/packages/accessibility-ios-sys/src/macos/framebuffer.rs +++ b/packages/accessibility-ios-sys/src/macos/framebuffer.rs @@ -112,8 +112,18 @@ struct Registration { } /// State shared between the capture queue, the idle timer, and the owner. +/// +/// The wiring state (device, IO client, registrations, blocks) lives here +/// rather than on [`SimFramebuffer`] so the idle timer can rebuild the whole +/// pipeline without needing `&mut` access to the owner. struct CaptureState { + device: ObjcPtr, + queue: DispatchRetained, registrations: Mutex>, + /// SimulatorKit retains these blocks for the lifetime of the registration. + /// Dropping them early is a use-after-free, not a clean failure, so they + /// are held until the matching unregister call. + blocks: Mutex>, photocopier: Mutex, sink: Mutex>, last_capture: Mutex, @@ -223,91 +233,16 @@ impl CaptureState { }); } } -} - -/// Live framebuffer capture session for one simulator device. -pub struct SimFramebuffer { - device: ObjcPtr, - device_udid: String, - io_client: Option, - queue: DispatchRetained, - state: Arc, - /// SimulatorKit retains these blocks for the lifetime of the registration. - /// Dropping them early is a use-after-free, not a clean failure. - blocks: Vec, - idle_thread: Option>, -} - -// The raw pointers are only messaged from the capture queue or from methods -// that take `&mut self`, so the type is safe to move between threads. -unsafe impl Send for SimFramebuffer {} - -impl SimFramebuffer { - /// Attach to a booted simulator. `udid` of `None` picks the first booted one. - pub fn new(udid: Option<&str>) -> Result { - crate::frameworks::load_coresimulator_framework()?; - crate::frameworks::load_simulatorkit_framework()?; - - let device = unsafe { find_booted_device(udid)? }; - let device_udid = unsafe { device_udid_string(device)? }; - - Ok(Self { - device: ObjcPtr(device), - device_udid, - io_client: None, - queue: DispatchQueue::new( - "com.accessibility_cli.framebuffer", - DispatchQueueAttr::SERIAL, - ), - state: Arc::new(CaptureState { - registrations: Mutex::new(Vec::new()), - photocopier: Mutex::new(Photocopier::new()), - sink: Mutex::new(None), - last_capture: Mutex::new(Instant::now()), - frame_count: AtomicU64::new(0), - rewire_count: AtomicU64::new(0), - width: AtomicUsize::new(0), - height: AtomicUsize::new(0), - running: AtomicBool::new(false), - }), - blocks: Vec::new(), - idle_thread: None, - }) - } - - pub fn device_udid(&self) -> &str { - &self.device_udid - } - - pub fn stats(&self) -> FramebufferStats { - self.state.stats() - } - - /// Install the frame sink. Called on the capture queue, so it must be quick. - pub fn set_sink(&mut self, sink: Option) { - *self.state.sink.lock().unwrap() = sink; - } - - /// Begin capturing. Idempotent-ish: calling twice re-wires the pipeline. - pub fn start(&mut self) -> Result<()> { - self.state.running.store(true, Ordering::Relaxed); - self.wire_up()?; - self.start_idle_timer(); - Ok(()) - } /// Walk the IO port graph and register screen callbacks on every - /// framebuffer descriptor. - fn wire_up(&mut self) -> Result<()> { - let device = self.device.as_ptr(); - - let io: *mut AnyObject = unsafe { msg_send![device, io] }; + /// framebuffer descriptor, replacing any existing registration. + fn wire_up(self: &Arc) -> Result<()> { + let io: *mut AnyObject = unsafe { msg_send![self.device.as_ptr(), io] }; if io.is_null() { return Err(anyhow!( "SimDevice returned no IO client; is the simulator still booted?" )); } - self.io_client = Some(ObjcPtr(io)); let _: () = unsafe { msg_send![io, updateIOPorts] }; @@ -326,10 +261,10 @@ impl SimFramebuffer { for descriptor in descriptors { let uuid = NSUUID::new(); - let state = Arc::clone(&self.state); + let state = Arc::clone(self); let frame_cb = VoidBlock::new(move || state.capture(false)); // `surfacesChanged` carries no payload; it just means "re-query". - let state = Arc::clone(&self.state); + let state = Arc::clone(self); let surfaces_cb = VoidBlock::new(move || state.capture(true)); let props_cb = VoidBlock::new(|| {}); @@ -355,15 +290,99 @@ impl SimFramebuffer { }); } - *self.state.registrations.lock().unwrap() = registrations; - self.blocks = blocks; + *self.registrations.lock().unwrap() = registrations; + *self.blocks.lock().unwrap() = blocks; // Registration alone does not deliver a first frame; prime it. - self.state.capture(true); + self.capture(true); + Ok(()) + } + + fn unregister_all(&self) { + let mut registrations = self.registrations.lock().unwrap(); + for registration in registrations.drain(..) { + let descriptor = registration.descriptor.as_ptr(); + let selector = sel!(unregisterScreenCallbacksWithUUID:); + if unsafe { responds_to(descriptor, selector) } { + unsafe { + send_void_with_id( + descriptor, + selector, + &*registration.uuid as *const NSUUID as *mut AnyObject, + ); + } + } + } + drop(registrations); + // Only safe to release the blocks once SimulatorKit has been told to + // stop calling them. + self.blocks.lock().unwrap().clear(); + self.width.store(0, Ordering::Relaxed); + self.height.store(0, Ordering::Relaxed); + } +} + +/// Live framebuffer capture session for one simulator device. +pub struct SimFramebuffer { + device_udid: String, + state: Arc, + idle_thread: Option>, +} + +impl SimFramebuffer { + /// Attach to a booted simulator. `udid` of `None` picks the first booted one. + pub fn new(udid: Option<&str>) -> Result { + crate::frameworks::load_coresimulator_framework()?; + crate::frameworks::load_simulatorkit_framework()?; + + let device = unsafe { find_booted_device(udid)? }; + let device_udid = unsafe { device_udid_string(device)? }; + + Ok(Self { + device_udid, + state: Arc::new(CaptureState { + device: ObjcPtr(device), + queue: DispatchQueue::new( + "com.accessibility_cli.framebuffer", + DispatchQueueAttr::SERIAL, + ), + registrations: Mutex::new(Vec::new()), + blocks: Mutex::new(Vec::new()), + photocopier: Mutex::new(Photocopier::new()), + sink: Mutex::new(None), + last_capture: Mutex::new(Instant::now()), + frame_count: AtomicU64::new(0), + rewire_count: AtomicU64::new(0), + width: AtomicUsize::new(0), + height: AtomicUsize::new(0), + running: AtomicBool::new(false), + }), + idle_thread: None, + }) + } + + pub fn device_udid(&self) -> &str { + &self.device_udid + } + + pub fn stats(&self) -> FramebufferStats { + self.state.stats() + } + + /// Install the frame sink. Called on the capture queue, so it must be quick. + pub fn set_sink(&mut self, sink: Option) { + *self.state.sink.lock().unwrap() = sink; + } + + /// Begin capturing. Calling twice rebuilds the pipeline. + pub fn start(&mut self) -> Result<()> { + self.state.running.store(true, Ordering::Relaxed); + self.state.wire_up()?; + self.start_idle_timer(); Ok(()) } - /// Drive forced re-emits, and re-wire while nothing has arrived yet. + /// Drive forced re-emits, and rebuild the pipeline while nothing arrives. fn start_idle_timer(&mut self) { if self.idle_thread.is_some() { return; @@ -383,40 +402,27 @@ impl SimFramebuffer { state.capture(true); } - // Self-heal only until the first frame ever lands. After that a - // silent pipeline means an idle screen, not a broken graph. - if state.frame_count.load(Ordering::Relaxed) == 0 && tick % REWIRE_TICKS == 0 { + // Self-heal only until the first frame ever lands. After that + // a silent pipeline means an idle screen, not a broken graph. + if state.frame_count.load(Ordering::Relaxed) == 0 + && tick.is_multiple_of(REWIRE_TICKS) + { state.rewire_count.fetch_add(1, Ordering::Relaxed); + // Descriptors are sometimes created lazily, so a + // registration that happened too early yields nothing. + // Failures are expected here; the next tick retries. + let _ = state.wire_up(); } } })); } - fn unregister_all(&mut self) { - let mut registrations = self.state.registrations.lock().unwrap(); - for registration in registrations.drain(..) { - let descriptor = registration.descriptor.as_ptr(); - let selector = sel!(unregisterScreenCallbacksWithUUID:); - if unsafe { responds_to(descriptor, selector) } { - unsafe { - send_void_with_id( - descriptor, - selector, - &*registration.uuid as *const NSUUID as *mut AnyObject, - ); - } - } - } - drop(registrations); - self.blocks.clear(); - } - pub fn stop(&mut self) { self.state.running.store(false, Ordering::Relaxed); if let Some(handle) = self.idle_thread.take() { let _ = handle.join(); } - self.unregister_all(); + self.state.unregister_all(); *self.state.sink.lock().unwrap() = None; } } diff --git a/packages/accessibility-ios-sys/src/macos/hid.rs b/packages/accessibility-ios-sys/src/macos/hid.rs index 43b7745..ea73d5f 100644 --- a/packages/accessibility-ios-sys/src/macos/hid.rs +++ b/packages/accessibility-ios-sys/src/macos/hid.rs @@ -25,7 +25,15 @@ type IndigoMessageForKeyboardFn = unsafe extern "C" fn(key_code: i32, action: i3 /// Uses the Indigo protocol via SimulatorKit's SimDeviceLegacyHIDClient /// to inject touch events, button presses, and keyboard input directly /// into the simulator's HID subsystem. -pub(super) struct SimulatorHID { +/// Phase of an interactive touch stream. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TouchPhase { + Begin, + Move, + End, +} + +pub struct SimulatorHID { client: *mut AnyObject, // SimDeviceLegacyHIDClient queue: *mut AnyObject, // dispatch_queue_t screen_size: (f64, f64), @@ -131,6 +139,17 @@ impl SimulatorHID { }) } + /// Create a HID client for a booted device, resolving it by UDID. + /// + /// `None` picks the first booted simulator. This exists so an input path + /// can be opened independently of the accessibility reader, which keeps + /// pointer events from queueing behind slow AX tree fetches. + pub fn for_device(udid: Option<&str>) -> Result { + crate::frameworks::load_coresimulator_framework()?; + let device = unsafe { super::common::find_booted_device(udid)? }; + Self::new(device) + } + /// Get the screen size in points. pub fn screen_size(&self) -> (f64, f64) { self.screen_size @@ -192,6 +211,27 @@ impl SimulatorHID { Ok(()) } + /// Send a single interactive touch event in normalized screen space. + /// + /// Unlike [`Self::tap`] and [`Self::swipe`], this does not synthesize a + /// whole gesture: the caller drives the phases itself, which is what a live + /// pointer stream from a browser needs. + /// + /// `x` and `y` are 0..1 fractions of the screen, matching what the web UI + /// already computes, so no point/pixel/scale conversion is involved. + pub fn touch_normalized(&self, x: f64, y: f64, phase: TouchPhase) -> Result<()> { + let x = x.clamp(0.0, 1.0); + let y = y.clamp(0.0, 1.0); + // Indigo has no distinct "move" phase; contact is maintained by + // repeating the down event at the new position, which is exactly what + // `swipe` does internally. + let direction = match phase { + TouchPhase::Begin | TouchPhase::Move => ButtonDirection::Down, + TouchPhase::End => ButtonDirection::Up, + }; + self.send_touch(x, y, direction) + } + /// Press a hardware button. /// /// # Arguments diff --git a/packages/accessibility-serve/Cargo.toml b/packages/accessibility-serve/Cargo.toml new file mode 100644 index 0000000..b3ec46c --- /dev/null +++ b/packages/accessibility-serve/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "accessibility-serve" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +description = "Serve a live, interactive iOS Simulator stream over WebRTC with accessibility element inspection." +readme = "../../README.md" +keywords = ["accessibility", "ios", "simulator", "webrtc", "streaming"] +categories = ["accessibility", "web-programming", "os::macos-apis"] + +[dependencies] +accessibility-core.workspace = true +anyhow.workspace = true +axum = { workspace = true, features = ["ws"] } +bytes.workspace = true +futures-util.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true +tower-http = { workspace = true, features = ["cors"] } +tracing.workspace = true +webrtc.workspace = true + +[target.'cfg(target_os = "macos")'.dependencies] +accessibility-ios-sys.workspace = true diff --git a/packages/accessibility-serve/src/avcc.rs b/packages/accessibility-serve/src/avcc.rs new file mode 100644 index 0000000..1ce148f --- /dev/null +++ b/packages/accessibility-serve/src/avcc.rs @@ -0,0 +1,195 @@ +//! Annex-B to AVCC conversion for browser `VideoDecoder` clients. +//! +//! The encoder runs in Annex-B because that is what WebRTC's H.264 payloader +//! consumes. WebCodecs wants the other framing: 4-byte length prefixes plus a +//! one-time `avcC` record carrying the parameter sets. Converting here means a +//! single encoder can feed both transports at once. + +/// Wire framing for the raw H.264 WebSocket transport. +/// +/// `u32` big-endian length covering the tag byte and payload, then the tag, +/// then the payload. +pub mod tag { + pub const PARAMETER_SET: u8 = 0x01; + pub const KEYFRAME: u8 = 0x02; + pub const DELTA: u8 = 0x03; +} + +/// Frame a payload for the raw H.264 WebSocket transport. +pub fn envelope(tag: u8, payload: &[u8]) -> Vec { + let mut out = Vec::with_capacity(payload.len() + 5); + out.extend_from_slice(&((payload.len() + 1) as u32).to_be_bytes()); + out.push(tag); + out.extend_from_slice(payload); + out +} + +/// Iterate the NAL units in an Annex-B stream. +/// +/// Handles both 3- and 4-byte start codes, since VideoToolbox is consistent +/// but the parameter sets we prepend may not be. +pub fn nal_units(annex_b: &[u8]) -> Vec<&[u8]> { + let mut starts = Vec::new(); + let mut index = 0usize; + while index + 3 <= annex_b.len() { + if annex_b[index] == 0 && annex_b[index + 1] == 0 { + if annex_b[index + 2] == 1 { + starts.push((index, 3)); + index += 3; + continue; + } + if index + 4 <= annex_b.len() && annex_b[index + 2] == 0 && annex_b[index + 3] == 1 { + starts.push((index, 4)); + index += 4; + continue; + } + } + index += 1; + } + + let mut units = Vec::with_capacity(starts.len()); + for (position, (offset, code_len)) in starts.iter().enumerate() { + let begin = offset + code_len; + let end = starts + .get(position + 1) + .map(|(next, _)| *next) + .unwrap_or(annex_b.len()); + if begin < end { + units.push(&annex_b[begin..end]); + } + } + units +} + +/// Rewrite an Annex-B access unit as length-prefixed AVCC. +/// +/// Parameter set NALs are dropped: in AVCC they belong in the `avcC` record, +/// not the bitstream. +pub fn to_avcc(annex_b: &[u8]) -> Vec { + let mut out = Vec::with_capacity(annex_b.len()); + for unit in nal_units(annex_b) { + if matches!(nal_type(unit), Some(7 | 8)) { + continue; + } + out.extend_from_slice(&(unit.len() as u32).to_be_bytes()); + out.extend_from_slice(unit); + } + out +} + +/// Pull the SPS (type 7) and PPS (type 8) out of an Annex-B access unit. +pub fn parameter_sets(annex_b: &[u8]) -> Option<(Vec, Vec)> { + let mut sps = None; + let mut pps = None; + for unit in nal_units(annex_b) { + match nal_type(unit) { + Some(7) if sps.is_none() => sps = Some(unit.to_vec()), + Some(8) if pps.is_none() => pps = Some(unit.to_vec()), + _ => {} + } + } + let sps = sps?; + let pps = pps?; + // The avcC record copies profile/compatibility/level out of the SPS. + (sps.len() >= 4).then_some((sps, pps)) +} + +fn nal_type(unit: &[u8]) -> Option { + unit.first().map(|byte| byte & 0x1F) +} + +/// Build an ISO/IEC 14496-15 `avcC` configuration record. +pub fn avcc_record(sps: &[u8], pps: &[u8]) -> Vec { + let mut out = Vec::with_capacity(sps.len() + pps.len() + 11); + out.push(0x01); // configurationVersion + out.extend_from_slice(&sps[1..4]); // profile, compatibility, level + out.push(0xFF); // reserved | lengthSizeMinusOne = 3 (4-byte lengths) + out.push(0xE1); // reserved | numOfSequenceParameterSets = 1 + out.extend_from_slice(&(sps.len() as u16).to_be_bytes()); + out.extend_from_slice(sps); + out.push(0x01); // numOfPictureParameterSets + out.extend_from_slice(&(pps.len() as u16).to_be_bytes()); + out.extend_from_slice(pps); + out +} + +/// The `codecs` string a browser needs to configure a decoder for this SPS. +pub fn codec_string(sps: &[u8]) -> String { + format!("avc1.{:02X}{:02X}{:02X}", sps[1], sps[2], sps[3]) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn annex_b(units: &[&[u8]]) -> Vec { + let mut out = Vec::new(); + for unit in units { + out.extend_from_slice(&[0, 0, 0, 1]); + out.extend_from_slice(unit); + } + out + } + + #[test] + fn splits_four_byte_start_codes() { + let stream = annex_b(&[&[0x67, 1, 2, 3], &[0x68, 9], &[0x65, 7, 7]]); + let units = nal_units(&stream); + assert_eq!(units.len(), 3); + assert_eq!(units[0], &[0x67, 1, 2, 3]); + assert_eq!(units[2], &[0x65, 7, 7]); + } + + #[test] + fn splits_three_byte_start_codes() { + let stream = vec![0, 0, 1, 0x67, 1, 2, 3, 0, 0, 1, 0x65, 9]; + let units = nal_units(&stream); + assert_eq!(units.len(), 2); + assert_eq!(units[0], &[0x67, 1, 2, 3]); + assert_eq!(units[1], &[0x65, 9]); + } + + #[test] + fn avcc_drops_parameter_sets_and_prefixes_lengths() { + let stream = annex_b(&[&[0x67, 1, 2, 3], &[0x68, 9], &[0x65, 7, 7]]); + let avcc = to_avcc(&stream); + // Only the IDR survives: 4 length bytes + 3 payload bytes. + assert_eq!(avcc, vec![0, 0, 0, 3, 0x65, 7, 7]); + } + + #[test] + fn extracts_parameter_sets() { + let stream = annex_b(&[&[0x67, 0x42, 0xC0, 0x1F], &[0x68, 9], &[0x65, 7]]); + let (sps, pps) = parameter_sets(&stream).expect("parameter sets present"); + assert_eq!(sps, vec![0x67, 0x42, 0xC0, 0x1F]); + assert_eq!(pps, vec![0x68, 9]); + assert_eq!(codec_string(&sps), "avc1.42C01F"); + } + + #[test] + fn parameter_sets_absent_on_delta_frames() { + let stream = annex_b(&[&[0x41, 1, 2]]); + assert!(parameter_sets(&stream).is_none()); + } + + #[test] + fn avcc_record_layout() { + let sps = [0x67, 0x42, 0xC0, 0x1F, 0xAA]; + let pps = [0x68, 0xCE]; + let record = avcc_record(&sps, &pps); + assert_eq!(record[0], 1); + assert_eq!(&record[1..4], &[0x42, 0xC0, 0x1F]); + assert_eq!(record[4], 0xFF); + assert_eq!(record[5], 0xE1); + assert_eq!(&record[6..8], &(sps.len() as u16).to_be_bytes()); + assert_eq!(&record[8..8 + sps.len()], &sps); + } + + #[test] + fn envelope_length_covers_tag_and_payload() { + let framed = envelope(tag::KEYFRAME, &[1, 2, 3]); + assert_eq!(&framed[0..4], &4u32.to_be_bytes()); + assert_eq!(framed[4], tag::KEYFRAME); + assert_eq!(&framed[5..], &[1, 2, 3]); + } +} diff --git a/packages/accessibility-serve/src/ax.rs b/packages/accessibility-serve/src/ax.rs new file mode 100644 index 0000000..d66fc8c --- /dev/null +++ b/packages/accessibility-serve/src/ax.rs @@ -0,0 +1,201 @@ +//! Accessibility element inspection for the web UI. +//! +//! Two things are exposed: +//! +//! - A whole-tree snapshot, which the browser uses for instant hover feedback. +//! - A live hit test, which asks the simulator what is actually at a point. +//! +//! The snapshot can go stale between fetches and mis-picks overlapping or +//! transformed views; the hit test is authoritative but costs a round trip. +//! The UI uses the snapshot while the pointer is moving and confirms with the +//! hit test once it settles, which is why both exist. +//! +//! # Coordinate spaces +//! +//! Accessibility frames come back in macOS screen points, positioned wherever +//! the Simulator window happens to be. The browser knows nothing about that, +//! so every rect is converted to a 0..1 fraction of the app's own bounds +//! before it leaves this module: +//! +//! ```text +//! normalized = (ax_rect.origin - app_bounds.origin) / app_bounds.size +//! ``` +//! +//! That also makes the values independent of the display scale factor, which +//! is why no pixel/point conversion appears here. + +use anyhow::{Result, anyhow}; +use serde::Serialize; +use tokio::sync::{mpsc, oneshot}; + +use accessibility_core::accessibility::{Element, Rect, TreeFilter}; + +/// A rectangle in normalized display space (0..1 on both axes). +#[derive(Debug, Clone, Copy, Serialize)] +pub struct NormalizedRect { + pub x: f64, + pub y: f64, + pub width: f64, + pub height: f64, +} + +impl NormalizedRect { + fn from_screen(rect: &Rect, app_bounds: &Rect) -> Option { + if app_bounds.size.width <= 0.0 || app_bounds.size.height <= 0.0 { + return None; + } + Some(Self { + x: (rect.origin.x - app_bounds.origin.x) / app_bounds.size.width, + y: (rect.origin.y - app_bounds.origin.y) / app_bounds.size.height, + width: rect.size.width / app_bounds.size.width, + height: rect.size.height / app_bounds.size.height, + }) + } +} + +/// One inspectable element, flattened for the browser. +#[derive(Debug, Clone, Serialize)] +pub struct ElementDetail { + pub id: String, + pub role: String, + pub label: Option, + pub value: Option, + pub identifier: Option, + pub enabled: bool, + pub focused: bool, + pub actions: Vec, + pub bounds: Option, + /// Depth in the tree, used by the UI to prefer the innermost hit. + pub depth: u32, + /// A selector that would target this element from the CLI. + pub selector: String, +} + +#[derive(Debug, Clone, Serialize)] +pub struct AxSnapshot { + pub app_name: Option, + pub pid: Option, + pub elements: Vec, +} + +pub enum AxCommand { + Snapshot { + reply: oneshot::Sender>, + }, + HitTest { + x: f64, + y: f64, + reply: oneshot::Sender>>, + }, +} + +/// Build the `accessibility-cli` selector that would resolve this element. +/// +/// Preferring the identifier keeps the selector stable across copy changes, +/// which is the whole point of exposing it in the inspector. +fn selector_for(element: &Element, role: &str) -> String { + if let Some(identifier) = element.identifier.as_deref().filter(|s| !s.is_empty()) { + return format!("#{identifier}"); + } + if let Some(label) = element.title.as_deref().filter(|s| !s.is_empty()) { + return format!("{role}[label=\"{}\"]", label.replace('"', "\\\"")); + } + role.to_string() +} + +fn to_detail(element: &Element, app_bounds: &Rect, depth: u32) -> ElementDetail { + let role = format!("{:?}", element.role); + ElementDetail { + id: element.id.to_string(), + selector: selector_for(element, &role), + role, + label: element.title.clone().filter(|s| !s.is_empty()), + value: element.value.clone().filter(|s| !s.is_empty()), + identifier: element.identifier.clone().filter(|s| !s.is_empty()), + enabled: element.enabled, + focused: element.focused, + actions: element.actions.clone(), + bounds: element + .bounds + .as_ref() + .and_then(|bounds| NormalizedRect::from_screen(bounds, app_bounds)), + depth, + } +} + +fn flatten(element: &Element, app_bounds: &Rect, depth: u32, out: &mut Vec) { + out.push(to_detail(element, app_bounds, depth)); + for child in &element.children { + flatten(child, app_bounds, depth + 1, out); + } +} + +/// Start the accessibility worker thread and return its command channel. +#[cfg(target_os = "macos")] +pub fn spawn_ax_worker(udid: &str) -> Result> { + use accessibility_core::platform::ios_simulator::IOSSimulatorAccessibility; + + let mut reader = IOSSimulatorAccessibility::new(Some(udid))?; + let (tx, mut rx) = mpsc::unbounded_channel::(); + + std::thread::Builder::new() + .name("sim-ax".into()) + .spawn(move || { + while let Some(command) = rx.blocking_recv() { + match command { + AxCommand::Snapshot { reply } => { + let _ = reply.send(snapshot(&mut reader)); + } + AxCommand::HitTest { x, y, reply } => { + let _ = reply.send(hit_test(&mut reader, x, y)); + } + } + } + })?; + + Ok(tx) +} + +#[cfg(target_os = "macos")] +fn snapshot( + reader: &mut accessibility_core::platform::ios_simulator::IOSSimulatorAccessibility, +) -> Result { + let tree = reader.get_tree(&TreeFilter::default())?; + // `get_screen_bounds` is only populated once a tree has been read, so it + // has to be queried after the fetch above. + let app_bounds = reader.get_screen_bounds()?; + + let mut elements = Vec::with_capacity(tree.element_count); + flatten(&tree.root, &app_bounds, 0, &mut elements); + + Ok(AxSnapshot { + app_name: tree.app_name, + pid: tree.pid, + elements, + }) +} + +#[cfg(target_os = "macos")] +fn hit_test( + reader: &mut accessibility_core::platform::ios_simulator::IOSSimulatorAccessibility, + x: f64, + y: f64, +) -> Result> { + // The hit test wants macOS screen points, so the browser's normalized + // coordinates are mapped back through the app's bounds. + let app_bounds = reader + .get_screen_bounds() + .map_err(|_| anyhow!("no accessibility snapshot yet; fetch the tree first"))?; + + let screen_x = app_bounds.origin.x + x * app_bounds.size.width; + let screen_y = app_bounds.origin.y + y * app_bounds.size.height; + + Ok(reader + .element_at_point(screen_x, screen_y)? + .map(|element| to_detail(&element, &app_bounds, 0))) +} + +#[cfg(not(target_os = "macos"))] +pub fn spawn_ax_worker(_udid: &str) -> Result> { + anyhow::bail!("Simulator accessibility requires macOS") +} diff --git a/packages/accessibility-serve/src/http.rs b/packages/accessibility-serve/src/http.rs new file mode 100644 index 0000000..c06da72 --- /dev/null +++ b/packages/accessibility-serve/src/http.rs @@ -0,0 +1,203 @@ +//! HTTP and WebSocket surface. + +use std::sync::Arc; + +use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; +use axum::extract::{Query, State}; +use axum::http::StatusCode; +use axum::response::{Html, IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use serde::{Deserialize, Serialize}; + +use accessibility_core::video::FrameKind; + +use crate::avcc; +use crate::input::InputCommand; +use crate::session::SimSession; +use crate::webrtc_stream::WebRtcEngine; + +const INDEX_HTML: &str = include_str!("../static/index.html"); + +#[derive(Clone)] +pub struct AppState { + pub session: Arc, + pub webrtc: Arc, + pub default_transport: String, +} + +pub fn router(state: AppState) -> Router { + Router::new() + .route("/", get(index)) + .route("/api/config", get(config)) + .route("/api/ax/tree", get(ax_tree)) + .route("/api/ax/hit", get(ax_hit)) + .route("/webrtc/offer", post(webrtc_offer)) + .route("/ws/stream", get(stream_socket)) + .route("/ws/input", get(input_socket)) + .with_state(state) +} + +async fn index() -> Html<&'static str> { + Html(INDEX_HTML) +} + +#[derive(Serialize)] +struct ConfigResponse { + udid: String, + width: u32, + height: u32, + default_transport: String, + transports: Vec<&'static str>, + home_indicator_band: f64, +} + +async fn config(State(state): State) -> Json { + let device = state.session.device_info(); + Json(ConfigResponse { + udid: device.udid, + width: device.width, + height: device.height, + default_transport: state.default_transport.clone(), + transports: vec!["webrtc", "h264"], + home_indicator_band: crate::input::HOME_INDICATOR_BAND, + }) +} + +/// Map an `anyhow` error onto a 500 with the message preserved. +/// +/// These are developer-facing diagnostics on a locally served tool, so the +/// detail is more useful than it would be on a public endpoint. +fn internal_error(error: anyhow::Error) -> Response { + (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response() +} + +async fn ax_tree(State(state): State) -> Response { + match state.session.ax_snapshot().await { + Ok(snapshot) => Json(snapshot).into_response(), + Err(error) => internal_error(error), + } +} + +#[derive(Deserialize)] +struct HitQuery { + x: f64, + y: f64, +} + +async fn ax_hit(State(state): State, Query(query): Query) -> Response { + match state.session.ax_hit_test(query.x, query.y).await { + Ok(element) => Json(element).into_response(), + Err(error) => internal_error(error), + } +} + +#[derive(Deserialize)] +struct OfferRequest { + sdp: String, +} + +#[derive(Serialize)] +struct AnswerResponse { + sdp: String, +} + +async fn webrtc_offer(State(state): State, Json(offer): Json) -> Response { + match state + .webrtc + .answer(Arc::clone(&state.session), offer.sdp) + .await + { + Ok(sdp) => Json(AnswerResponse { sdp }).into_response(), + Err(error) => internal_error(error), + } +} + +async fn stream_socket( + State(state): State, + upgrade: WebSocketUpgrade, +) -> impl IntoResponse { + upgrade.on_upgrade(move |socket| pump_h264(state, socket)) +} + +/// Raw H.264 transport for browsers driving WebCodecs directly. +/// +/// This is the "expose a port and point something at it" path: no signaling, +/// no ICE, just length-prefixed frames. Also the fallback when WebRTC codec +/// negotiation fails. +async fn pump_h264(state: AppState, mut socket: WebSocket) { + let mut frames = state.session.subscribe(); + let mut sent_parameter_set = false; + + loop { + let frame = match frames.recv().await { + Ok(frame) => frame, + Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => { + state.session.request_keyframe(); + continue; + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => break, + }; + + // The encoder runs in Annex-B for WebRTC, so parameter sets arrive + // inline on each IDR rather than as their own frame. + if frame.kind == FrameKind::ParameterSet { + continue; + } + + // A decoder cannot start on a delta frame, and it needs the avcC + // record before anything else. + if !sent_parameter_set { + let Some((sps, pps)) = avcc::parameter_sets(&frame.data) else { + continue; + }; + let record = avcc::avcc_record(&sps, &pps); + let message = avcc::envelope(avcc::tag::PARAMETER_SET, &record); + if socket.send(Message::Binary(message.into())).await.is_err() { + break; + } + sent_parameter_set = true; + } + + let tag = match frame.kind { + FrameKind::Keyframe => avcc::tag::KEYFRAME, + _ => avcc::tag::DELTA, + }; + let payload = avcc::to_avcc(&frame.data); + if payload.is_empty() { + continue; + } + let message = avcc::envelope(tag, &payload); + if socket.send(Message::Binary(message.into())).await.is_err() { + break; + } + } +} + +async fn input_socket( + State(state): State, + upgrade: WebSocketUpgrade, +) -> impl IntoResponse { + upgrade.on_upgrade(move |socket| pump_input(state, socket)) +} + +async fn pump_input(state: AppState, mut socket: WebSocket) { + use futures_util::StreamExt; + + while let Some(Ok(message)) = socket.next().await { + let payload = match message { + Message::Text(text) => text.to_string(), + Message::Binary(bytes) => match String::from_utf8(bytes.to_vec()) { + Ok(text) => text, + Err(_) => continue, + }, + Message::Close(_) => break, + _ => continue, + }; + + match serde_json::from_str::(&payload) { + Ok(command) => state.session.send_input(command), + Err(error) => tracing::debug!("ignoring malformed input event: {error}"), + } + } +} diff --git a/packages/accessibility-serve/src/input.rs b/packages/accessibility-serve/src/input.rs new file mode 100644 index 0000000..2d3fa2b --- /dev/null +++ b/packages/accessibility-serve/src/input.rs @@ -0,0 +1,101 @@ +//! Input forwarding from the browser to the simulator's HID subsystem. +//! +//! All coordinates on this path are normalized 0..1 fractions of the display. +//! Keeping them normalized end to end avoids the points-vs-pixels-vs-scale +//! conversions that the accessibility side has to deal with. + +use anyhow::Result; +use serde::Deserialize; +use tokio::sync::mpsc; + +/// Touches below this fraction of the screen height are tagged as originating +/// from the bottom edge, which is what makes swipe-up-to-home work. Without +/// the edge hint iOS treats the drag as a normal in-app gesture. +pub const HOME_INDICATOR_BAND: f64 = 0.93; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum TouchPhase { + Begin, + Move, + End, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum HardwareButton { + Home, + Lock, + Siri, + SideButton, + ApplePay, +} + +/// A single input action to apply to the simulator. +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum InputCommand { + Touch { + phase: TouchPhase, + x: f64, + y: f64, + }, + Button { + button: HardwareButton, + }, + /// A US-keyboard virtual key code (HIToolbox `Events.h`). + Key { + key_code: u32, + }, +} + +/// Start the HID worker thread and return its command channel. +/// +/// The worker owns the `SimulatorHID` because it is not `Sync`, and because +/// HID sends block on a dispatch queue round trip. +#[cfg(target_os = "macos")] +pub fn spawn_input_worker(udid: &str) -> Result> { + use accessibility_ios_sys::{HardwareButton as SysButton, SimulatorHID, TouchPhase as SysPhase}; + + let hid = SimulatorHID::for_device(Some(udid))?; + let (tx, mut rx) = mpsc::unbounded_channel::(); + + std::thread::Builder::new() + .name("sim-input".into()) + .spawn(move || { + while let Some(command) = rx.blocking_recv() { + let result = match command { + InputCommand::Touch { phase, x, y } => { + let phase = match phase { + TouchPhase::Begin => SysPhase::Begin, + TouchPhase::Move => SysPhase::Move, + TouchPhase::End => SysPhase::End, + }; + hid.touch_normalized(x, y, phase) + } + InputCommand::Button { button } => { + let button = match button { + HardwareButton::Home => SysButton::Home, + HardwareButton::Lock => SysButton::Lock, + HardwareButton::Siri => SysButton::Siri, + HardwareButton::SideButton => SysButton::SideButton, + HardwareButton::ApplePay => SysButton::ApplePay, + }; + hid.press_button(button, 0) + } + InputCommand::Key { key_code } => hid.send_key(key_code), + }; + + if let Err(error) = result { + tracing::warn!("input event failed: {error}"); + } + } + })?; + + Ok(tx) +} + +#[cfg(not(target_os = "macos"))] +pub fn spawn_input_worker(_udid: &str) -> Result> { + anyhow::bail!("Simulator input requires macOS") +} diff --git a/packages/accessibility-serve/src/lib.rs b/packages/accessibility-serve/src/lib.rs new file mode 100644 index 0000000..d275a26 --- /dev/null +++ b/packages/accessibility-serve/src/lib.rs @@ -0,0 +1,109 @@ +//! Serve a live, interactive iOS Simulator stream in the browser. +//! +//! Captures the simulator framebuffer, encodes it with VideoToolbox, and +//! offers it over WebRTC (default) or as raw H.264 for browsers driving +//! WebCodecs. Pointer and keyboard input flow back over a WebSocket, and the +//! accessibility tree is exposed so the UI can inspect elements. + +pub mod avcc; +pub mod ax; +pub mod http; +pub mod input; +pub mod session; +pub mod webrtc_stream; + +use std::net::SocketAddr; +use std::sync::Arc; + +use anyhow::{Context, Result}; + +use accessibility_core::video::VideoConfig; + +pub use session::SimSession; + +/// Which transport the web UI should try first. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Transport { + /// Real-time media track. Lowest latency, adapts to loss. + #[default] + WebRtc, + /// Length-prefixed H.264 over a WebSocket, decoded with WebCodecs. + /// Simpler to tunnel and to consume from non-browser clients. + H264, +} + +impl Transport { + fn as_str(self) -> &'static str { + match self { + Transport::WebRtc => "webrtc", + Transport::H264 => "h264", + } + } +} + +impl std::str::FromStr for Transport { + type Err = anyhow::Error; + + fn from_str(value: &str) -> Result { + match value { + "webrtc" => Ok(Transport::WebRtc), + "h264" | "avcc" => Ok(Transport::H264), + other => anyhow::bail!("Unknown transport '{other}' (expected webrtc or h264)"), + } + } +} + +#[derive(Debug, Clone)] +pub struct ServeConfig { + /// Simulator to serve. `None` picks the first booted device. + pub udid: Option, + pub address: SocketAddr, + pub transport: Transport, + pub video: VideoConfig, + /// ICE servers for WebRTC. Empty means host candidates only, which is + /// correct for loopback and LAN use. + pub ice_servers: Vec, +} + +impl Default for ServeConfig { + fn default() -> Self { + Self { + udid: None, + address: SocketAddr::from(([127, 0, 0, 1], 3200)), + transport: Transport::default(), + video: VideoConfig::default(), + ice_servers: Vec::new(), + } + } +} + +/// Start capturing and serve until the process is interrupted. +pub async fn serve(config: ServeConfig) -> Result<()> { + let session = SimSession::start(config.udid.as_deref(), config.video) + .context("failed to start simulator capture")?; + let device = session.device_info(); + + let webrtc = Arc::new( + webrtc_stream::WebRtcEngine::new(config.ice_servers.clone()) + .context("failed to initialize WebRTC")?, + ); + + let state = http::AppState { + session, + webrtc, + default_transport: config.transport.as_str().to_string(), + }; + + let listener = tokio::net::TcpListener::bind(config.address) + .await + .with_context(|| format!("failed to bind {}", config.address))?; + let bound = listener.local_addr()?; + + println!("serving simulator {}", device.udid); + println!(" transport : {}", config.transport.as_str()); + println!(" preview : http://{bound}"); + + axum::serve(listener, http::router(state)) + .await + .context("server error") +} diff --git a/packages/accessibility-serve/src/session.rs b/packages/accessibility-serve/src/session.rs new file mode 100644 index 0000000..dfa101f --- /dev/null +++ b/packages/accessibility-serve/src/session.rs @@ -0,0 +1,160 @@ +//! Ownership of the per-device simulator resources. +//! +//! The simulator's Objective-C objects are not `Sync` and their calls block, so +//! each one lives on a dedicated thread behind a command channel rather than in +//! a mutex on the async runtime. +//! +//! Input and accessibility get *separate* threads on purpose: an accessibility +//! tree fetch can take hundreds of milliseconds, and pointer events queued +//! behind one would make the stream feel broken. + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use anyhow::{Result, anyhow}; +use tokio::sync::{broadcast, mpsc, oneshot}; + +use accessibility_core::video::{EncodedFrame, FrameKind, VideoCapture, VideoConfig}; + +use crate::ax::{AxCommand, AxSnapshot, ElementDetail, spawn_ax_worker}; +use crate::input::{InputCommand, spawn_input_worker}; + +/// How many encoded frames to buffer per subscriber. +/// +/// Small on purpose: for interactive video a dropped frame is better than a +/// late one, and a slow client should not be able to inflate memory. +const FRAME_BUFFER: usize = 16; + +/// Geometry and identity of the device being served. +#[derive(Debug, Clone, serde::Serialize)] +pub struct DeviceInfo { + pub udid: String, + pub width: u32, + pub height: u32, +} + +pub struct SimSession { + device_udid: String, + capture: Box, + frames: broadcast::Sender, + /// Most recent parameter set, replayed to clients that join mid-stream. + latest_parameter_set: Arc>>, + frames_encoded: Arc, + input: mpsc::UnboundedSender, + ax: mpsc::UnboundedSender, +} + +impl SimSession { + /// Attach to a booted simulator and start capturing. + pub fn start(udid: Option<&str>, config: VideoConfig) -> Result> { + let (frames, _) = broadcast::channel(FRAME_BUFFER); + let latest_parameter_set = Arc::new(std::sync::Mutex::new(None)); + let frames_encoded = Arc::new(AtomicU64::new(0)); + + let sink = { + let frames = frames.clone(); + let latest_parameter_set = Arc::clone(&latest_parameter_set); + let frames_encoded = Arc::clone(&frames_encoded); + Arc::new(move |frame: EncodedFrame| { + if frame.kind == FrameKind::ParameterSet { + *latest_parameter_set.lock().unwrap() = Some(frame.clone()); + } + frames_encoded.fetch_add(1, Ordering::Relaxed); + // A send error just means nobody is watching yet. + let _ = frames.send(frame); + }) + }; + + let (capture, resolved_udid) = start_capture(udid, &config, sink)?; + let input = spawn_input_worker(&resolved_udid)?; + let ax = spawn_ax_worker(&resolved_udid)?; + + Ok(Arc::new(Self { + device_udid: resolved_udid, + capture, + frames, + latest_parameter_set, + frames_encoded, + input, + ax, + })) + } + + pub fn subscribe(&self) -> broadcast::Receiver { + let receiver = self.frames.subscribe(); + // A new subscriber cannot decode anything until the next keyframe, so + // ask for one immediately instead of making them wait out the interval. + self.capture.request_keyframe(); + receiver + } + + pub fn latest_parameter_set(&self) -> Option { + self.latest_parameter_set.lock().unwrap().clone() + } + + pub fn request_keyframe(&self) { + self.capture.request_keyframe(); + } + + pub fn frames_encoded(&self) -> u64 { + self.frames_encoded.load(Ordering::Relaxed) + } + + pub fn device_info(&self) -> DeviceInfo { + let geometry = self.capture.geometry(); + DeviceInfo { + udid: self.device_udid.clone(), + width: geometry.width, + height: geometry.height, + } + } + + /// Queue an input event. Fire-and-forget: pointer events must never block + /// the socket reader. + pub fn send_input(&self, command: InputCommand) { + let _ = self.input.send(command); + } + + pub async fn ax_snapshot(&self) -> Result { + let (tx, rx) = oneshot::channel(); + self.ax + .send(AxCommand::Snapshot { reply: tx }) + .map_err(|_| anyhow!("accessibility worker stopped"))?; + rx.await.map_err(|_| anyhow!("accessibility worker stopped"))? + } + + pub async fn ax_hit_test(&self, x: f64, y: f64) -> Result> { + let (tx, rx) = oneshot::channel(); + self.ax + .send(AxCommand::HitTest { x, y, reply: tx }) + .map_err(|_| anyhow!("accessibility worker stopped"))?; + rx.await.map_err(|_| anyhow!("accessibility worker stopped"))? + } +} + +#[cfg(target_os = "macos")] +fn start_capture( + udid: Option<&str>, + config: &VideoConfig, + sink: accessibility_core::video::FrameSink, +) -> Result<(Box, String)> { + use accessibility_core::platform::ios_simulator::SimulatorVideoCapture; + + // Resolve the concrete UDID up front so the input and accessibility + // workers bind to the same device the video came from, even when the + // caller passed `None`. + let resolved = accessibility_ios_sys::SimFramebuffer::new(udid)? + .device_udid() + .to_string(); + let capture = SimulatorVideoCapture::start(&resolved, config, sink)?; + Ok((Box::new(capture), resolved)) +} + +#[cfg(not(target_os = "macos"))] +fn start_capture( + _udid: Option<&str>, + _config: &VideoConfig, + _sink: accessibility_core::video::FrameSink, +) -> Result<(Box, String)> { + anyhow::bail!("Serving an iOS Simulator requires macOS") +} diff --git a/packages/accessibility-serve/src/webrtc_stream.rs b/packages/accessibility-serve/src/webrtc_stream.rs new file mode 100644 index 0000000..e5f5a2b --- /dev/null +++ b/packages/accessibility-serve/src/webrtc_stream.rs @@ -0,0 +1,183 @@ +//! WebRTC video transport. +//! +//! Signaling is a single HTTP round trip: the browser posts a complete offer, +//! we answer with a complete SDP once ICE gathering finishes. No trickle, no +//! long-lived signaling socket. For a locally served simulator that is enough, +//! and it keeps the browser side to about thirty lines. +//! +//! Each viewer gets its own peer connection and its own forwarding task, so +//! one stalled client cannot wedge the others. + +use std::sync::Arc; + +use anyhow::{Result, anyhow}; +use webrtc::api::interceptor_registry::register_default_interceptors; +use webrtc::api::media_engine::{MIME_TYPE_H264, MediaEngine}; +use webrtc::api::{API, APIBuilder}; +use webrtc::ice_transport::ice_server::RTCIceServer; +use webrtc::interceptor::registry::Registry; +use webrtc::media::Sample; +use webrtc::peer_connection::configuration::RTCConfiguration; +use webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState; +use webrtc::peer_connection::sdp::session_description::RTCSessionDescription; +use webrtc::rtcp::payload_feedbacks::full_intra_request::FullIntraRequest; +use webrtc::rtcp::payload_feedbacks::picture_loss_indication::PictureLossIndication; +use webrtc::rtp_transceiver::rtp_codec::RTCRtpCodecCapability; +use webrtc::track::track_local::TrackLocal; +use webrtc::track::track_local::track_local_static_sample::TrackLocalStaticSample; + +use accessibility_core::video::FrameKind; + +use crate::session::SimSession; + +pub struct WebRtcEngine { + api: API, + config: RTCConfiguration, +} + +impl WebRtcEngine { + pub fn new(ice_servers: Vec) -> Result { + let mut media_engine = MediaEngine::default(); + media_engine.register_default_codecs()?; + + let registry = register_default_interceptors(Registry::new(), &mut media_engine)?; + let api = APIBuilder::new() + .with_media_engine(media_engine) + .with_interceptor_registry(registry) + .build(); + + let config = RTCConfiguration { + ice_servers: if ice_servers.is_empty() { + // A simulator served on loopback or a LAN needs no STUN; host + // candidates are sufficient and avoid a pointless round trip + // to a public server. + Vec::new() + } else { + vec![RTCIceServer { + urls: ice_servers, + ..Default::default() + }] + }, + ..Default::default() + }; + + Ok(Self { api, config }) + } + + /// Answer a browser offer, wiring a fresh track to the capture stream. + pub async fn answer(&self, session: Arc, offer_sdp: String) -> Result { + let peer = Arc::new(self.api.new_peer_connection(self.config.clone()).await?); + + let track = Arc::new(TrackLocalStaticSample::new( + RTCRtpCodecCapability { + mime_type: MIME_TYPE_H264.to_owned(), + ..Default::default() + }, + "video".to_owned(), + format!("sim-{}", session.device_info().udid), + )); + + let sender = peer + .add_track(Arc::clone(&track) as Arc) + .await?; + + // Sender RTCP has to be drained or feedback never gets processed. PLI + // and FIR both mean "I cannot decode, send me a fresh IDR". + { + let session = Arc::clone(&session); + tokio::spawn(async move { + let mut buffer = vec![0u8; 1500]; + while let Ok((packets, _)) = sender.read(&mut buffer).await { + for packet in packets { + let any = packet.as_any(); + if any.downcast_ref::().is_some() + || any.downcast_ref::().is_some() + { + session.request_keyframe(); + } + } + } + }); + } + + let forwarder = spawn_forwarder(Arc::clone(&session), Arc::clone(&track)); + + // Tear the forwarding task down when the viewer goes away, otherwise + // every reconnect would leak a subscriber on the broadcast channel. + { + let forwarder = Arc::new(std::sync::Mutex::new(Some(forwarder))); + peer.on_peer_connection_state_change(Box::new(move |state| { + if matches!( + state, + RTCPeerConnectionState::Failed + | RTCPeerConnectionState::Disconnected + | RTCPeerConnectionState::Closed + ) && let Some(handle) = forwarder.lock().unwrap().take() + { + handle.abort(); + } + Box::pin(async {}) + })); + } + + peer.set_remote_description(RTCSessionDescription::offer(offer_sdp)?) + .await?; + let answer = peer.create_answer(None).await?; + + let mut gathering_complete = peer.gathering_complete_promise().await; + peer.set_local_description(answer).await?; + let _ = gathering_complete.recv().await; + + peer.local_description() + .await + .map(|description| description.sdp) + .ok_or_else(|| anyhow!("WebRTC produced no local description")) + } +} + +/// Pump encoded frames from the capture broadcast onto a viewer's track. +fn spawn_forwarder( + session: Arc, + track: Arc, +) -> tokio::task::JoinHandle<()> { + let mut frames = session.subscribe(); + + tokio::spawn(async move { + // Timestamps are derived from arrival rather than a fixed cadence, + // because the simulator only paints when something changes: an idle + // screen produces ~5fps and a busy one ~60fps. + let mut previous = tokio::time::Instant::now(); + + loop { + let frame = match frames.recv().await { + Ok(frame) => frame, + // Lagging just means this viewer fell behind; the next + // keyframe will resynchronize it. + Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => { + session.request_keyframe(); + continue; + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => break, + }; + + // In Annex-B the parameter sets ride inline ahead of each IDR, so + // there is nothing separate to send. + if frame.kind == FrameKind::ParameterSet { + continue; + } + + let now = tokio::time::Instant::now(); + let duration = now.duration_since(previous); + previous = now; + + let sample = Sample { + data: frame.data, + duration, + ..Default::default() + }; + if track.write_sample(&sample).await.is_err() { + break; + } + } + }) +} diff --git a/packages/accessibility-serve/static/index.html b/packages/accessibility-serve/static/index.html new file mode 100644 index 0000000..be42eb2 --- /dev/null +++ b/packages/accessibility-serve/static/index.html @@ -0,0 +1,519 @@ + + + + + +Simulator + + + + +
+
+ + +
+
+ +
+ + +
connecting
+
+
+ + + + + + From 6e2f280ae950b1b5c2c0cfe7835b3dbb97346168 Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Sun, 26 Jul 2026 20:47:28 -0700 Subject: [PATCH 04/19] Document simulator workflow and private framework pitfalls Records the things that cost real debugging time: CoreSimulator's proxied objects breaking msg_send!, blocks needing BLOCK_HAS_SIGNATURE, the load- bearing screen-callback registration, and the WebCodecs hardwareAcceleration trap. Also notes that the cli_macos tests need a TCC-permitted GUI session, so their failures are environmental rather than regressions. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <166158716+staging-devin-ai-integration[bot]@users.noreply.github.com> --- AGENTS.md | 102 ++++++++++++++++++ .../src/macos/framebuffer.rs | 3 - 2 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..7dcfdb8 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,102 @@ +# accessibility-cli + +Cross-platform accessibility tree reading, querying, and automation, plus a +live iOS Simulator stream. + +## Build and verify + +```sh +cargo build --workspace +cargo clippy --workspace --all-targets +cargo test --workspace --lib # unit tests, all green +cargo test -p accessibility-cli --test cli_smoke +``` + +### Tests that need a permitted GUI session + +`packages/accessibility-cli/tests/cli_macos.rs` drives the real macOS +accessibility API: it launches Calculator and reads its window tree. It fails +with "Accessibility permissions not granted" or "Calculator never opened a +window" unless the terminal running the tests has been granted +System Settings > Privacy & Security > Accessibility. These failures are +environmental, not regressions. + +## iOS Simulator work + +Anything touching the simulator needs Xcode (not just Command Line Tools) and +a booted device: + +```sh +export DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer +xcrun simctl list devices booted +``` + +`xcode-select -p` on this machine points at `/Library/Developer/CommandLineTools`, +which has no simulator frameworks, so `DEVELOPER_DIR` matters. + +Verify the capture and encode pipeline end to end without a browser: + +```sh +cargo run -p accessibility-ios-sys --example framebuffer_probe +``` + +It fails loudly if no frames arrive, if no keyframe is produced, or if any +access unit is not Annex-B framed. + +Serve it: + +```sh +cargo run -p accessibility-cli -- serve-sim --port 3200 +``` + +## Private framework notes + +These cost real debugging time; they are not obvious from the outside. + +- **CoreSimulator hands back proxies.** IO ports and display descriptors are + `ROCKRemoteProxy` objects that implement their interface through forwarding. + `objc2::msg_send!` panics on them in debug builds because its verification + looks the selector up with `class_getInstanceMethod`. Use the helpers in + `macos/dynamic.rs`, always guarded by `responds_to`. + +- **Blocks need a type signature.** ROCKit marshals block arguments across the + proxy boundary by reading the block's ObjC type encoding, which requires + `BLOCK_HAS_SIGNATURE`. `block2` does not emit that flag (there is a TODO in + its `global.rs`), so passing an `RcBlock` aborts with "Block is missing + signature field". `macos/blocks.c` creates the blocks with clang instead. + See `macos/void_block.rs`. + +- **Registering screen callbacks is load-bearing.** It is what makes + SimulatorKit attach the display pipeline and populate `framebufferSurface`. + Reading the property without registering does not reliably work. + +- **Several ports share `com.apple.framebuffer.display`** (main screen plus + secondary planes). Register on all of them and pick the largest live surface + each frame; the first match is often a small overlay. + +- **The framebuffer IOSurface is recycled in place.** Retaining the + `CVPixelBuffer` does not help because the surface mutates underneath it, so + frames are deep-copied before going downstream. This is the main CPU cost in + the capture path. + +- **SimulatorKit moved in Xcode 27** from `Developer/Library/PrivateFrameworks` + to `Contents/SharedFrameworks`. Both are probed. + +## Coordinate spaces + +- **Input** is normalized 0..1 across the whole path, browser to HID. Nothing + converts to points or pixels, which avoids the scale ambiguity entirely. +- **Accessibility frames** come back in macOS screen points, positioned + wherever the Simulator window sits. Normalize against the app's own bounds + before exposing them: `(rect.origin - app_bounds.origin) / app_bounds.size`. + `get_screen_bounds` is only populated after a tree has been read. + +## Browser video gotcha + +Do not pass `hardwareAcceleration: "prefer-hardware"` to `VideoDecoder`. +Despite the name it is treated as a requirement, and phone-shaped resolutions +like 1206x2622 exceed what hardware decoders accept, making the configuration +unsupported outright. Also note the real WebCodecs member is +`optimizeForLatency`, not `optimizeFor`. `configure()` reports success +synchronously and only surfaces the failure through the async error callback, +so check `isConfigSupported` first. diff --git a/packages/accessibility-ios-sys/src/macos/framebuffer.rs b/packages/accessibility-ios-sys/src/macos/framebuffer.rs index 82fcbb2..8523191 100644 --- a/packages/accessibility-ios-sys/src/macos/framebuffer.rs +++ b/packages/accessibility-ios-sys/src/macos/framebuffer.rs @@ -542,6 +542,3 @@ unsafe fn device_udid_string(device: *mut AnyObject) -> Result { unsafe { super::common::nsstring_to_string_static(string) } .ok_or_else(|| anyhow!("Failed to read simulator UDID")) } - -// Silence an unused import warning on the c_void alias used by the pool module. -const _: Option<*const c_void> = None; From 4f8e4de89d4082b7a6ecd04e12e0facf98b79bd1 Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Sun, 26 Jul 2026 21:05:15 -0700 Subject: [PATCH 05/19] Rotate the simulator via GSEvent; measure and reject a GPU frame copy Adds SimulatorHID::set_orientation. Orientation does not travel over Indigo like touches do: it is a GSEvent delivered by mach message to the simulator's PurpleWorkspacePort, the same path Simulator.app uses for Device > Rotate. Requires Simulator.app to be running, since it publishes the port. Also investigated replacing the per-frame framebuffer memcpy with a Metal blit, since both surfaces are already IOSurface-backed. Benchmarked against a real 1206x2622 surface with cache-cold sources: CPU memcpy 0.377 ms/copy 33.5 GB/s Metal blit 0.517 ms/copy 24.5 GB/s The GPU path is 37% slower. Command buffer submission and the waitUntilCompleted round trip cost more than the copy saves, because unified memory already makes the CPU path fast. At 60fps the memcpy is ~23 ms/s, about 2% of one core, so it was never the bottleneck it looked like. Reverted the Metal path and recorded the numbers so this is not retried blindly. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <166158716+staging-devin-ai-integration[bot]@users.noreply.github.com> --- .../examples/rotation_probe.rs | 40 +++++++++ packages/accessibility-ios-sys/src/macos.rs | 2 +- .../accessibility-ios-sys/src/macos/hid.rs | 86 +++++++++++++++++++ .../src/macos/pixel_buffer.rs | 15 +++- 4 files changed, 140 insertions(+), 3 deletions(-) create mode 100644 packages/accessibility-ios-sys/examples/rotation_probe.rs diff --git a/packages/accessibility-ios-sys/examples/rotation_probe.rs b/packages/accessibility-ios-sys/examples/rotation_probe.rs new file mode 100644 index 0000000..d413606 --- /dev/null +++ b/packages/accessibility-ios-sys/examples/rotation_probe.rs @@ -0,0 +1,40 @@ +//! Determine whether the framebuffer reflects device orientation. +//! +//! Run with: `cargo run -p accessibility-ios-sys --example rotation_probe` + +#[cfg(not(target_os = "macos"))] +fn main() { + eprintln!("rotation_probe only runs on macOS"); + std::process::exit(1); +} + +#[cfg(target_os = "macos")] +fn main() -> anyhow::Result<()> { + use accessibility_ios_sys::{Orientation, SimFramebuffer, SimulatorHID}; + use std::time::Duration; + + let mut framebuffer = SimFramebuffer::new(None)?; + framebuffer.start()?; + let hid = SimulatorHID::for_device(None)?; + + let settle = |label: &str, framebuffer: &SimFramebuffer| { + std::thread::sleep(Duration::from_millis(2500)); + let stats = framebuffer.stats(); + println!("{label:<22} framebuffer = {}x{}", stats.width, stats.height); + }; + + settle("initial", &framebuffer); + + for (label, orientation) in [ + ("landscape_left", Orientation::LandscapeLeft), + ("landscape_right", Orientation::LandscapeRight), + ("portrait_upside_down", Orientation::PortraitUpsideDown), + ("portrait", Orientation::Portrait), + ] { + hid.set_orientation(orientation)?; + settle(label, &framebuffer); + } + + framebuffer.stop(); + Ok(()) +} diff --git a/packages/accessibility-ios-sys/src/macos.rs b/packages/accessibility-ios-sys/src/macos.rs index fef79b8..be4692d 100644 --- a/packages/accessibility-ios-sys/src/macos.rs +++ b/packages/accessibility-ios-sys/src/macos.rs @@ -71,6 +71,6 @@ pub use common::{ }; pub use encoder::{ChunkKind, ChunkSink, EncodedChunk, EncoderConfig, H264Encoder, NalFormat}; pub use framebuffer::{CapturedFrame, FrameSink, FramebufferStats, SimFramebuffer}; -pub use hid::{SimulatorHID, TouchPhase}; +pub use hid::{Orientation, SimulatorHID, TouchPhase}; pub use reader::IOSSimulatorAccessibility; pub use stream::{ScreenGeometry, SimVideoStream}; diff --git a/packages/accessibility-ios-sys/src/macos/hid.rs b/packages/accessibility-ios-sys/src/macos/hid.rs index ea73d5f..9a32005 100644 --- a/packages/accessibility-ios-sys/src/macos/hid.rs +++ b/packages/accessibility-ios-sys/src/macos/hid.rs @@ -33,8 +33,19 @@ pub enum TouchPhase { End, } +/// Device orientation, using the GSEvent numbering. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u32)] +pub enum Orientation { + Portrait = 1, + PortraitUpsideDown = 2, + LandscapeRight = 3, + LandscapeLeft = 4, +} + pub struct SimulatorHID { client: *mut AnyObject, // SimDeviceLegacyHIDClient + device: *mut AnyObject, // SimDevice, retained for GSEvent port lookup queue: *mut AnyObject, // dispatch_queue_t screen_size: (f64, f64), screen_scale: f64, @@ -130,6 +141,7 @@ impl SimulatorHID { Ok(Self { client, + device, queue, screen_size, screen_scale, @@ -232,6 +244,80 @@ impl SimulatorHID { self.send_touch(x, y, direction) } + /// Rotate the device. + /// + /// Orientation does not travel over Indigo like touches do. It is a + /// GSEvent delivered by mach message to the simulator's + /// `PurpleWorkspacePort`, which is the same path Simulator.app itself uses + /// when you pick Device > Rotate. + pub fn set_orientation(&self, orientation: Orientation) -> Result<()> { + // GSEvent constants, as used by Simulator.app and idb. + const GSEVENT_MACH_MESSAGE_ID: i32 = 0x7B; + const GSEVENT_TYPE_ORIENTATION_CHANGED: u32 = 50; + const GSEVENT_HOST_FLAG: u32 = 0x0002_0000; + const MACH_MSG_TYPE_COPY_SEND: u32 = 19; + /// `align4(4 + 0x6B)` — a GSEvent header plus a 4-byte payload. + const MESSAGE_SIZE: u32 = 108; + + unsafe extern "C" { + fn mach_msg_send(message: *mut c_void) -> i32; + } + + let port = self.purple_workspace_port()?; + + // Oversized so the 108-byte message is comfortably in bounds. + let mut buffer = [0u8; 112]; + let base = buffer.as_mut_ptr(); + unsafe { + // mach_msg_header_t: bits, size, remote, local, voucher, id. + std::ptr::write_unaligned(base.add(0x00) as *mut u32, MACH_MSG_TYPE_COPY_SEND); + std::ptr::write_unaligned(base.add(0x04) as *mut u32, MESSAGE_SIZE); + std::ptr::write_unaligned(base.add(0x08) as *mut u32, port); + std::ptr::write_unaligned(base.add(0x0c) as *mut u32, 0); + std::ptr::write_unaligned(base.add(0x10) as *mut u32, 0); + std::ptr::write_unaligned(base.add(0x14) as *mut i32, GSEVENT_MACH_MESSAGE_ID); + + std::ptr::write_unaligned( + base.add(0x18) as *mut u32, + GSEVENT_TYPE_ORIENTATION_CHANGED | GSEVENT_HOST_FLAG, + ); + // record_info_size, then the orientation itself. + std::ptr::write_unaligned(base.add(0x48) as *mut u32, 4); + std::ptr::write_unaligned(base.add(0x4c) as *mut u32, orientation as u32); + } + + let result = unsafe { mach_msg_send(base as *mut c_void) }; + if result != 0 { + return Err(anyhow!("mach_msg_send for orientation failed: {result}")); + } + Ok(()) + } + + /// Look up the simulator's `PurpleWorkspacePort` mach port. + fn purple_workspace_port(&self) -> Result { + let name = NSString::from_str("PurpleWorkspacePort"); + let mut error: *mut AnyObject = std::ptr::null_mut(); + let port: u32 = unsafe { msg_send![self.device, lookup: &*name, error: &mut error] }; + + if port == 0 { + let detail = unsafe { + (!error.is_null()) + .then(|| { + let description: *mut AnyObject = msg_send![error, localizedDescription]; + nsstring_to_string_static(description) + }) + .flatten() + }; + // The port is published by Simulator.app, not by the runtime, so a + // headless `simctl boot` will not have one. + return Err(anyhow!( + "PurpleWorkspacePort unavailable ({}). Rotation needs Simulator.app running.", + detail.as_deref().unwrap_or("no error detail") + )); + } + Ok(port) + } + /// Press a hardware button. /// /// # Arguments diff --git a/packages/accessibility-ios-sys/src/macos/pixel_buffer.rs b/packages/accessibility-ios-sys/src/macos/pixel_buffer.rs index bd02c34..28d6695 100644 --- a/packages/accessibility-ios-sys/src/macos/pixel_buffer.rs +++ b/packages/accessibility-ios-sys/src/macos/pixel_buffer.rs @@ -7,8 +7,19 @@ //! copy or it will encode torn frames. //! //! The copy is a straightforward row-wise `memcpy` into a size-keyed -//! `CVPixelBufferPool`. It is the single largest CPU cost in the capture path; -//! a Metal blit or an IOSurface ring would avoid it, but correctness first. +//! `CVPixelBufferPool`. +//! +//! This looks like an obvious target for a GPU blit, and it isn't. Both +//! surfaces are IOSurface-backed, so they can be wrapped as `MTLTexture`s and +//! copied with a blit encoder — but that was measured at **0.517 ms/frame +//! against 0.377 ms for the `memcpy`** on an iPhone 17 surface (1206x2622, +//! 12.1 MB), with cache-cold sources. Command buffer submission plus the +//! `waitUntilCompleted` round trip costs more than the copy saves, because +//! unified memory already gives the CPU path ~34 GB/s. +//! +//! At 60fps the `memcpy` is ~23 ms/s, or about 2% of one core. It is not worth +//! optimizing, and the GPU version was slower and more complex. Measure before +//! trying again. use std::ptr::NonNull; From f11e81f6f76d75dbeca0b730a1cbc0034257d131 Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Sun, 26 Jul 2026 21:14:54 -0700 Subject: [PATCH 06/19] Add scroll, orientation, device settings, and a side-ribbon UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scroll. iOS has no scroll wheel, so wheel deltas are synthesized into a touch drag. The awkward part is that a real finger runs out of screen, so the virtual contact point is lifted and re-planted in the middle whenever it nears an edge, and lifted entirely once the wheel goes quiet. The input worker moved to a channel with a timeout so it can notice that quiet period. Orientation. Rotation goes out as a GSEvent to PurpleWorkspacePort. The framebuffer never rotates — the simulator draws rotated content into a fixed portrait surface — so the browser rotates for display and un-rotates pointer coordinates. Orientation is tracked server-side and seeded at startup from the accessibility bounds aspect, so attaching to an already-rotated device no longer renders sideways. Device settings. Appearance, increase contrast and content size over simctl ui. Those are the only three simctl implements; the rest of Xcode's Devices window needs an in-simulator helper and is deliberately out of scope. System edge gestures. Swipe-up-to-home never worked: the edge argument to IndigoHIDMessageForMouseNSEvent was declared as a bool in the register the ABI uses for the edge, so every gesture was flagged as edgeless. Fixed the signature and plumbed the edge through, rotating it with the device. Fixed an accessibility coordinate bug this surfaced. AX frames are in logical space — iOS reports landscape bounds as 874x402 — so they were being rotated a second time for display and hit tests were being sent in the wrong space. Correct in portrait, wrong everywhere else. The two spaces are now documented where they are used. The UI is now a vertical ribbon beside the device rather than a toolbar underneath, with the inspector and settings as drawers. Also investigated replacing the per-frame memcpy with a Metal blit. It is 37% slower (0.517 ms vs 0.377 ms on a cache-cold 12.1 MB surface) because the command buffer round trip costs more than the copy saves on unified memory. Reverted, with the numbers recorded. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <166158716+staging-devin-ai-integration[bot]@users.noreply.github.com> --- AGENTS.md | 64 +- packages/accessibility-ios-sys/src/macos.rs | 2 +- .../accessibility-ios-sys/src/macos/hid.rs | 71 +- packages/accessibility-serve/src/ax.rs | 23 +- packages/accessibility-serve/src/http.rs | 52 +- packages/accessibility-serve/src/input.rs | 281 +++++++- packages/accessibility-serve/src/lib.rs | 4 + packages/accessibility-serve/src/session.rs | 73 +- packages/accessibility-serve/src/settings.rs | 173 +++++ .../accessibility-serve/static/index.html | 680 ++++++++++++------ 10 files changed, 1163 insertions(+), 260 deletions(-) create mode 100644 packages/accessibility-serve/src/settings.rs diff --git a/AGENTS.md b/AGENTS.md index 7dcfdb8..2f8501d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -84,12 +84,64 @@ These cost real debugging time; they are not obvious from the outside. ## Coordinate spaces -- **Input** is normalized 0..1 across the whole path, browser to HID. Nothing - converts to points or pixels, which avoids the scale ambiguity entirely. -- **Accessibility frames** come back in macOS screen points, positioned - wherever the Simulator window sits. Normalize against the app's own bounds - before exposing them: `(rect.origin - app_bounds.origin) / app_bounds.size`. - `get_screen_bounds` is only populated after a tree has been read. +There are **two** normalized spaces and they only coincide in portrait, which +makes conflating them very easy and the bug invisible until you rotate. + +- **Raw framebuffer space** — what HID input uses. The framebuffer is always + portrait-native: rotating the device rotates the *content* inside a + fixed-size surface, so pointer coordinates must be un-rotated before + injection. +- **Logical space** — what accessibility uses. iOS has already applied the + rotation: in landscape the app reports its own bounds as 874x402 rather than + 402x874, so normalizing against them yields upright coordinates that need no + further rotation. + +Concretely, in the web UI a tap sends raw coordinates while a hit test sends +display coordinates, and AX rects are drawn without rotation. + +Normalize AX rects with `(rect.origin - app_bounds.origin) / app_bounds.size`. +`get_screen_bounds` is only populated after a tree has been read. + +## Orientation + +The framebuffer never changes size, so orientation cannot be recovered from +the video. It is tracked server-side and seeded at startup from the +accessibility bounds aspect ratio, which is the only cheap signal — and it +only distinguishes landscape from portrait, not left from right. + +Rotation itself is a GSEvent mach message to `PurpleWorkspacePort`, not an +Indigo event. It needs Simulator.app running, because the runtime alone does +not publish that port. + +## System edge gestures + +Swipe-up-to-home does not work unless the touch is flagged with the screen +edge it started from. That edge is the 7th argument to +`IndigoHIDMessageForMouseNSEvent`; on arm64 it lands in x4 while the `NSSize` +argument occupies d0/d1, so a wrong declaration can silently pass zero there +and every gesture just becomes an ordinary drag. The same edge must be +supplied for every event in the gesture, and the edge is in *raw* framebuffer +space, so it rotates with the device. + +## Device settings + +`simctl ui` only implements `appearance`, `increase_contrast` and +`content_size`. The other options in Xcode's Devices window (reduce motion, +colour filters, transparency, VoiceOver) have no simctl verb and need a helper +binary spawned inside the simulator that drives the private libAccessibility +setters. + +## Performance + +The per-frame framebuffer `memcpy` looks like an obvious optimization target +and is not. Measured on a 1206x2622 surface with cache-cold sources: + + CPU memcpy 0.377 ms/copy 33.5 GB/s + Metal blit 0.517 ms/copy 24.5 GB/s + +A GPU blit is *slower*, because command buffer submission plus the +`waitUntilCompleted` round trip costs more than the copy saves on unified +memory. At 60fps the copy is ~23 ms/s, about 2% of one core. ## Browser video gotcha diff --git a/packages/accessibility-ios-sys/src/macos.rs b/packages/accessibility-ios-sys/src/macos.rs index be4692d..dece57e 100644 --- a/packages/accessibility-ios-sys/src/macos.rs +++ b/packages/accessibility-ios-sys/src/macos.rs @@ -71,6 +71,6 @@ pub use common::{ }; pub use encoder::{ChunkKind, ChunkSink, EncodedChunk, EncoderConfig, H264Encoder, NalFormat}; pub use framebuffer::{CapturedFrame, FrameSink, FramebufferStats, SimFramebuffer}; -pub use hid::{Orientation, SimulatorHID, TouchPhase}; +pub use hid::{Orientation, SimulatorHID, TouchEdge, TouchPhase}; pub use reader::IOSSimulatorAccessibility; pub use stream::{ScreenGeometry, SimVideoStream}; diff --git a/packages/accessibility-ios-sys/src/macos/hid.rs b/packages/accessibility-ios-sys/src/macos/hid.rs index 9a32005..620098a 100644 --- a/packages/accessibility-ios-sys/src/macos/hid.rs +++ b/packages/accessibility-ios-sys/src/macos/hid.rs @@ -11,12 +11,24 @@ use super::*; /// Function pointer types for Indigo message creation (loaded from SimulatorKit via dlsym). type IndigoMessageForButtonFn = unsafe extern "C" fn(source: i32, action: i32, target: i32) -> *mut c_void; +/// `IndigoHIDMessageForMouseNSEvent(CGPoint*, CGPoint*, IndigoHIDTarget, +/// NSEventType, NSSize, IndigoHIDEdge)` +/// +/// On arm64 the integer and floating-point arguments are numbered +/// independently, so the pointers, target, event type and edge land in x0-x4 +/// while the `NSSize` occupies d0/d1. Declaring the size last therefore still +/// produces the correct register assignment. +/// +/// Apple's Simulator.app always passes `NSSize(1.0, 1.0)`, which makes the +/// ratio computation inside the function reduce to the point itself. type IndigoMessageForTouchFn = unsafe extern "C" fn( point0: *const objc2_core_foundation::CGPoint, point1: *const objc2_core_foundation::CGPoint, target: i32, event_type: i32, - something: Bool, + edge: u32, + size_width: f64, + size_height: f64, ) -> *mut c_void; type IndigoMessageForKeyboardFn = unsafe extern "C" fn(key_code: i32, action: i32) -> *mut c_void; @@ -33,6 +45,24 @@ pub enum TouchPhase { End, } +/// Screen edge a touch is treated as originating from. +/// +/// iOS only recognizes system gestures — most importantly swipe-up-to-home on +/// Face ID devices — when the touch is flagged with the edge it started from. +/// Without this a drag from the bottom is just an in-app drag. +/// +/// These are edges of the *raw framebuffer*, which never rotates, so callers +/// working in display space have to map through the current orientation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u32)] +pub enum TouchEdge { + None = 0, + Left = 1, + Top = 2, + Bottom = 3, + Right = 4, +} + /// Device orientation, using the GSEvent numbering. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[repr(u32)] @@ -232,6 +262,20 @@ impl SimulatorHID { /// `x` and `y` are 0..1 fractions of the screen, matching what the web UI /// already computes, so no point/pixel/scale conversion is involved. pub fn touch_normalized(&self, x: f64, y: f64, phase: TouchPhase) -> Result<()> { + self.touch_normalized_edge(x, y, phase, TouchEdge::None) + } + + /// As [`Self::touch_normalized`], but flagged as a system edge gesture. + /// + /// The same edge must be supplied for every event in the gesture, or iOS + /// will not recognize it. + pub fn touch_normalized_edge( + &self, + x: f64, + y: f64, + phase: TouchPhase, + edge: TouchEdge, + ) -> Result<()> { let x = x.clamp(0.0, 1.0); let y = y.clamp(0.0, 1.0); // Indigo has no distinct "move" phase; contact is maintained by @@ -241,7 +285,7 @@ impl SimulatorHID { TouchPhase::Begin | TouchPhase::Move => ButtonDirection::Down, TouchPhase::End => ButtonDirection::Up, }; - self.send_touch(x, y, direction) + self.send_touch_edge(x, y, direction, edge) } /// Rotate the device. @@ -357,6 +401,16 @@ impl SimulatorHID { /// Send a touch event at the given ratio coordinates. fn send_touch(&self, x_ratio: f64, y_ratio: f64, direction: ButtonDirection) -> Result<()> { + self.send_touch_edge(x_ratio, y_ratio, direction, TouchEdge::None) + } + + fn send_touch_edge( + &self, + x_ratio: f64, + y_ratio: f64, + direction: ButtonDirection, + edge: TouchEdge, + ) -> Result<()> { // First get a template message from IndigoHIDMessageForMouseNSEvent let point = objc2_core_foundation::CGPoint { x: x_ratio, @@ -368,8 +422,17 @@ impl SimulatorHID { ButtonDirection::Up => 2, }; - let template_msg = - unsafe { (self.msg_for_touch)(&point, std::ptr::null(), 0x32, event_type, Bool::NO) }; + let template_msg = unsafe { + (self.msg_for_touch)( + &point, + std::ptr::null(), + 0x32, + event_type, + edge as u32, + 1.0, + 1.0, + ) + }; if template_msg.is_null() { return Err(anyhow!("Failed to create template touch message")); diff --git a/packages/accessibility-serve/src/ax.rs b/packages/accessibility-serve/src/ax.rs index d66fc8c..5348754 100644 --- a/packages/accessibility-serve/src/ax.rs +++ b/packages/accessibility-serve/src/ax.rs @@ -12,8 +12,7 @@ //! //! # Coordinate spaces //! -//! Accessibility frames come back in macOS screen points, positioned wherever -//! the Simulator window happens to be. The browser knows nothing about that, +//! Accessibility frames come back in points relative to the Simulator window, //! so every rect is converted to a 0..1 fraction of the app's own bounds //! before it leaves this module: //! @@ -21,8 +20,17 @@ //! normalized = (ax_rect.origin - app_bounds.origin) / app_bounds.size //! ``` //! -//! That also makes the values independent of the display scale factor, which -//! is why no pixel/point conversion appears here. +//! That makes the values independent of the display scale factor, which is +//! why no pixel/point conversion appears here. +//! +//! Crucially, these are **logical** coordinates, already rotated by iOS: in +//! landscape the app reports its bounds as 874x402 rather than 402x874, so the +//! normalized rects are upright and need no further rotation to be drawn. +//! +//! This is the opposite of the HID input path, which is in *raw framebuffer* +//! space — the framebuffer never rotates, so pointer coordinates have to be +//! un-rotated before injection. The two spaces coincide in portrait, which is +//! exactly what makes the difference easy to miss. use anyhow::{Result, anyhow}; use serde::Serialize; @@ -76,6 +84,12 @@ pub struct AxSnapshot { pub app_name: Option, pub pid: Option, pub elements: Vec, + /// Whether the app reports itself wider than tall. + /// + /// Accessibility bounds are in logical space, so this is the one cheap + /// signal that reveals the device's real orientation. The framebuffer + /// cannot provide it: it never rotates. + pub is_landscape: bool, } pub enum AxCommand { @@ -172,6 +186,7 @@ fn snapshot( app_name: tree.app_name, pid: tree.pid, elements, + is_landscape: app_bounds.size.width > app_bounds.size.height, }) } diff --git a/packages/accessibility-serve/src/http.rs b/packages/accessibility-serve/src/http.rs index c06da72..b0aa4fa 100644 --- a/packages/accessibility-serve/src/http.rs +++ b/packages/accessibility-serve/src/http.rs @@ -13,8 +13,9 @@ use serde::{Deserialize, Serialize}; use accessibility_core::video::FrameKind; use crate::avcc; -use crate::input::InputCommand; +use crate::input::{InputCommand, Orientation}; use crate::session::SimSession; +use crate::settings::{Setting, SettingKey}; use crate::webrtc_stream::WebRtcEngine; const INDEX_HTML: &str = include_str!("../static/index.html"); @@ -32,6 +33,8 @@ pub fn router(state: AppState) -> Router { .route("/api/config", get(config)) .route("/api/ax/tree", get(ax_tree)) .route("/api/ax/hit", get(ax_hit)) + .route("/api/settings", get(settings).post(set_setting)) + .route("/api/orientation", post(set_orientation)) .route("/webrtc/offer", post(webrtc_offer)) .route("/ws/stream", get(stream_socket)) .route("/ws/input", get(input_socket)) @@ -45,8 +48,10 @@ async fn index() -> Html<&'static str> { #[derive(Serialize)] struct ConfigResponse { udid: String, + /// Raw framebuffer size. Constant regardless of orientation. width: u32, height: u32, + orientation: Orientation, default_transport: String, transports: Vec<&'static str>, home_indicator_band: f64, @@ -58,12 +63,57 @@ async fn config(State(state): State) -> Json { udid: device.udid, width: device.width, height: device.height, + orientation: device.orientation, default_transport: state.default_transport.clone(), transports: vec!["webrtc", "h264"], home_indicator_band: crate::input::HOME_INDICATOR_BAND, }) } +async fn settings(State(state): State) -> Json> { + // Each read shells out to simctl, so keep it off the async worker threads. + let session = Arc::clone(&state.session); + Json( + tokio::task::spawn_blocking(move || session.settings()) + .await + .unwrap_or_default(), + ) +} + +#[derive(Deserialize)] +struct SettingRequest { + key: SettingKey, + value: String, +} + +async fn set_setting( + State(state): State, + Json(request): Json, +) -> Response { + let session = Arc::clone(&state.session); + let result = + tokio::task::spawn_blocking(move || session.set_setting(request.key, &request.value)).await; + + match result { + Ok(Ok(value)) => Json(serde_json::json!({ "value": value })).into_response(), + Ok(Err(error)) => (StatusCode::BAD_REQUEST, error.to_string()).into_response(), + Err(error) => internal_error(anyhow::anyhow!(error)), + } +} + +#[derive(Deserialize)] +struct OrientationRequest { + orientation: Orientation, +} + +async fn set_orientation( + State(state): State, + Json(request): Json, +) -> Response { + state.session.set_orientation(request.orientation); + Json(serde_json::json!({ "orientation": request.orientation })).into_response() +} + /// Map an `anyhow` error onto a 500 with the message preserved. /// /// These are developer-facing diagnostics on a locally served tool, so the diff --git a/packages/accessibility-serve/src/input.rs b/packages/accessibility-serve/src/input.rs index 2d3fa2b..e49879d 100644 --- a/packages/accessibility-serve/src/input.rs +++ b/packages/accessibility-serve/src/input.rs @@ -1,18 +1,35 @@ //! Input forwarding from the browser to the simulator's HID subsystem. //! -//! All coordinates on this path are normalized 0..1 fractions of the display. -//! Keeping them normalized end to end avoids the points-vs-pixels-vs-scale -//! conversions that the accessibility side has to deal with. +//! All coordinates on this path are normalized 0..1 fractions of the *raw* +//! framebuffer. The browser un-rotates them before sending, so nothing here +//! needs to know about orientation, and no points/pixels/scale conversion is +//! involved anywhere. + +use std::sync::mpsc::{self, RecvTimeoutError, Sender}; +use std::time::{Duration, Instant}; use anyhow::Result; use serde::Deserialize; -use tokio::sync::mpsc; /// Touches below this fraction of the screen height are tagged as originating /// from the bottom edge, which is what makes swipe-up-to-home work. Without /// the edge hint iOS treats the drag as a normal in-app gesture. pub const HOME_INDICATOR_BAND: f64 = 0.93; +/// How far a wheel delta moves the virtual finger, as a multiple of the +/// delta itself. +const SCROLL_GAIN: f64 = 1.0; + +/// How close to an edge the virtual finger may get before the drag is +/// restarted from the middle of the screen. +const SCROLL_EDGE_MARGIN: f64 = 0.08; + +/// Quiet period after which a scroll gesture is lifted. +const SCROLL_IDLE: Duration = Duration::from_millis(100); + +/// How often the worker wakes to check for an expired scroll gesture. +const WORKER_TICK: Duration = Duration::from_millis(25); + #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] #[serde(rename_all = "lowercase")] pub enum TouchPhase { @@ -21,6 +38,22 @@ pub enum TouchPhase { End, } +/// Raw-framebuffer edge a touch is flagged as coming from. +/// +/// Required for iOS to recognize system gestures such as swipe-up-to-home. +/// The client decides this, because only it knows the current orientation and +/// the framebuffer never rotates. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Default)] +#[serde(rename_all = "lowercase")] +pub enum TouchEdge { + #[default] + None, + Left, + Top, + Bottom, + Right, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] #[serde(rename_all = "snake_case")] pub enum HardwareButton { @@ -31,6 +64,25 @@ pub enum HardwareButton { ApplePay, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Orientation { + Portrait, + PortraitUpsideDown, + LandscapeLeft, + LandscapeRight, +} + +impl Orientation { + /// Whether the display is wider than it is tall in this orientation. + pub fn is_landscape(self) -> bool { + matches!( + self, + Orientation::LandscapeLeft | Orientation::LandscapeRight + ) + } +} + /// A single input action to apply to the simulator. #[derive(Debug, Clone, Deserialize)] #[serde(tag = "type", rename_all = "snake_case")] @@ -39,6 +91,8 @@ pub enum InputCommand { phase: TouchPhase, x: f64, y: f64, + #[serde(default)] + edge: TouchEdge, }, Button { button: HardwareButton, @@ -47,55 +101,220 @@ pub enum InputCommand { Key { key_code: u32, }, + /// A wheel or trackpad delta, as a fraction of the display. + Scroll { + dx: f64, + dy: f64, + x: f64, + y: f64, + }, + Rotate { + orientation: Orientation, + }, +} + +/// Turns a stream of wheel deltas into a touch drag. +/// +/// iOS has no notion of a scroll wheel, so scrolling has to be a finger. The +/// awkward part is that a real finger runs out of screen: once the virtual +/// contact point nears an edge it is lifted and re-planted in the middle, so +/// an unbounded wheel can keep producing motion. +#[derive(Default)] +struct ScrollGesture { + active: bool, + x: f64, + y: f64, + last_event: Option, +} + +impl ScrollGesture { + fn near_edge(&self) -> bool { + self.x < SCROLL_EDGE_MARGIN + || self.x > 1.0 - SCROLL_EDGE_MARGIN + || self.y < SCROLL_EDGE_MARGIN + || self.y > 1.0 - SCROLL_EDGE_MARGIN + } } /// Start the HID worker thread and return its command channel. /// /// The worker owns the `SimulatorHID` because it is not `Sync`, and because -/// HID sends block on a dispatch queue round trip. +/// HID sends block on a dispatch queue round trip. It wakes periodically even +/// when idle so a scroll gesture can be lifted after the wheel stops. #[cfg(target_os = "macos")] -pub fn spawn_input_worker(udid: &str) -> Result> { - use accessibility_ios_sys::{HardwareButton as SysButton, SimulatorHID, TouchPhase as SysPhase}; +pub fn spawn_input_worker(udid: &str) -> Result> { + use accessibility_ios_sys::{ + HardwareButton as SysButton, Orientation as SysOrientation, SimulatorHID, + TouchEdge as SysEdge, TouchPhase as SysPhase, + }; let hid = SimulatorHID::for_device(Some(udid))?; - let (tx, mut rx) = mpsc::unbounded_channel::(); + let (tx, rx) = mpsc::channel::(); std::thread::Builder::new() .name("sim-input".into()) .spawn(move || { - while let Some(command) = rx.blocking_recv() { - let result = match command { - InputCommand::Touch { phase, x, y } => { - let phase = match phase { - TouchPhase::Begin => SysPhase::Begin, - TouchPhase::Move => SysPhase::Move, - TouchPhase::End => SysPhase::End, + let mut scroll = ScrollGesture::default(); + + loop { + match rx.recv_timeout(WORKER_TICK) { + Ok(command) => { + // Any direct touch interrupts an in-flight scroll, or + // the two gestures would fight over the same finger. + if !matches!(command, InputCommand::Scroll { .. }) && scroll.active { + let _ = hid.touch_normalized(scroll.x, scroll.y, SysPhase::End); + scroll = ScrollGesture::default(); + } + + let result = match command { + InputCommand::Touch { phase, x, y, edge } => { + let phase = match phase { + TouchPhase::Begin => SysPhase::Begin, + TouchPhase::Move => SysPhase::Move, + TouchPhase::End => SysPhase::End, + }; + let edge = match edge { + TouchEdge::None => SysEdge::None, + TouchEdge::Left => SysEdge::Left, + TouchEdge::Top => SysEdge::Top, + TouchEdge::Bottom => SysEdge::Bottom, + TouchEdge::Right => SysEdge::Right, + }; + hid.touch_normalized_edge(x, y, phase, edge) + } + InputCommand::Button { button } => { + let button = match button { + HardwareButton::Home => SysButton::Home, + HardwareButton::Lock => SysButton::Lock, + HardwareButton::Siri => SysButton::Siri, + HardwareButton::SideButton => SysButton::SideButton, + HardwareButton::ApplePay => SysButton::ApplePay, + }; + hid.press_button(button, 0) + } + InputCommand::Key { key_code } => hid.send_key(key_code), + InputCommand::Rotate { orientation } => { + hid.set_orientation(match orientation { + Orientation::Portrait => SysOrientation::Portrait, + Orientation::PortraitUpsideDown => { + SysOrientation::PortraitUpsideDown + } + Orientation::LandscapeLeft => SysOrientation::LandscapeLeft, + Orientation::LandscapeRight => SysOrientation::LandscapeRight, + }) + } + InputCommand::Scroll { dx, dy, x, y } => { + apply_scroll(&hid, &mut scroll, dx, dy, x, y) + } }; - hid.touch_normalized(x, y, phase) + + if let Err(error) = result { + tracing::warn!("input event failed: {error}"); + } } - InputCommand::Button { button } => { - let button = match button { - HardwareButton::Home => SysButton::Home, - HardwareButton::Lock => SysButton::Lock, - HardwareButton::Siri => SysButton::Siri, - HardwareButton::SideButton => SysButton::SideButton, - HardwareButton::ApplePay => SysButton::ApplePay, - }; - hid.press_button(button, 0) + Err(RecvTimeoutError::Timeout) => { + // Lift the finger once the wheel has gone quiet, + // otherwise the page keeps inertial-scrolling. + if scroll.active + && scroll + .last_event + .is_some_and(|at| at.elapsed() >= SCROLL_IDLE) + { + let _ = hid.touch_normalized(scroll.x, scroll.y, SysPhase::End); + scroll = ScrollGesture::default(); + } } - InputCommand::Key { key_code } => hid.send_key(key_code), - }; - - if let Err(error) = result { - tracing::warn!("input event failed: {error}"); + Err(RecvTimeoutError::Disconnected) => break, } } + + if scroll.active { + let _ = hid.touch_normalized(scroll.x, scroll.y, SysPhase::End); + } })?; Ok(tx) } +#[cfg(target_os = "macos")] +fn apply_scroll( + hid: &accessibility_ios_sys::SimulatorHID, + scroll: &mut ScrollGesture, + dx: f64, + dy: f64, + x: f64, + y: f64, +) -> Result<()> { + use accessibility_ios_sys::TouchPhase as SysPhase; + + if !scroll.active { + // Plant the finger under the pointer so the gesture lands on whatever + // the user is actually hovering. + scroll.x = x.clamp(0.0, 1.0); + scroll.y = y.clamp(0.0, 1.0); + scroll.active = true; + hid.touch_normalized(scroll.x, scroll.y, SysPhase::Begin)?; + } + + // Content follows the finger, so the finger moves opposite the wheel. + scroll.x = (scroll.x - dx * SCROLL_GAIN).clamp(0.0, 1.0); + scroll.y = (scroll.y - dy * SCROLL_GAIN).clamp(0.0, 1.0); + scroll.last_event = Some(Instant::now()); + + if scroll.near_edge() { + // Out of room: lift and re-plant in the middle so the next delta has + // somewhere to go. + hid.touch_normalized(scroll.x, scroll.y, SysPhase::End)?; + scroll.x = 0.5; + scroll.y = 0.5; + hid.touch_normalized(scroll.x, scroll.y, SysPhase::Begin)?; + return Ok(()); + } + + hid.touch_normalized(scroll.x, scroll.y, SysPhase::Move) +} + #[cfg(not(target_os = "macos"))] -pub fn spawn_input_worker(_udid: &str) -> Result> { +pub fn spawn_input_worker(_udid: &str) -> Result> { anyhow::bail!("Simulator input requires macOS") } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gesture_detects_each_edge() { + for (x, y) in [(0.01, 0.5), (0.99, 0.5), (0.5, 0.01), (0.5, 0.99)] { + let gesture = ScrollGesture { + active: true, + x, + y, + last_event: None, + }; + assert!( + gesture.near_edge(), + "expected ({x}, {y}) to be near an edge" + ); + } + } + + #[test] + fn gesture_center_is_not_near_edge() { + let gesture = ScrollGesture { + active: true, + x: 0.5, + y: 0.5, + last_event: None, + }; + assert!(!gesture.near_edge()); + } + + #[test] + fn landscape_classification() { + assert!(Orientation::LandscapeLeft.is_landscape()); + assert!(Orientation::LandscapeRight.is_landscape()); + assert!(!Orientation::Portrait.is_landscape()); + assert!(!Orientation::PortraitUpsideDown.is_landscape()); + } +} diff --git a/packages/accessibility-serve/src/lib.rs b/packages/accessibility-serve/src/lib.rs index d275a26..d954143 100644 --- a/packages/accessibility-serve/src/lib.rs +++ b/packages/accessibility-serve/src/lib.rs @@ -10,6 +10,7 @@ pub mod ax; pub mod http; pub mod input; pub mod session; +pub mod settings; pub mod webrtc_stream; use std::net::SocketAddr; @@ -81,6 +82,9 @@ impl Default for ServeConfig { pub async fn serve(config: ServeConfig) -> Result<()> { let session = SimSession::start(config.udid.as_deref(), config.video) .context("failed to start simulator capture")?; + // The framebuffer cannot reveal orientation, so ask accessibility once + // before serving; otherwise an already-rotated device renders sideways. + session.seed_orientation().await; let device = session.device_info(); let webrtc = Arc::new( diff --git a/packages/accessibility-serve/src/session.rs b/packages/accessibility-serve/src/session.rs index dfa101f..24fd1ad 100644 --- a/packages/accessibility-serve/src/session.rs +++ b/packages/accessibility-serve/src/session.rs @@ -17,7 +17,8 @@ use tokio::sync::{broadcast, mpsc, oneshot}; use accessibility_core::video::{EncodedFrame, FrameKind, VideoCapture, VideoConfig}; use crate::ax::{AxCommand, AxSnapshot, ElementDetail, spawn_ax_worker}; -use crate::input::{InputCommand, spawn_input_worker}; +use crate::input::{InputCommand, Orientation, spawn_input_worker}; +use crate::settings::{Setting, SettingKey}; /// How many encoded frames to buffer per subscriber. /// @@ -29,8 +30,11 @@ const FRAME_BUFFER: usize = 16; #[derive(Debug, Clone, serde::Serialize)] pub struct DeviceInfo { pub udid: String, + /// Raw framebuffer width. Does not change with orientation. pub width: u32, + /// Raw framebuffer height. Does not change with orientation. pub height: u32, + pub orientation: Orientation, } pub struct SimSession { @@ -40,8 +44,14 @@ pub struct SimSession { /// Most recent parameter set, replayed to clients that join mid-stream. latest_parameter_set: Arc>>, frames_encoded: Arc, - input: mpsc::UnboundedSender, + input: std::sync::mpsc::Sender, ax: mpsc::UnboundedSender, + /// Last orientation we asked for. + /// + /// The framebuffer is always portrait-native — rotating the device + /// rotates the *content* inside a fixed-size surface — so orientation + /// cannot be recovered from the video and has to be tracked here. + orientation: std::sync::Mutex, } impl SimSession { @@ -77,6 +87,7 @@ impl SimSession { frames_encoded, input, ax, + orientation: std::sync::Mutex::new(Orientation::Portrait), })) } @@ -106,12 +117,34 @@ impl SimSession { udid: self.device_udid.clone(), width: geometry.width, height: geometry.height, + orientation: self.orientation(), } } + pub fn orientation(&self) -> Orientation { + *self.orientation.lock().unwrap() + } + + /// Rotate the device and remember the new orientation. + pub fn set_orientation(&self, orientation: Orientation) { + *self.orientation.lock().unwrap() = orientation; + self.send_input(InputCommand::Rotate { orientation }); + } + + pub fn settings(&self) -> Vec { + crate::settings::read_all(&self.device_udid) + } + + pub fn set_setting(&self, key: SettingKey, value: &str) -> Result { + crate::settings::write(&self.device_udid, key, value) + } + /// Queue an input event. Fire-and-forget: pointer events must never block /// the socket reader. pub fn send_input(&self, command: InputCommand) { + if let InputCommand::Rotate { orientation } = command { + *self.orientation.lock().unwrap() = orientation; + } let _ = self.input.send(command); } @@ -120,7 +153,41 @@ impl SimSession { self.ax .send(AxCommand::Snapshot { reply: tx }) .map_err(|_| anyhow!("accessibility worker stopped"))?; - rx.await.map_err(|_| anyhow!("accessibility worker stopped"))? + let snapshot = rx + .await + .map_err(|_| anyhow!("accessibility worker stopped"))??; + self.reconcile_orientation(snapshot.is_landscape); + Ok(snapshot) + } + + /// Correct the tracked orientation against what the device reports. + /// + /// Orientation can change without us: the user can rotate from the + /// Simulator menu, an app can force an orientation, or the server can be + /// restarted while the device is already sideways. Accessibility bounds + /// are the only cheap signal, and they only reveal landscape vs portrait, + /// so a disagreement resolves to a sensible default of the right kind + /// rather than to an exact rotation. + fn reconcile_orientation(&self, is_landscape: bool) { + let mut orientation = self.orientation.lock().unwrap(); + if orientation.is_landscape() == is_landscape { + return; + } + *orientation = if is_landscape { + Orientation::LandscapeLeft + } else { + Orientation::Portrait + }; + } + + /// Best-effort orientation seed at startup. + /// + /// Without this the server would assume portrait and render a sideways + /// device whenever it attaches to an already-rotated simulator. + pub async fn seed_orientation(&self) { + if let Ok(snapshot) = self.ax_snapshot().await { + self.reconcile_orientation(snapshot.is_landscape); + } } pub async fn ax_hit_test(&self, x: f64, y: f64) -> Result> { diff --git a/packages/accessibility-serve/src/settings.rs b/packages/accessibility-serve/src/settings.rs new file mode 100644 index 0000000..e234aa4 --- /dev/null +++ b/packages/accessibility-serve/src/settings.rs @@ -0,0 +1,173 @@ +//! Simulator-wide UI settings. +//! +//! These go through `simctl ui`, which is the same mechanism Xcode's Devices +//! window uses. Only the three options simctl actually implements are exposed: +//! appearance, increase contrast, and content size. +//! +//! The Devices window also offers reduce-motion, colour filters, transparency +//! and VoiceOver, but simctl has no verb for those — they require a helper +//! binary spawned *inside* the simulator that drives the private +//! libAccessibility setters. That is a meaningfully larger piece of work and +//! is deliberately not attempted here. + +use anyhow::{Context, Result, anyhow}; +use serde::{Deserialize, Serialize}; + +/// Content size categories, smallest to largest. +/// +/// The five `accessibility-*` entries are the extended range that only appears +/// once a user opts into larger accessibility text. +pub const CONTENT_SIZES: &[&str] = &[ + "extra-small", + "small", + "medium", + "large", + "extra-large", + "extra-extra-large", + "extra-extra-extra-large", + "accessibility-medium", + "accessibility-large", + "accessibility-extra-large", + "accessibility-extra-extra-large", + "accessibility-extra-extra-extra-large", +]; + +pub const APPEARANCES: &[&str] = &["light", "dark"]; +pub const TOGGLE: &[&str] = &["enabled", "disabled"]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SettingKey { + Appearance, + IncreaseContrast, + ContentSize, +} + +impl SettingKey { + /// The `simctl ui` subcommand for this setting. + fn verb(self) -> &'static str { + match self { + SettingKey::Appearance => "appearance", + SettingKey::IncreaseContrast => "increase_contrast", + SettingKey::ContentSize => "content_size", + } + } + + pub fn allowed_values(self) -> &'static [&'static str] { + match self { + SettingKey::Appearance => APPEARANCES, + SettingKey::IncreaseContrast => TOGGLE, + SettingKey::ContentSize => CONTENT_SIZES, + } + } + + pub fn all() -> [SettingKey; 3] { + [ + SettingKey::Appearance, + SettingKey::IncreaseContrast, + SettingKey::ContentSize, + ] + } +} + +/// One setting and its current value, as reported by the simulator. +#[derive(Debug, Clone, Serialize)] +pub struct Setting { + pub key: SettingKey, + /// Current value, or `unsupported`/`unknown` if the runtime says so. + pub value: String, + pub allowed: &'static [&'static str], +} + +/// Read every supported setting from the device. +pub fn read_all(udid: &str) -> Vec { + SettingKey::all() + .into_iter() + .map(|key| Setting { + key, + // A failed read is reported as unknown rather than failing the + // whole request; one unsupported option should not blank the UI. + value: read(udid, key).unwrap_or_else(|_| "unknown".to_string()), + allowed: key.allowed_values(), + }) + .collect() +} + +pub fn read(udid: &str, key: SettingKey) -> Result { + let output = simctl(&["ui", udid, key.verb()])?; + Ok(output.trim().to_string()) +} + +pub fn write(udid: &str, key: SettingKey, value: &str) -> Result { + // `content_size` also accepts increment/decrement, which are not in the + // reported value set but are the ergonomic way to drive it from a UI. + let stepping = matches!(key, SettingKey::ContentSize) + && matches!(value, "increment" | "decrement"); + + if !stepping && !key.allowed_values().contains(&value) { + return Err(anyhow!( + "'{value}' is not valid for {:?}; expected one of {}", + key, + key.allowed_values().join(", ") + )); + } + + simctl(&["ui", udid, key.verb(), value])?; + read(udid, key) +} + +fn simctl(args: &[&str]) -> Result { + let output = std::process::Command::new("xcrun") + .arg("simctl") + .args(args) + .output() + .context("failed to run xcrun simctl")?; + + if !output.status.success() { + return Err(anyhow!( + "simctl {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + )); + } + Ok(String::from_utf8_lossy(&output.stdout).to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_values_outside_the_allowed_set() { + let error = write("no-such-device", SettingKey::Appearance, "chartreuse") + .expect_err("invalid appearance should be rejected"); + // Rejected locally, without ever shelling out to simctl. + assert!(error.to_string().contains("chartreuse")); + } + + #[test] + fn content_size_accepts_stepping_verbs() { + // These are not reported values, so they must be allowed explicitly. + assert!(!CONTENT_SIZES.contains(&"increment")); + for value in ["increment", "decrement"] { + let error = write("no-such-device", SettingKey::ContentSize, value) + .expect_err("no such device"); + assert!( + !error.to_string().contains("is not valid"), + "{value} should reach simctl rather than being rejected" + ); + } + } + + #[test] + fn every_key_has_values_and_a_distinct_verb() { + let mut verbs = Vec::new(); + for key in SettingKey::all() { + assert!(!key.allowed_values().is_empty()); + verbs.push(key.verb()); + } + verbs.sort_unstable(); + verbs.dedup(); + assert_eq!(verbs.len(), SettingKey::all().len()); + } +} diff --git a/packages/accessibility-serve/static/index.html b/packages/accessibility-serve/static/index.html index be42eb2..d4aca88 100644 --- a/packages/accessibility-serve/static/index.html +++ b/packages/accessibility-serve/static/index.html @@ -7,140 +7,208 @@
- - +
+ + +
- -
- - -
connecting
-
-