From a86fefaa79f463d82ed16216b7ac5d6c8a1f8cfb Mon Sep 17 00:00:00 2001 From: Jonathan Kelley Date: Thu, 6 Aug 2026 18:33:08 -0700 Subject: [PATCH] Expose reusable iOS Simulator sessions. Move simulator lifecycle orchestration into accessibility-core so embedded hosts can reuse hardware video, input, and inspection without the HTTP server. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <166158716+staging-devin-ai-integration[bot]@users.noreply.github.com> --- .devin/skills/ios-simulator-input/SKILL.md | 5 +- Cargo.lock | 2 +- packages/accessibility-core/Cargo.toml | 1 + .../src/platform/ios_simulator.rs | 17 +- .../src/platform/ios_simulator/ax.rs | 470 +++++++++++++++++ .../src/platform/ios_simulator/coverage.rs | 138 +++++ .../src/platform/ios_simulator/input.rs | 337 ++++++++++++ .../src/platform/ios_simulator/keymap.rs | 236 +++++++++ .../src/platform/ios_simulator/session.rs | 343 ++++++++++++ .../src/platform/ios_simulator/settings.rs | 173 +++++++ packages/accessibility-ios-sys/src/macos.rs | 4 +- .../accessibility-ios-sys/src/macos/common.rs | 101 +++- packages/accessibility-serve/Cargo.toml | 3 - packages/accessibility-serve/src/ax.rs | 488 +----------------- packages/accessibility-serve/src/coverage.rs | 139 +---- packages/accessibility-serve/src/input.rs | 346 +------------ packages/accessibility-serve/src/keymap.rs | 237 +-------- packages/accessibility-serve/src/lib.rs | 23 +- packages/accessibility-serve/src/session.rs | 354 +------------ packages/accessibility-serve/src/settings.rs | 174 +------ 20 files changed, 1836 insertions(+), 1755 deletions(-) create mode 100644 packages/accessibility-core/src/platform/ios_simulator/ax.rs create mode 100644 packages/accessibility-core/src/platform/ios_simulator/coverage.rs create mode 100644 packages/accessibility-core/src/platform/ios_simulator/input.rs create mode 100644 packages/accessibility-core/src/platform/ios_simulator/keymap.rs create mode 100644 packages/accessibility-core/src/platform/ios_simulator/session.rs create mode 100644 packages/accessibility-core/src/platform/ios_simulator/settings.rs diff --git a/.devin/skills/ios-simulator-input/SKILL.md b/.devin/skills/ios-simulator-input/SKILL.md index f490339..3a3bc67 100644 --- a/.devin/skills/ios-simulator-input/SKILL.md +++ b/.devin/skills/ios-simulator-input/SKILL.md @@ -44,8 +44,9 @@ Modifiers are ordinary key events held around the target key, so shifted characters are Shift-down, key-down, key-up, Shift-up. There is no shift flag on the Indigo message. -The character-to-key table lives in `accessibility-serve/src/keymap.rs` and is -US-ASCII only; unmappable characters fail the whole string rather than typing +The character-to-key table lives in +`accessibility-core/src/platform/ios_simulator/keymap.rs` and is US-ASCII only; +unmappable characters fail the whole string rather than typing a subtly wrong one. On Xcode 27 / CoreSimulator 1155.4+ an active `dtuhidd` silently disables diff --git a/Cargo.lock b/Cargo.lock index 928c3d0..8cb7562 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,6 +68,7 @@ dependencies = [ "serial_test", "slotmap", "tokio", + "tracing", "viuer", ] @@ -128,7 +129,6 @@ name = "accessibility-serve" version = "0.1.0" dependencies = [ "accessibility-core", - "accessibility-ios-sys", "anyhow", "axum", "bytes", diff --git a/packages/accessibility-core/Cargo.toml b/packages/accessibility-core/Cargo.toml index d46c75b..5773af1 100644 --- a/packages/accessibility-core/Cargo.toml +++ b/packages/accessibility-core/Cargo.toml @@ -33,6 +33,7 @@ viuer.workspace = true [target.'cfg(target_os = "macos")'.dependencies] accessibility-ios-sys.workspace = true accessibility-macos-sys.workspace = true +tracing.workspace = true [target.'cfg(target_os = "windows")'.dependencies] accessibility-windows-sys.workspace = true diff --git a/packages/accessibility-core/src/platform/ios_simulator.rs b/packages/accessibility-core/src/platform/ios_simulator.rs index 9232731..8a15bd7 100644 --- a/packages/accessibility-core/src/platform/ios_simulator.rs +++ b/packages/accessibility-core/src/platform/ios_simulator.rs @@ -23,7 +23,22 @@ use crate::video::{ Tuning, VideoCapture, VideoConfig, }; -pub use sys::{ButtonDirection, HardwareButton}; +pub mod ax; +pub mod coverage; +pub mod input; +pub mod keymap; +pub mod session; +pub mod settings; + +pub use ax::{AxCommand, AxSnapshot, Discovery, ElementDetail, NormalizedRect, spawn_ax_worker}; +pub use input::{ + HOME_INDICATOR_BAND, HardwareButton as InputHardwareButton, InputCommand, Orientation, + TouchEdge, TouchPhase, spawn_input_worker, +}; +pub use keymap::{KeyStroke, keystroke_for, keystrokes_for}; +pub use session::{DeviceInfo, SimSession, StatsReport, StreamStats}; +pub use settings::{Setting, SettingKey}; +pub use sys::{BootedSimulator, ButtonDirection, HardwareButton, booted_simulators}; /// Load all required private frameworks. pub fn load_frameworks() -> Result<()> { diff --git a/packages/accessibility-core/src/platform/ios_simulator/ax.rs b/packages/accessibility-core/src/platform/ios_simulator/ax.rs new file mode 100644 index 0000000..f3a62f1 --- /dev/null +++ b/packages/accessibility-core/src/platform/ios_simulator/ax.rs @@ -0,0 +1,470 @@ +//! Accessibility element inspection for the web UI. +//! +//! Two things are exposed: +//! +//! - A whole-tree snapshot, which the browser uses for instant hover feedback. +//! - A live hit test, which asks the simulator what is actually at a point. +//! +//! The snapshot can go stale between fetches and mis-picks overlapping or +//! transformed views; the hit test is authoritative but costs a round trip. +//! The UI uses the snapshot while the pointer is moving and confirms with the +//! hit test once it settles, which is why both exist. +//! +//! # Coordinate spaces +//! +//! Accessibility frames come back in points relative to the Simulator window, +//! so every rect is converted to a 0..1 fraction of the app's own bounds +//! before it leaves this module: +//! +//! ```text +//! normalized = (ax_rect.origin - app_bounds.origin) / app_bounds.size +//! ``` +//! +//! That makes the values independent of the display scale factor, which is +//! why no pixel/point conversion appears here. +//! +//! Crucially, these are **logical** coordinates, already rotated by iOS: in +//! landscape the app reports its bounds as 874x402 rather than 402x874, so the +//! normalized rects are upright and need no further rotation to be drawn. +//! +//! This is the opposite of the HID input path, which is in *raw framebuffer* +//! space — the framebuffer never rotates, so pointer coordinates have to be +//! un-rotated before injection. The two spaces coincide in portrait, which is +//! exactly what makes the difference easy to miss. + +use anyhow::{Result, anyhow}; +use serde::Serialize; +use tokio::sync::{mpsc, oneshot}; + +use crate::accessibility::{Element, Rect, TreeFilter}; + +use super::IOSSimulatorAccessibility; +use super::coverage::CoverageGrid; + +/// A rectangle in normalized display space (0..1 on both axes). +#[derive(Debug, Clone, Copy, Serialize)] +pub struct NormalizedRect { + pub x: f64, + pub y: f64, + pub width: f64, + pub height: f64, +} + +/// Fraction of the screen above which an element is treated as a backdrop +/// rather than something you can point at. +/// +/// Every app has an Application node and usually one or more full-bleed +/// container groups. Hit testing an empty region resolves to one of them, and +/// highlighting it paints a box over the entire device, which reads as "the +/// picker is broken" rather than "there is nothing here". +const BACKDROP_AREA: f64 = 0.9; + +impl NormalizedRect { + /// Whether this covers essentially the whole screen. + pub fn is_backdrop(&self) -> bool { + self.width * self.height >= BACKDROP_AREA + } + + fn from_screen(rect: &Rect, app_bounds: &Rect) -> Option { + if app_bounds.size.width <= 0.0 || app_bounds.size.height <= 0.0 { + return None; + } + Some(Self { + x: (rect.origin.x - app_bounds.origin.x) / app_bounds.size.width, + y: (rect.origin.y - app_bounds.origin.y) / app_bounds.size.height, + width: rect.size.width / app_bounds.size.width, + height: rect.size.height / app_bounds.size.height, + }) + } +} + +/// How an element was found. +/// +/// Worth surfacing: a swept element is a point sample with no parent, no +/// children and no document order, so consumers should not treat it as +/// equivalent to a node the tree walk returned. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Discovery { + /// Walked from the application root. + Recursive, + /// Found by hit testing a grid point the tree could not explain. + PointGrid, +} + +/// One inspectable element, flattened for the browser. +#[derive(Debug, Clone, Serialize)] +pub struct ElementDetail { + pub id: String, + pub role: String, + pub label: Option, + pub value: Option, + pub identifier: Option, + pub enabled: bool, + pub focused: bool, + pub actions: Vec, + pub bounds: Option, + /// Depth in the tree, used by the UI to prefer the innermost hit. + pub depth: u32, + /// A selector that would target this element from the CLI. + pub selector: String, + pub discovery: Discovery, +} + +#[derive(Debug, Clone, Serialize)] +pub struct AxSnapshot { + pub app_name: Option, + pub pid: Option, + pub elements: Vec, + /// Fraction of the display explained by the tree walk alone, before any + /// sweep. Low numbers on a busy screen mean out-of-process content. + pub coverage: f64, + /// Coverage after sweeping, when a scan was requested. + pub coverage_after_scan: Option, + /// How many points the sweep probed. `None` when no scan was requested. + pub probes: Option, + /// Whether the app reports itself wider than tall. + /// + /// Accessibility bounds are in logical space, so this is the one cheap + /// signal that reveals the device's real orientation. The framebuffer + /// cannot provide it: it never rotates. + pub is_landscape: bool, +} + +pub enum AxCommand { + Snapshot { + /// Also sweep the regions the tree walk cannot explain. + scan: bool, + reply: oneshot::Sender>, + }, + HitTest { + x: f64, + y: f64, + reply: oneshot::Sender>>, + }, +} + +/// Build the `accessibility-cli` selector that would resolve this element. +/// +/// Preferring the identifier keeps the selector stable across copy changes, +/// which is the whole point of exposing it in the inspector. +fn selector_for(element: &Element, role: &str) -> String { + if let Some(identifier) = element.identifier.as_deref().filter(|s| !s.is_empty()) { + return format!("#{identifier}"); + } + if let Some(label) = element.title.as_deref().filter(|s| !s.is_empty()) { + return format!("{role}[label=\"{}\"]", label.replace('"', "\\\"")); + } + role.to_string() +} + +fn to_detail( + element: &Element, + app_bounds: &Rect, + depth: u32, + discovery: Discovery, +) -> ElementDetail { + let role = format!("{:?}", element.role); + ElementDetail { + discovery, + id: element.id.to_string(), + selector: selector_for(element, &role), + role, + label: element.title.clone().filter(|s| !s.is_empty()), + value: element.value.clone().filter(|s| !s.is_empty()), + identifier: element.identifier.clone().filter(|s| !s.is_empty()), + enabled: element.enabled, + focused: element.focused, + actions: element.actions.clone(), + bounds: element + .bounds + .as_ref() + .and_then(|bounds| NormalizedRect::from_screen(bounds, app_bounds)), + depth, + } +} + +fn flatten(element: &Element, app_bounds: &Rect, depth: u32, out: &mut Vec) { + out.push(to_detail(element, app_bounds, depth, Discovery::Recursive)); + for child in &element.children { + flatten(child, app_bounds, depth + 1, out); + } +} + +/// Start the accessibility worker thread and return its command channel. +pub fn spawn_ax_worker(udid: &str) -> Result> { + let mut reader = IOSSimulatorAccessibility::new(Some(udid))?; + let (tx, mut rx) = mpsc::unbounded_channel::(); + + std::thread::Builder::new() + .name("sim-ax".into()) + .spawn(move || { + while let Some(command) = rx.blocking_recv() { + match command { + AxCommand::Snapshot { scan, reply } => { + let _ = reply.send(snapshot(&mut reader, scan)); + } + AxCommand::HitTest { x, y, reply } => { + let _ = reply.send(hit_test(&mut reader, x, y)); + } + } + } + })?; + + Ok(tx) +} + +fn snapshot(reader: &mut IOSSimulatorAccessibility, scan: bool) -> Result { + let tree = reader.get_tree(&TreeFilter::default())?; + // `get_screen_bounds` is only populated once a tree has been read, so it + // has to be queried after the fetch above. + let app_bounds = reader.get_screen_bounds()?; + + let mut elements: Vec = Vec::with_capacity(tree.element_count); + flatten(&tree.root, &app_bounds, 0, &mut elements); + + // Drop backdrops and anything with no usable geometry. Both are real parts + // of the tree but neither can be pointed at, and leaving them in makes the + // client's containment search pick them constantly. + elements.retain(|element| { + element.bounds.is_some_and(|bounds| { + !bounds.is_backdrop() && bounds.width > 0.0 && bounds.height > 0.0 + }) + }); + + // Everything the tree walk explained, so the sweep can skip it. + let mut coverage = CoverageGrid::new(); + for element in &elements { + if let Some(bounds) = element.bounds { + coverage.mark(&bounds); + } + } + let coverage_before = coverage.ratio(); + + let mut probes = None; + let mut coverage_after_scan = None; + if scan { + let swept = sweep(reader, &app_bounds, &mut coverage, &elements)?; + probes = Some(swept.probes); + elements.extend(swept.elements); + coverage_after_scan = Some(coverage.ratio()); + } + + Ok(AxSnapshot { + app_name: tree.app_name, + pid: tree.pid, + elements, + coverage: coverage_before, + coverage_after_scan, + probes, + is_landscape: app_bounds.size.width > app_bounds.size.height, + }) +} + +fn hit_test( + reader: &mut IOSSimulatorAccessibility, + x: f64, + y: f64, +) -> Result> { + // The hit test wants macOS screen points, so the browser's normalized + // coordinates are mapped back through the app's bounds. + let app_bounds = reader + .get_screen_bounds() + .map_err(|_| anyhow!("no accessibility snapshot yet; fetch the tree first"))?; + + let screen_x = app_bounds.origin.x + x * app_bounds.size.width; + let screen_y = app_bounds.origin.y + y * app_bounds.size.height; + + let Some(element) = reader.element_at_point(screen_x, screen_y)? else { + return Ok(None); + }; + let detail = to_detail(&element, &app_bounds, 0, Discovery::Recursive); + + // Nothing pointable here. Reporting the backdrop would highlight the whole + // device; reporting nothing lets the caller leave the previous selection + // or clear it. + if detail.bounds.is_none_or(|bounds| bounds.is_backdrop()) { + return Ok(None); + } + Ok(Some(detail)) +} + +/// Spacing between sweep probes, in device points. +/// +/// idb uses 50; 40 is a little denser for phone-sized screens, where rows are +/// around 50 points tall and a coarser grid can step straight over one. Every +/// probe is a hit test, so this trades roughly linearly against scan time. +const SWEEP_STEP_POINTS: f64 = 40.0; + +/// Upper bound on probes, so a scan cannot run away on a large display. +const SWEEP_MAX_PROBES: usize = 400; + +struct SweepResult { + elements: Vec, + probes: usize, +} + +/// Hit test the regions the tree walk could not explain. +/// +/// This is the only way to reach content in another process — `WKWebView` and +/// Safari above all — because such content is individually addressable by +/// point but has no traversable hierarchy. The result is a flat set of point +/// samples, which is why each is tagged [`Discovery::PointGrid`]. +fn sweep( + reader: &mut IOSSimulatorAccessibility, + app_bounds: &Rect, + coverage: &mut CoverageGrid, + known: &[ElementDetail], +) -> Result { + use std::collections::HashSet; + + // Elements the tree already reported, keyed by position, so a probe that + // lands on one does not report it twice under a different provenance. + let mut seen: HashSet = known + .iter() + .filter_map(|element| element.bounds.map(frame_key)) + .collect(); + + let mut elements = Vec::new(); + let mut probes = 0usize; + + let columns = (app_bounds.size.width / SWEEP_STEP_POINTS).floor().max(1.0) as usize; + let rows = (app_bounds.size.height / SWEEP_STEP_POINTS) + .floor() + .max(1.0) as usize; + + for row in 0..rows { + for column in 0..columns { + if probes >= SWEEP_MAX_PROBES { + return Ok(SweepResult { elements, probes }); + } + + // Probe cell centres so a point never lands exactly on a boundary. + let x = (column as f64 + 0.5) / columns as f64; + let y = (row as f64 + 0.5) / rows as f64; + + // Already explained by the tree; nothing to learn here. + if coverage.is_filled(x, y) { + continue; + } + probes += 1; + + let screen_x = app_bounds.origin.x + x * app_bounds.size.width; + let screen_y = app_bounds.origin.y + y * app_bounds.size.height; + let Ok(Some(element)) = reader.element_at_point(screen_x, screen_y) else { + continue; + }; + + let detail = to_detail(&element, app_bounds, 0, Discovery::PointGrid); + let Some(bounds) = detail.bounds else { + continue; + }; + if bounds.is_backdrop() || bounds.width <= 0.0 || bounds.height <= 0.0 { + continue; + } + // A 40 point grid lands on a large element many times over. + if !seen.insert(frame_key(bounds)) { + // Still mark it: further probes inside it are wasted. + coverage.mark(&bounds); + continue; + } + + coverage.mark(&bounds); + elements.push(detail); + } + } + + Ok(SweepResult { elements, probes }) +} + +/// Position key for deduplication, rounded so float noise does not defeat it. +fn frame_key(bounds: NormalizedRect) -> String { + format!( + "{:.4},{:.4},{:.4},{:.4}", + bounds.x, bounds.y, bounds.width, bounds.height + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::accessibility::{Point, Size}; + + fn rect(x: f64, y: f64, w: f64, h: f64) -> Rect { + Rect::new(Point::new(x, y), Size::new(w, h)) + } + + #[test] + fn normalizes_against_app_bounds_not_the_screen() { + // Landscape: the app reports itself 874x402, and accessibility frames + // are already in that logical space. + let app = rect(0.0, 0.0, 874.0, 402.0); + let button = rect(754.5, 82.0, 27.0, 44.0); + let normalized = NormalizedRect::from_screen(&button, &app).expect("normalizes"); + assert!((normalized.x - 0.863).abs() < 0.001); + assert!((normalized.y - 0.204).abs() < 0.001); + assert!((normalized.width - 0.031).abs() < 0.001); + } + + #[test] + fn offsets_by_the_app_origin() { + let app = rect(100.0, 50.0, 400.0, 800.0); + let element = rect(300.0, 450.0, 200.0, 400.0); + let normalized = NormalizedRect::from_screen(&element, &app).expect("normalizes"); + assert!((normalized.x - 0.5).abs() < 1e-9); + assert!((normalized.y - 0.5).abs() < 1e-9); + } + + #[test] + fn degenerate_app_bounds_do_not_divide_by_zero() { + let app = rect(0.0, 0.0, 0.0, 0.0); + assert!(NormalizedRect::from_screen(&rect(1.0, 1.0, 2.0, 2.0), &app).is_none()); + } + + #[test] + fn full_screen_containers_are_backdrops() { + // Every app has these; highlighting one paints over the whole device. + assert!( + NormalizedRect { + x: 0.0, + y: 0.0, + width: 1.0, + height: 1.0 + } + .is_backdrop() + ); + assert!( + NormalizedRect { + x: 0.0, + y: 0.0, + width: 0.98, + height: 0.96 + } + .is_backdrop() + ); + } + + #[test] + fn ordinary_controls_are_not_backdrops() { + // A full-width settings row is large but perfectly pointable. + assert!( + !NormalizedRect { + x: 0.05, + y: 0.45, + width: 0.90, + height: 0.06 + } + .is_backdrop() + ); + // So is a tall sidebar. + assert!( + !NormalizedRect { + x: 0.0, + y: 0.0, + width: 0.25, + height: 1.0 + } + .is_backdrop() + ); + } +} diff --git a/packages/accessibility-core/src/platform/ios_simulator/coverage.rs b/packages/accessibility-core/src/platform/ios_simulator/coverage.rs new file mode 100644 index 0000000..acf4215 --- /dev/null +++ b/packages/accessibility-core/src/platform/ios_simulator/coverage.rs @@ -0,0 +1,138 @@ +//! Tracking which parts of the screen the accessibility tree explains. +//! +//! Two jobs. It measures how much of the display is accounted for by elements +//! the tree walk found, which is a direct health signal — a screen full of +//! content reporting 5% coverage means most of the UI is invisible to every +//! tree-based tool. And it tells the point-grid sweep where not to bother +//! probing, so discovery only pays for the unexplained regions. + +use super::ax::NormalizedRect; + +/// Cells along each axis. +/// +/// 32x64 over a normalized display is about 12x13 points per cell on a phone — +/// fine enough that a table row does not mark a neighbouring row as covered, +/// coarse enough to stay a cheap bitmap. +const COLUMNS: usize = 32; +const ROWS: usize = 64; + +pub struct CoverageGrid { + filled: [bool; COLUMNS * ROWS], +} + +impl Default for CoverageGrid { + fn default() -> Self { + Self::new() + } +} + +impl CoverageGrid { + pub fn new() -> Self { + Self { + filled: [false; COLUMNS * ROWS], + } + } + + /// Mark every cell a normalized rect touches. + pub fn mark(&mut self, rect: &NormalizedRect) { + // Backdrops cover everything and would leave nothing to discover. + if rect.is_backdrop() { + return; + } + let (min_col, max_col) = span(rect.x, rect.width, COLUMNS); + let (min_row, max_row) = span(rect.y, rect.height, ROWS); + for row in min_row..=max_row { + for column in min_col..=max_col { + self.filled[row * COLUMNS + column] = true; + } + } + } + + pub fn is_filled(&self, x: f64, y: f64) -> bool { + let column = index(x, COLUMNS); + let row = index(y, ROWS); + self.filled[row * COLUMNS + column] + } + + /// Fraction of the display explained so far, 0.0 to 1.0. + pub fn ratio(&self) -> f64 { + let filled = self.filled.iter().filter(|cell| **cell).count(); + filled as f64 / self.filled.len() as f64 + } +} + +fn index(value: f64, cells: usize) -> usize { + ((value.clamp(0.0, 1.0) * cells as f64) as usize).min(cells - 1) +} + +/// Inclusive cell range covered by an interval, clamped to the display. +fn span(start: f64, length: f64, cells: usize) -> (usize, usize) { + let low = index(start, cells); + let high = index(start + length.max(0.0), cells); + (low.min(high), high.max(low)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rect(x: f64, y: f64, width: f64, height: f64) -> NormalizedRect { + NormalizedRect { + x, + y, + width, + height, + } + } + + #[test] + fn empty_grid_explains_nothing() { + assert_eq!(CoverageGrid::new().ratio(), 0.0); + assert!(!CoverageGrid::new().is_filled(0.5, 0.5)); + } + + #[test] + fn marking_covers_the_rect_and_not_its_surroundings() { + let mut grid = CoverageGrid::new(); + grid.mark(&rect(0.4, 0.4, 0.2, 0.2)); + assert!(grid.is_filled(0.5, 0.5)); + assert!(!grid.is_filled(0.1, 0.1)); + assert!(!grid.is_filled(0.9, 0.9)); + } + + #[test] + fn backdrops_are_ignored() { + // Otherwise the Application node alone would report full coverage and + // the sweep would never probe anywhere. + let mut grid = CoverageGrid::new(); + grid.mark(&rect(0.0, 0.0, 1.0, 1.0)); + assert_eq!(grid.ratio(), 0.0); + } + + #[test] + fn ratio_grows_with_marked_area() { + let mut grid = CoverageGrid::new(); + let before = grid.ratio(); + grid.mark(&rect(0.0, 0.0, 0.5, 0.5)); + let after = grid.ratio(); + assert!(after > before); + // A quarter of the display, within one cell of rounding on each axis. + assert!((after - 0.25).abs() < 0.05, "ratio was {after}"); + } + + #[test] + fn out_of_range_rects_do_not_panic() { + let mut grid = CoverageGrid::new(); + // Rows can extend past the bottom of the screen; seen in real trees. + grid.mark(&rect(0.9, 0.95, 0.5, 0.5)); + grid.mark(&rect(-0.2, -0.2, 0.1, 0.1)); + assert!(grid.is_filled(0.99, 0.99)); + } + + #[test] + fn thin_rects_still_mark_a_cell() { + let mut grid = CoverageGrid::new(); + grid.mark(&rect(0.5, 0.5, 0.0, 0.0)); + assert!(grid.is_filled(0.5, 0.5)); + } +} diff --git a/packages/accessibility-core/src/platform/ios_simulator/input.rs b/packages/accessibility-core/src/platform/ios_simulator/input.rs new file mode 100644 index 0000000..7ea8a68 --- /dev/null +++ b/packages/accessibility-core/src/platform/ios_simulator/input.rs @@ -0,0 +1,337 @@ +//! Input forwarding from the browser to the simulator's HID subsystem. +//! +//! All coordinates on this path are normalized 0..1 fractions of the *raw* +//! framebuffer. The browser un-rotates them before sending, so nothing here +//! needs to know about orientation, and no points/pixels/scale conversion is +//! involved anywhere. + +use std::sync::mpsc::{self, RecvTimeoutError, Sender}; +use std::time::{Duration, Instant}; + +use anyhow::Result; +use serde::Deserialize; + +/// Touches below this fraction of the screen height are tagged as originating +/// from the bottom edge, which is what makes swipe-up-to-home work. Without +/// the edge hint iOS treats the drag as a normal in-app gesture. +pub const HOME_INDICATOR_BAND: f64 = 0.93; + +/// How far a wheel delta moves the virtual finger, as a multiple of the +/// delta itself. +const SCROLL_GAIN: f64 = 1.0; + +/// How close to an edge the virtual finger may get before the drag is +/// restarted from the middle of the screen. +const SCROLL_EDGE_MARGIN: f64 = 0.08; + +/// Quiet period after which a scroll gesture is lifted. +const SCROLL_IDLE: Duration = Duration::from_millis(100); + +/// How often the worker wakes to check for an expired scroll gesture. +const WORKER_TICK: Duration = Duration::from_millis(25); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum TouchPhase { + Begin, + Move, + End, +} + +/// Raw-framebuffer edge a touch is flagged as coming from. +/// +/// Required for iOS to recognize system gestures such as swipe-up-to-home. +/// The client decides this, because only it knows the current orientation and +/// the framebuffer never rotates. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Default)] +#[serde(rename_all = "lowercase")] +pub enum TouchEdge { + #[default] + None, + Left, + Top, + Bottom, + Right, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum HardwareButton { + Home, + Lock, + Siri, + SideButton, + ApplePay, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Orientation { + Portrait, + PortraitUpsideDown, + LandscapeLeft, + LandscapeRight, +} + +impl Orientation { + /// Whether the display is wider than it is tall in this orientation. + pub fn is_landscape(self) -> bool { + matches!( + self, + Orientation::LandscapeLeft | Orientation::LandscapeRight + ) + } +} + +/// A single input action to apply to the simulator. +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum InputCommand { + Touch { + phase: TouchPhase, + x: f64, + y: f64, + #[serde(default)] + edge: TouchEdge, + }, + Button { + button: HardwareButton, + }, + /// A single key press by USB HID usage code, with optional held modifiers. + /// + /// Used for navigation and shortcuts; text goes through [`InputCommand::Text`] + /// so the character-to-key table lives in one place. + Key { + key_code: u32, + #[serde(default)] + modifiers: Vec, + }, + /// Type a string, expanded server-side into key presses. + Text { + text: String, + }, + /// A wheel or trackpad delta, as a fraction of the display. + Scroll { + dx: f64, + dy: f64, + x: f64, + y: f64, + }, + Rotate { + orientation: Orientation, + }, +} + +/// Turns a stream of wheel deltas into a touch drag. +/// +/// iOS has no notion of a scroll wheel, so scrolling has to be a finger. The +/// awkward part is that a real finger runs out of screen: once the virtual +/// contact point nears an edge it is lifted and re-planted in the middle, so +/// an unbounded wheel can keep producing motion. +#[derive(Default)] +struct ScrollGesture { + active: bool, + x: f64, + y: f64, + last_event: Option, +} + +impl ScrollGesture { + fn near_edge(&self) -> bool { + self.x < SCROLL_EDGE_MARGIN + || self.x > 1.0 - SCROLL_EDGE_MARGIN + || self.y < SCROLL_EDGE_MARGIN + || self.y > 1.0 - SCROLL_EDGE_MARGIN + } +} + +/// Start the HID worker thread and return its command channel. +/// +/// The worker owns the `SimulatorHID` because it is not `Sync`, and because +/// HID sends block on a dispatch queue round trip. It wakes periodically even +/// when idle so a scroll gesture can be lifted after the wheel stops. +pub fn spawn_input_worker(udid: &str) -> Result> { + use accessibility_ios_sys::{ + HardwareButton as SysButton, Orientation as SysOrientation, SimulatorHID, + TouchEdge as SysEdge, TouchPhase as SysPhase, + }; + + let hid = SimulatorHID::for_device(Some(udid))?; + let (tx, rx) = mpsc::channel::(); + + std::thread::Builder::new() + .name("sim-input".into()) + .spawn(move || { + let mut scroll = ScrollGesture::default(); + + loop { + match rx.recv_timeout(WORKER_TICK) { + Ok(command) => { + // Any direct touch interrupts an in-flight scroll, or + // the two gestures would fight over the same finger. + if !matches!(command, InputCommand::Scroll { .. }) && scroll.active { + let _ = hid.touch_normalized(scroll.x, scroll.y, SysPhase::End); + scroll = ScrollGesture::default(); + } + + let result = match command { + InputCommand::Touch { phase, x, y, edge } => { + let phase = match phase { + TouchPhase::Begin => SysPhase::Begin, + TouchPhase::Move => SysPhase::Move, + TouchPhase::End => SysPhase::End, + }; + let edge = match edge { + TouchEdge::None => SysEdge::None, + TouchEdge::Left => SysEdge::Left, + TouchEdge::Top => SysEdge::Top, + TouchEdge::Bottom => SysEdge::Bottom, + TouchEdge::Right => SysEdge::Right, + }; + hid.touch_normalized_edge(x, y, phase, edge) + } + InputCommand::Button { button } => { + let button = match button { + HardwareButton::Home => SysButton::Home, + HardwareButton::Lock => SysButton::Lock, + HardwareButton::Siri => SysButton::Siri, + HardwareButton::SideButton => SysButton::SideButton, + HardwareButton::ApplePay => SysButton::ApplePay, + }; + hid.press_button(button, 0) + } + InputCommand::Key { + key_code, + ref modifiers, + } => hid.send_key_with_modifiers(key_code, modifiers), + InputCommand::Text { ref text } => type_text(&hid, text), + InputCommand::Rotate { orientation } => { + hid.set_orientation(match orientation { + Orientation::Portrait => SysOrientation::Portrait, + Orientation::PortraitUpsideDown => { + SysOrientation::PortraitUpsideDown + } + Orientation::LandscapeLeft => SysOrientation::LandscapeLeft, + Orientation::LandscapeRight => SysOrientation::LandscapeRight, + }) + } + InputCommand::Scroll { dx, dy, x, y } => { + apply_scroll(&hid, &mut scroll, dx, dy, x, y) + } + }; + + if let Err(error) = result { + tracing::warn!("input event failed: {error}"); + } + } + Err(RecvTimeoutError::Timeout) => { + // Lift the finger once the wheel has gone quiet, + // otherwise the page keeps inertial-scrolling. + if scroll.active + && scroll + .last_event + .is_some_and(|at| at.elapsed() >= SCROLL_IDLE) + { + let _ = hid.touch_normalized(scroll.x, scroll.y, SysPhase::End); + scroll = ScrollGesture::default(); + } + } + Err(RecvTimeoutError::Disconnected) => break, + } + } + + if scroll.active { + let _ = hid.touch_normalized(scroll.x, scroll.y, SysPhase::End); + } + })?; + + Ok(tx) +} + +/// Expand text into key presses and send them. +/// +/// Rejects the whole string if any character is untypeable, so a partial or +/// subtly wrong string is never entered. +fn type_text(hid: &accessibility_ios_sys::SimulatorHID, text: &str) -> Result<()> { + for stroke in super::keymap::keystrokes_for(text)? { + hid.send_key_with_modifiers(stroke.usage, &stroke.modifiers())?; + } + Ok(()) +} + +fn apply_scroll( + hid: &accessibility_ios_sys::SimulatorHID, + scroll: &mut ScrollGesture, + dx: f64, + dy: f64, + x: f64, + y: f64, +) -> Result<()> { + use accessibility_ios_sys::TouchPhase as SysPhase; + + if !scroll.active { + // Plant the finger under the pointer so the gesture lands on whatever + // the user is actually hovering. + scroll.x = x.clamp(0.0, 1.0); + scroll.y = y.clamp(0.0, 1.0); + scroll.active = true; + hid.touch_normalized(scroll.x, scroll.y, SysPhase::Begin)?; + } + + // Content follows the finger, so the finger moves opposite the wheel. + scroll.x = (scroll.x - dx * SCROLL_GAIN).clamp(0.0, 1.0); + scroll.y = (scroll.y - dy * SCROLL_GAIN).clamp(0.0, 1.0); + scroll.last_event = Some(Instant::now()); + + if scroll.near_edge() { + // Out of room: lift and re-plant in the middle so the next delta has + // somewhere to go. + hid.touch_normalized(scroll.x, scroll.y, SysPhase::End)?; + scroll.x = 0.5; + scroll.y = 0.5; + hid.touch_normalized(scroll.x, scroll.y, SysPhase::Begin)?; + return Ok(()); + } + + hid.touch_normalized(scroll.x, scroll.y, SysPhase::Move) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gesture_detects_each_edge() { + for (x, y) in [(0.01, 0.5), (0.99, 0.5), (0.5, 0.01), (0.5, 0.99)] { + let gesture = ScrollGesture { + active: true, + x, + y, + last_event: None, + }; + assert!( + gesture.near_edge(), + "expected ({x}, {y}) to be near an edge" + ); + } + } + + #[test] + fn gesture_center_is_not_near_edge() { + let gesture = ScrollGesture { + active: true, + x: 0.5, + y: 0.5, + last_event: None, + }; + assert!(!gesture.near_edge()); + } + + #[test] + fn landscape_classification() { + assert!(Orientation::LandscapeLeft.is_landscape()); + assert!(Orientation::LandscapeRight.is_landscape()); + assert!(!Orientation::Portrait.is_landscape()); + assert!(!Orientation::PortraitUpsideDown.is_landscape()); + } +} diff --git a/packages/accessibility-core/src/platform/ios_simulator/keymap.rs b/packages/accessibility-core/src/platform/ios_simulator/keymap.rs new file mode 100644 index 0000000..f3d7f6a --- /dev/null +++ b/packages/accessibility-core/src/platform/ios_simulator/keymap.rs @@ -0,0 +1,236 @@ +//! Mapping text to simulator key presses. +//! +//! The simulator's keyboard takes USB HID usage codes and has no notion of a +//! shift flag, so a capital letter or a shifted symbol is a *sequence*: hold +//! Left Shift, press the base key, release both. This module owns that +//! translation so the browser does not have to duplicate the table. +//! +//! Scope is deliberately US ASCII. There is no layout awareness and no +//! Unicode: a character outside the table is reported as an error rather than +//! silently dropped or turned into the wrong key, which is how the previous +//! implementation lost every `@` and quietly lowercased every capital. + +use anyhow::{Result, anyhow}; + +/// USB HID keyboard usage codes, re-exported for the input layer. +pub mod usage { + pub const RETURN: u32 = 40; + pub const ESCAPE: u32 = 41; + pub const BACKSPACE: u32 = 42; + pub const TAB: u32 = 43; + pub const RIGHT_ARROW: u32 = 79; + pub const LEFT_ARROW: u32 = 80; + pub const DOWN_ARROW: u32 = 81; + pub const UP_ARROW: u32 = 82; + pub const LEFT_CONTROL: u32 = 224; + pub const LEFT_SHIFT: u32 = 225; + pub const LEFT_ALT: u32 = 226; + pub const LEFT_GUI: u32 = 227; +} + +/// A single key press: the usage code and whether Shift is held for it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeyStroke { + pub usage: u32, + pub shift: bool, +} + +impl KeyStroke { + const fn plain(usage: u32) -> Self { + Self { + usage, + shift: false, + } + } + + const fn shifted(usage: u32) -> Self { + Self { usage, shift: true } + } + + /// The modifier usages to hold while pressing this key. + pub fn modifiers(&self) -> Vec { + if self.shift { + vec![usage::LEFT_SHIFT] + } else { + Vec::new() + } + } +} + +/// Symbols reachable without Shift, in USB HID usage order. +const UNSHIFTED_SYMBOLS: &[(char, u32)] = &[ + ('-', 45), + ('=', 46), + ('[', 47), + (']', 48), + ('\\', 49), + (';', 51), + ('\'', 52), + ('`', 53), + (',', 54), + ('.', 55), + ('/', 56), +]; + +/// Symbols produced by holding Shift over another key's usage. +const SHIFTED_SYMBOLS: &[(char, u32)] = &[ + ('!', 30), + ('@', 31), + ('#', 32), + ('$', 33), + ('%', 34), + ('^', 35), + ('&', 36), + ('*', 37), + ('(', 38), + (')', 39), + ('_', 45), + ('+', 46), + ('{', 47), + ('}', 48), + ('|', 49), + (':', 51), + ('"', 52), + ('~', 53), + ('<', 54), + ('>', 55), + ('?', 56), +]; + +/// The key press that produces `character`, if one exists on a US keyboard. +pub fn keystroke_for(character: char) -> Option { + match character { + // Letters share a usage; case is the Shift modifier. + 'a'..='z' => Some(KeyStroke::plain(character as u32 - 'a' as u32 + 4)), + 'A'..='Z' => Some(KeyStroke::shifted(character as u32 - 'A' as u32 + 4)), + // Digits are not contiguous with zero: 1-9 are 30-38 and 0 is 39. + '1'..='9' => Some(KeyStroke::plain(character as u32 - '1' as u32 + 30)), + '0' => Some(KeyStroke::plain(39)), + '\n' | '\r' => Some(KeyStroke::plain(usage::RETURN)), + '\t' => Some(KeyStroke::plain(usage::TAB)), + ' ' => Some(KeyStroke::plain(44)), + _ => UNSHIFTED_SYMBOLS + .iter() + .find(|(c, _)| *c == character) + .map(|(_, usage)| KeyStroke::plain(*usage)) + .or_else(|| { + SHIFTED_SYMBOLS + .iter() + .find(|(c, _)| *c == character) + .map(|(_, usage)| KeyStroke::shifted(*usage)) + }), + } +} + +/// Translate a string into key presses. +/// +/// Fails on the first unmappable character rather than typing a partial or +/// wrong string. +pub fn keystrokes_for(text: &str) -> Result> { + text.chars() + .map(|character| { + keystroke_for(character).ok_or_else(|| { + anyhow!( + "cannot type {character:?}: only US-ASCII characters are supported \ + (no unicode or emoji)" + ) + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn type_text(text: &str) -> Vec<(u32, bool)> { + keystrokes_for(text) + .expect("typeable") + .into_iter() + .map(|k| (k.usage, k.shift)) + .collect() + } + + #[test] + fn letters_use_usb_hid_usages_not_hitoolbox() { + // The bug this module exists to fix: 'a' is usage 4, not HIToolbox 0. + assert_eq!(type_text("a"), vec![(4, false)]); + assert_eq!(type_text("b"), vec![(5, false)]); + assert_eq!(type_text("z"), vec![(29, false)]); + } + + #[test] + fn capitals_hold_shift_over_the_same_usage() { + assert_eq!(type_text("aA"), vec![(4, false), (4, true)]); + } + + #[test] + fn digits_are_not_contiguous_with_zero() { + assert_eq!(type_text("1"), vec![(30, false)]); + assert_eq!(type_text("9"), vec![(38, false)]); + assert_eq!(type_text("0"), vec![(39, false)]); + } + + #[test] + fn types_an_email_address() { + // The exact string the old implementation mangled into + // "testexample.com": @ and ? dropped, capitals lowercased. + let strokes = type_text("Test@Example.com?"); + assert_eq!(strokes.len(), 17, "every character must produce a key"); + assert_eq!(strokes[0], (23, true), "T is shifted t"); + assert_eq!(strokes[4], (31, true), "@ is shift-2"); + assert_eq!(strokes[16], (56, true), "? is shift-/"); + } + + #[test] + fn shifted_and_unshifted_symbols_share_usages() { + assert_eq!(type_text(";"), vec![(51, false)]); + assert_eq!(type_text(":"), vec![(51, true)]); + assert_eq!(type_text("/"), vec![(56, false)]); + assert_eq!(type_text("?"), vec![(56, true)]); + } + + #[test] + fn whitespace_maps_to_real_keys() { + assert_eq!(type_text(" "), vec![(44, false)]); + assert_eq!(type_text("\n"), vec![(usage::RETURN, false)]); + assert_eq!(type_text("\t"), vec![(usage::TAB, false)]); + } + + #[test] + fn unmappable_characters_fail_loudly() { + // Better a clear error than a silently wrong string. + for text in ["café", "hello 👋", "→"] { + let error = keystrokes_for(text).expect_err("should reject"); + assert!(error.to_string().contains("cannot type"), "{error}"); + } + } + + #[test] + fn failure_is_reported_before_anything_is_typed() { + // keystrokes_for collects into a Result, so a bad character partway + // through yields no partial output for the caller to send. + assert!(keystrokes_for("ok\u{1F600}bad").is_err()); + } + + #[test] + fn shift_is_the_only_modifier_for_text() { + assert_eq!( + keystroke_for('A').expect("mapped").modifiers(), + vec![usage::LEFT_SHIFT] + ); + assert!(keystroke_for('a').expect("mapped").modifiers().is_empty()); + } + + #[test] + fn every_printable_ascii_character_is_typeable() { + // If this ever regresses, some character silently becomes untypeable. + for byte in 0x20u8..0x7F { + let character = byte as char; + assert!( + keystroke_for(character).is_some(), + "no key for {character:?}" + ); + } + } +} diff --git a/packages/accessibility-core/src/platform/ios_simulator/session.rs b/packages/accessibility-core/src/platform/ios_simulator/session.rs new file mode 100644 index 0000000..ffcd8a3 --- /dev/null +++ b/packages/accessibility-core/src/platform/ios_simulator/session.rs @@ -0,0 +1,343 @@ +//! Ownership of the per-device simulator resources. +//! +//! The simulator's Objective-C objects are not `Sync` and their calls block, so +//! each one lives on a dedicated thread behind a command channel rather than in +//! a mutex on the async runtime. +//! +//! Input and accessibility get *separate* threads on purpose: an accessibility +//! tree fetch can take hundreds of milliseconds, and pointer events queued +//! behind one would make the stream feel broken. + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Instant; + +use anyhow::{Result, anyhow}; +use tokio::sync::{broadcast, mpsc, oneshot}; + +use crate::video::{ + EncodedFrame, FrameKind, Recording, RecordingConfig, VideoCapture, VideoConfig, +}; + +use super::SimulatorVideoCapture; +use super::ax::{AxCommand, AxSnapshot, ElementDetail, spawn_ax_worker}; +use super::input::{InputCommand, Orientation, spawn_input_worker}; +use super::settings::{self, Setting, SettingKey}; + +/// How many encoded frames to buffer per subscriber. +/// +/// Small on purpose: for interactive video a dropped frame is better than a +/// late one, and a slow client should not be able to inflate memory. +const FRAME_BUFFER: usize = 16; + +/// Counters for diagnosing stream quality and pacing. +/// +/// Cheap enough to always collect: the interesting failures here (keyframe +/// storms, subscribers falling behind) are invisible without them and only +/// show up under load, which is exactly when you cannot attach a profiler. +#[derive(Default)] +pub struct StreamStats { + pub frames: AtomicU64, + pub keyframes: AtomicU64, + pub bytes: AtomicU64, + /// Keyframes asked for by a new subscriber, an RTCP PLI, or a lagging + /// receiver. A high rate here starves the stream of bitrate for delta + /// frames and is self-reinforcing. + pub keyframe_requests: AtomicU64, + /// Times a subscriber fell far enough behind to drop frames. + pub lag_events: AtomicU64, +} + +/// A snapshot of [`StreamStats`] with rates worked out. +#[derive(Debug, Clone, serde::Serialize)] +pub struct StatsReport { + pub uptime_secs: f64, + pub frames: u64, + pub keyframes: u64, + pub bytes: u64, + pub fps: f64, + pub mbps: f64, + pub bits_per_pixel: f64, + pub mean_frame_kb: f64, + pub keyframe_requests: u64, + pub lag_events: u64, + pub subscribers: usize, + /// Frames written to the current recording, or `None` when idle. + pub recording_frames: Option, + /// Capture resolution. + pub width: u32, + pub height: u32, + /// Resolution actually encoded, after downscaling. + pub encoded_width: u32, + pub encoded_height: u32, +} + +/// Geometry and identity of the device being served. +#[derive(Debug, Clone, serde::Serialize)] +pub struct DeviceInfo { + pub udid: String, + /// Raw framebuffer width. Does not change with orientation. + pub width: u32, + /// Raw framebuffer height. Does not change with orientation. + pub height: u32, + pub orientation: Orientation, +} + +/// A transport-neutral, hardware-encoded iOS Simulator session. +pub struct SimSession { + device_udid: String, + capture: Box, + frames: broadcast::Sender, + /// Most recent parameter set, replayed to clients that join mid-stream. + latest_parameter_set: Arc>>, + stats: Arc, + started: Instant, + input: std::sync::mpsc::Sender, + ax: mpsc::UnboundedSender, + /// Last orientation we asked for. + /// + /// The framebuffer is always portrait-native — rotating the device + /// rotates the *content* inside a fixed-size surface — so orientation + /// cannot be recovered from the video and has to be tracked here. + orientation: std::sync::Mutex, +} + +impl SimSession { + /// Attach to a booted simulator and start capturing. + pub fn start(udid: Option<&str>, config: VideoConfig) -> Result> { + let (frames, _) = broadcast::channel(FRAME_BUFFER); + let latest_parameter_set = Arc::new(std::sync::Mutex::new(None)); + let stats = Arc::new(StreamStats::default()); + + let sink = { + let frames = frames.clone(); + let latest_parameter_set = Arc::clone(&latest_parameter_set); + let stats = Arc::clone(&stats); + Arc::new(move |frame: EncodedFrame| { + if frame.kind == FrameKind::ParameterSet { + *latest_parameter_set.lock().unwrap() = Some(frame.clone()); + } + stats.frames.fetch_add(1, Ordering::Relaxed); + stats + .bytes + .fetch_add(frame.data.len() as u64, Ordering::Relaxed); + if frame.kind == FrameKind::Keyframe { + stats.keyframes.fetch_add(1, Ordering::Relaxed); + } + // A send error just means nobody is watching yet. + let _ = frames.send(frame); + }) + }; + + let (capture, resolved_udid) = start_capture(udid, &config, sink)?; + let input = spawn_input_worker(&resolved_udid)?; + let ax = spawn_ax_worker(&resolved_udid)?; + + Ok(Arc::new(Self { + device_udid: resolved_udid, + capture, + frames, + latest_parameter_set, + stats, + started: Instant::now(), + input, + ax, + orientation: std::sync::Mutex::new(Orientation::Portrait), + })) + } + + pub fn subscribe(&self) -> broadcast::Receiver { + let receiver = self.frames.subscribe(); + #[allow(clippy::let_and_return)] + // A new subscriber cannot decode anything until the next keyframe, so + // ask for one immediately instead of making them wait out the interval. + self.capture.request_keyframe(); + receiver + } + + pub fn latest_parameter_set(&self) -> Option { + self.latest_parameter_set.lock().unwrap().clone() + } + + pub fn request_keyframe(&self) { + self.stats.keyframe_requests.fetch_add(1, Ordering::Relaxed); + self.capture.request_keyframe(); + } + + pub fn note_lag(&self) { + self.stats.lag_events.fetch_add(1, Ordering::Relaxed); + } + + pub fn stats(&self) -> StatsReport { + let elapsed = self.started.elapsed().as_secs_f64().max(1e-6); + let frames = self.stats.frames.load(Ordering::Relaxed); + let bytes = self.stats.bytes.load(Ordering::Relaxed); + let geometry = self.capture.geometry(); + let encoded = self.capture.encoded_geometry(); + // Bits per pixel only means anything against the encoded size. + let pixels = (encoded.width as f64) * (encoded.height as f64); + let fps = frames as f64 / elapsed; + + StatsReport { + uptime_secs: (elapsed * 10.0).round() / 10.0, + frames, + keyframes: self.stats.keyframes.load(Ordering::Relaxed), + bytes, + fps: (fps * 10.0).round() / 10.0, + mbps: ((bytes as f64 * 8.0 / elapsed / 1e6) * 100.0).round() / 100.0, + // The headline number: anything much under 0.1 will visibly + // block up on motion. + bits_per_pixel: if pixels > 0.0 && fps > 0.0 { + ((bytes as f64 * 8.0 / elapsed) / (pixels * fps) * 10000.0).round() / 10000.0 + } else { + 0.0 + }, + mean_frame_kb: if frames > 0 { + ((bytes as f64 / frames as f64 / 1024.0) * 100.0).round() / 100.0 + } else { + 0.0 + }, + keyframe_requests: self.stats.keyframe_requests.load(Ordering::Relaxed), + lag_events: self.stats.lag_events.load(Ordering::Relaxed), + subscribers: self.frames.receiver_count(), + recording_frames: self.capture.recording_frames(), + width: geometry.width, + height: geometry.height, + encoded_width: encoded.width, + encoded_height: encoded.height, + } + } + + pub fn device_info(&self) -> DeviceInfo { + let geometry = self.capture.geometry(); + DeviceInfo { + udid: self.device_udid.clone(), + width: geometry.width, + height: geometry.height, + orientation: self.orientation(), + } + } + + pub fn orientation(&self) -> Orientation { + *self.orientation.lock().unwrap() + } + + /// Rotate the device and remember the new orientation. + pub fn set_orientation(&self, orientation: Orientation) { + *self.orientation.lock().unwrap() = orientation; + self.send_input(InputCommand::Rotate { orientation }); + } + + /// Begin recording to a file beside the system temp directory. + /// + /// Runs a second encode of the same frames, so the live stream is + /// unaffected and the recording can use B-frames and its own resolution. + pub fn start_recording(&self, config: RecordingConfig) -> Result { + let path = std::env::temp_dir().join(format!( + "serve-sim-{}-{}.mp4", + self.device_udid, + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or_default() + )); + self.capture.start_recording(&path, &config)?; + Ok(path) + } + + pub fn stop_recording(&self) -> Result { + self.capture.stop_recording() + } + + pub fn recording_frames(&self) -> Option { + self.capture.recording_frames() + } + + pub fn settings(&self) -> Vec { + settings::read_all(&self.device_udid) + } + + pub fn set_setting(&self, key: SettingKey, value: &str) -> Result { + settings::write(&self.device_udid, key, value) + } + + /// Queue an input event. Fire-and-forget: pointer events must never block + /// the socket reader. + pub fn send_input(&self, command: InputCommand) { + if let InputCommand::Rotate { orientation } = command { + *self.orientation.lock().unwrap() = orientation; + } + let _ = self.input.send(command); + } + + /// Read the accessibility tree. + /// + /// `scan` additionally hit-tests the regions the tree walk cannot explain, + /// which is the only way to reach `WKWebView` and Safari content. It costs + /// a few hundred milliseconds, so it is opt-in. + pub async fn ax_snapshot(&self, scan: bool) -> Result { + let (tx, rx) = oneshot::channel(); + self.ax + .send(AxCommand::Snapshot { scan, reply: tx }) + .map_err(|_| anyhow!("accessibility worker stopped"))?; + let snapshot = rx + .await + .map_err(|_| anyhow!("accessibility worker stopped"))??; + self.reconcile_orientation(snapshot.is_landscape); + Ok(snapshot) + } + + /// Correct the tracked orientation against what the device reports. + /// + /// Orientation can change without us: the user can rotate from the + /// Simulator menu, an app can force an orientation, or the server can be + /// restarted while the device is already sideways. Accessibility bounds + /// are the only cheap signal, and they only reveal landscape vs portrait, + /// so a disagreement resolves to a sensible default of the right kind + /// rather than to an exact rotation. + fn reconcile_orientation(&self, is_landscape: bool) { + let mut orientation = self.orientation.lock().unwrap(); + if orientation.is_landscape() == is_landscape { + return; + } + *orientation = if is_landscape { + Orientation::LandscapeLeft + } else { + Orientation::Portrait + }; + } + + /// Best-effort orientation seed at startup. + /// + /// Without this the server would assume portrait and render a sideways + /// device whenever it attaches to an already-rotated simulator. + pub async fn seed_orientation(&self) { + if let Ok(snapshot) = self.ax_snapshot(false).await { + self.reconcile_orientation(snapshot.is_landscape); + } + } + + pub async fn ax_hit_test(&self, x: f64, y: f64) -> Result> { + let (tx, rx) = oneshot::channel(); + self.ax + .send(AxCommand::HitTest { x, y, reply: tx }) + .map_err(|_| anyhow!("accessibility worker stopped"))?; + rx.await + .map_err(|_| anyhow!("accessibility worker stopped"))? + } +} + +fn start_capture( + udid: Option<&str>, + config: &VideoConfig, + sink: crate::video::FrameSink, +) -> Result<(Box, String)> { + // Resolve the concrete UDID up front so the input and accessibility + // workers bind to the same device the video came from, even when the + // caller passed `None`. + let resolved = accessibility_ios_sys::SimFramebuffer::new(udid)? + .device_udid() + .to_string(); + let capture = SimulatorVideoCapture::start(&resolved, config, sink)?; + Ok((Box::new(capture), resolved)) +} diff --git a/packages/accessibility-core/src/platform/ios_simulator/settings.rs b/packages/accessibility-core/src/platform/ios_simulator/settings.rs new file mode 100644 index 0000000..a3bcd47 --- /dev/null +++ b/packages/accessibility-core/src/platform/ios_simulator/settings.rs @@ -0,0 +1,173 @@ +//! Simulator-wide UI settings. +//! +//! These go through `simctl ui`, which is the same mechanism Xcode's Devices +//! window uses. Only the three options simctl actually implements are exposed: +//! appearance, increase contrast, and content size. +//! +//! The Devices window also offers reduce-motion, colour filters, transparency +//! and VoiceOver, but simctl has no verb for those — they require a helper +//! binary spawned *inside* the simulator that drives the private +//! libAccessibility setters. That is a meaningfully larger piece of work and +//! is deliberately not attempted here. + +use anyhow::{Context, Result, anyhow}; +use serde::{Deserialize, Serialize}; + +/// Content size categories, smallest to largest. +/// +/// The five `accessibility-*` entries are the extended range that only appears +/// once a user opts into larger accessibility text. +pub const CONTENT_SIZES: &[&str] = &[ + "extra-small", + "small", + "medium", + "large", + "extra-large", + "extra-extra-large", + "extra-extra-extra-large", + "accessibility-medium", + "accessibility-large", + "accessibility-extra-large", + "accessibility-extra-extra-large", + "accessibility-extra-extra-extra-large", +]; + +pub const APPEARANCES: &[&str] = &["light", "dark"]; +pub const TOGGLE: &[&str] = &["enabled", "disabled"]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SettingKey { + Appearance, + IncreaseContrast, + ContentSize, +} + +impl SettingKey { + /// The `simctl ui` subcommand for this setting. + fn verb(self) -> &'static str { + match self { + SettingKey::Appearance => "appearance", + SettingKey::IncreaseContrast => "increase_contrast", + SettingKey::ContentSize => "content_size", + } + } + + pub fn allowed_values(self) -> &'static [&'static str] { + match self { + SettingKey::Appearance => APPEARANCES, + SettingKey::IncreaseContrast => TOGGLE, + SettingKey::ContentSize => CONTENT_SIZES, + } + } + + pub fn all() -> [SettingKey; 3] { + [ + SettingKey::Appearance, + SettingKey::IncreaseContrast, + SettingKey::ContentSize, + ] + } +} + +/// One setting and its current value, as reported by the simulator. +#[derive(Debug, Clone, Serialize)] +pub struct Setting { + pub key: SettingKey, + /// Current value, or `unsupported`/`unknown` if the runtime says so. + pub value: String, + pub allowed: &'static [&'static str], +} + +/// Read every supported setting from the device. +pub fn read_all(udid: &str) -> Vec { + SettingKey::all() + .into_iter() + .map(|key| Setting { + key, + // A failed read is reported as unknown rather than failing the + // whole request; one unsupported option should not blank the UI. + value: read(udid, key).unwrap_or_else(|_| "unknown".to_string()), + allowed: key.allowed_values(), + }) + .collect() +} + +pub fn read(udid: &str, key: SettingKey) -> Result { + let output = simctl(&["ui", udid, key.verb()])?; + Ok(output.trim().to_string()) +} + +pub fn write(udid: &str, key: SettingKey, value: &str) -> Result { + // `content_size` also accepts increment/decrement, which are not in the + // reported value set but are the ergonomic way to drive it from a UI. + let stepping = + matches!(key, SettingKey::ContentSize) && matches!(value, "increment" | "decrement"); + + if !stepping && !key.allowed_values().contains(&value) { + return Err(anyhow!( + "'{value}' is not valid for {:?}; expected one of {}", + key, + key.allowed_values().join(", ") + )); + } + + simctl(&["ui", udid, key.verb(), value])?; + read(udid, key) +} + +fn simctl(args: &[&str]) -> Result { + let output = std::process::Command::new("xcrun") + .arg("simctl") + .args(args) + .output() + .context("failed to run xcrun simctl")?; + + if !output.status.success() { + return Err(anyhow!( + "simctl {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + )); + } + Ok(String::from_utf8_lossy(&output.stdout).to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_values_outside_the_allowed_set() { + let error = write("no-such-device", SettingKey::Appearance, "chartreuse") + .expect_err("invalid appearance should be rejected"); + // Rejected locally, without ever shelling out to simctl. + assert!(error.to_string().contains("chartreuse")); + } + + #[test] + fn content_size_accepts_stepping_verbs() { + // These are not reported values, so they must be allowed explicitly. + assert!(!CONTENT_SIZES.contains(&"increment")); + for value in ["increment", "decrement"] { + let error = write("no-such-device", SettingKey::ContentSize, value) + .expect_err("no such device"); + assert!( + !error.to_string().contains("is not valid"), + "{value} should reach simctl rather than being rejected" + ); + } + } + + #[test] + fn every_key_has_values_and_a_distinct_verb() { + let mut verbs = Vec::new(); + for key in SettingKey::all() { + assert!(!key.allowed_values().is_empty()); + verbs.push(key.verb()); + } + verbs.sort_unstable(); + verbs.dedup(); + assert_eq!(verbs.len(), SettingKey::all().len()); + } +} diff --git a/packages/accessibility-ios-sys/src/macos.rs b/packages/accessibility-ios-sys/src/macos.rs index c0559b3..883dd24 100644 --- a/packages/accessibility-ios-sys/src/macos.rs +++ b/packages/accessibility-ios-sys/src/macos.rs @@ -67,8 +67,8 @@ mod stream; mod void_block; pub use common::{ - ButtonDirection, Element, ElementKey, ElementTree, HardwareButton, Point, Rect, ScreenSpace, - Screenshot, Size, TreeFilter, load_frameworks, + BootedSimulator, ButtonDirection, Element, ElementKey, ElementTree, HardwareButton, Point, + Rect, ScreenSpace, Screenshot, Size, TreeFilter, booted_simulators, load_frameworks, }; pub use encoder::{ ChunkKind, ChunkSink, EncodedChunk, EncoderConfig, H264Encoder, NalFormat, Tuning, diff --git a/packages/accessibility-ios-sys/src/macos/common.rs b/packages/accessibility-ios-sys/src/macos/common.rs index 9ebfd8d..938f9d3 100644 --- a/packages/accessibility-ios-sys/src/macos/common.rs +++ b/packages/accessibility-ios-sys/src/macos/common.rs @@ -8,6 +8,16 @@ use accesskit::Role; use euclid::{Point2D, Rect as EuclidRect, Size2D}; use slotmap::{Key, KeyData, SlotMap}; +/// Identity of a currently booted simulator. +/// +/// The UDID is the stable device identifier used to bind independent capture, +/// input, and accessibility sessions to the same simulator. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct BootedSimulator { + pub udid: String, + pub name: String, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct ScreenSpace; @@ -490,6 +500,68 @@ pub(super) unsafe fn nsstring_to_string_static(ns_string: *mut AnyObject) -> Opt Some(CStr::from_ptr(cstr).to_string_lossy().to_string()) } +/// Enumerate all currently booted simulators. +/// +/// Devices are returned in CoreSimulator's order. Each entry includes the +/// stable UDID used by the rest of this crate and the user-visible device name. +pub fn booted_simulators() -> Result> { + crate::frameworks::load_coresimulator_framework()?; + + unsafe { + let device_set = get_device_set()?; + + // Resolve each device while the owning NSArray is still in scope. The + // objects returned by objectAtIndex: are unretained and must not be + // collected as raw pointers for later processing. + let devices: *mut AnyObject = msg_send![device_set, devices]; + if devices.is_null() { + return Err(anyhow!("No devices found in SimDeviceSet")); + } + + let count: usize = msg_send![devices, count]; + let mut booted = Vec::new(); + for i in 0..count { + let device: *mut AnyObject = msg_send![devices, objectAtIndex: i]; + if device.is_null() { + continue; + } + + // Check if booted (state == 3) + let state: i64 = msg_send![device, state]; + if state != 3 { + continue; + } + + // A stale or partially initialized SimDevice should not hide the + // rest of the catalog. Keep every valid entry in native order. + if let Ok(info) = booted_simulator_info(device) { + booted.push(info); + } + } + Ok(booted) + } +} + +/// Read the stable identity fields from a SimDevice. +/// +/// # Safety +/// `device` must be a live SimDevice from the loaded CoreSimulator framework. +unsafe fn booted_simulator_info(device: *mut AnyObject) -> Result { + let device_udid: *mut AnyObject = msg_send![device, UDID]; + if device_udid.is_null() { + return Err(anyhow!("Booted simulator has no UDID")); + } + let udid_string: *mut AnyObject = msg_send![device_udid, UUIDString]; + let udid = nsstring_to_string_static(udid_string) + .ok_or_else(|| anyhow!("Failed to read booted simulator UDID"))?; + + let name_string: *mut AnyObject = msg_send![device, name]; + let name = nsstring_to_string_static(name_string) + .ok_or_else(|| anyhow!("Failed to read booted simulator name for {udid}"))?; + + Ok(BootedSimulator { udid, name }) +} + /// Find a booted simulator device by UDID or return the first booted one. /// /// # Safety @@ -504,7 +576,6 @@ pub(super) unsafe fn find_booted_device(udid: Option<&str>) -> Result<*mut AnyOb } let count: usize = msg_send![devices, count]; - for i in 0..count { let device: *mut AnyObject = msg_send![devices, objectAtIndex: i]; if device.is_null() { @@ -514,27 +585,21 @@ pub(super) unsafe fn find_booted_device(udid: Option<&str>) -> Result<*mut AnyOb // Check if booted (state == 3) let state: i64 = msg_send![device, state]; if state != 3 { - // Not booted continue; } - // Get UDID - let device_udid: *mut AnyObject = msg_send![device, UDID]; - if device_udid.is_null() { - continue; - } - - let udid_string: *mut AnyObject = msg_send![device_udid, UUIDString]; - if udid_string.is_null() { - continue; - } - - let udid_cstr: *const c_char = msg_send![udid_string, UTF8String]; - let device_udid_str = CStr::from_ptr(udid_cstr).to_string_lossy(); - - // If we're looking for a specific UDID, check it + // If we're looking for a specific UDID, check it. Keep malformed + // entries skippable just as the original lookup did. if let Some(target_udid) = udid { - if device_udid_str == target_udid { + let device_udid: *mut AnyObject = msg_send![device, UDID]; + if device_udid.is_null() { + continue; + } + let udid_string: *mut AnyObject = msg_send![device_udid, UUIDString]; + let Some(device_udid) = nsstring_to_string_static(udid_string) else { + continue; + }; + if device_udid == target_udid { return Ok(device); } } else { diff --git a/packages/accessibility-serve/Cargo.toml b/packages/accessibility-serve/Cargo.toml index b3ec46c..3d42ef5 100644 --- a/packages/accessibility-serve/Cargo.toml +++ b/packages/accessibility-serve/Cargo.toml @@ -22,6 +22,3 @@ tokio.workspace = true tower-http = { workspace = true, features = ["cors"] } tracing.workspace = true webrtc.workspace = true - -[target.'cfg(target_os = "macos")'.dependencies] -accessibility-ios-sys.workspace = true diff --git a/packages/accessibility-serve/src/ax.rs b/packages/accessibility-serve/src/ax.rs index 7fbbe25..f4bed4a 100644 --- a/packages/accessibility-serve/src/ax.rs +++ b/packages/accessibility-serve/src/ax.rs @@ -1,487 +1,3 @@ -//! Accessibility element inspection for the web UI. -//! -//! Two things are exposed: -//! -//! - A whole-tree snapshot, which the browser uses for instant hover feedback. -//! - A live hit test, which asks the simulator what is actually at a point. -//! -//! The snapshot can go stale between fetches and mis-picks overlapping or -//! transformed views; the hit test is authoritative but costs a round trip. -//! The UI uses the snapshot while the pointer is moving and confirms with the -//! hit test once it settles, which is why both exist. -//! -//! # Coordinate spaces -//! -//! Accessibility frames come back in points relative to the Simulator window, -//! so every rect is converted to a 0..1 fraction of the app's own bounds -//! before it leaves this module: -//! -//! ```text -//! normalized = (ax_rect.origin - app_bounds.origin) / app_bounds.size -//! ``` -//! -//! That makes the values independent of the display scale factor, which is -//! why no pixel/point conversion appears here. -//! -//! Crucially, these are **logical** coordinates, already rotated by iOS: in -//! landscape the app reports its bounds as 874x402 rather than 402x874, so the -//! normalized rects are upright and need no further rotation to be drawn. -//! -//! This is the opposite of the HID input path, which is in *raw framebuffer* -//! space — the framebuffer never rotates, so pointer coordinates have to be -//! un-rotated before injection. The two spaces coincide in portrait, which is -//! exactly what makes the difference easy to miss. +//! Compatibility re-exports for iOS Simulator accessibility inspection. -use anyhow::{Result, anyhow}; -use serde::Serialize; -use tokio::sync::{mpsc, oneshot}; - -use accessibility_core::accessibility::{Element, Rect, TreeFilter}; - -use crate::coverage::CoverageGrid; - -/// A rectangle in normalized display space (0..1 on both axes). -#[derive(Debug, Clone, Copy, Serialize)] -pub struct NormalizedRect { - pub x: f64, - pub y: f64, - pub width: f64, - pub height: f64, -} - -/// Fraction of the screen above which an element is treated as a backdrop -/// rather than something you can point at. -/// -/// Every app has an Application node and usually one or more full-bleed -/// container groups. Hit testing an empty region resolves to one of them, and -/// highlighting it paints a box over the entire device, which reads as "the -/// picker is broken" rather than "there is nothing here". -const BACKDROP_AREA: f64 = 0.9; - -impl NormalizedRect { - /// Whether this covers essentially the whole screen. - pub fn is_backdrop(&self) -> bool { - self.width * self.height >= BACKDROP_AREA - } - - fn from_screen(rect: &Rect, app_bounds: &Rect) -> Option { - if app_bounds.size.width <= 0.0 || app_bounds.size.height <= 0.0 { - return None; - } - Some(Self { - x: (rect.origin.x - app_bounds.origin.x) / app_bounds.size.width, - y: (rect.origin.y - app_bounds.origin.y) / app_bounds.size.height, - width: rect.size.width / app_bounds.size.width, - height: rect.size.height / app_bounds.size.height, - }) - } -} - -/// How an element was found. -/// -/// Worth surfacing: a swept element is a point sample with no parent, no -/// children and no document order, so consumers should not treat it as -/// equivalent to a node the tree walk returned. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum Discovery { - /// Walked from the application root. - Recursive, - /// Found by hit testing a grid point the tree could not explain. - PointGrid, -} - -/// One inspectable element, flattened for the browser. -#[derive(Debug, Clone, Serialize)] -pub struct ElementDetail { - pub id: String, - pub role: String, - pub label: Option, - pub value: Option, - pub identifier: Option, - pub enabled: bool, - pub focused: bool, - pub actions: Vec, - pub bounds: Option, - /// Depth in the tree, used by the UI to prefer the innermost hit. - pub depth: u32, - /// A selector that would target this element from the CLI. - pub selector: String, - pub discovery: Discovery, -} - -#[derive(Debug, Clone, Serialize)] -pub struct AxSnapshot { - pub app_name: Option, - pub pid: Option, - pub elements: Vec, - /// Fraction of the display explained by the tree walk alone, before any - /// sweep. Low numbers on a busy screen mean out-of-process content. - pub coverage: f64, - /// Coverage after sweeping, when a scan was requested. - pub coverage_after_scan: Option, - /// How many points the sweep probed. `None` when no scan was requested. - pub probes: Option, - /// Whether the app reports itself wider than tall. - /// - /// Accessibility bounds are in logical space, so this is the one cheap - /// signal that reveals the device's real orientation. The framebuffer - /// cannot provide it: it never rotates. - pub is_landscape: bool, -} - -pub enum AxCommand { - Snapshot { - /// Also sweep the regions the tree walk cannot explain. - scan: bool, - reply: oneshot::Sender>, - }, - HitTest { - x: f64, - y: f64, - reply: oneshot::Sender>>, - }, -} - -/// Build the `accessibility-cli` selector that would resolve this element. -/// -/// Preferring the identifier keeps the selector stable across copy changes, -/// which is the whole point of exposing it in the inspector. -fn selector_for(element: &Element, role: &str) -> String { - if let Some(identifier) = element.identifier.as_deref().filter(|s| !s.is_empty()) { - return format!("#{identifier}"); - } - if let Some(label) = element.title.as_deref().filter(|s| !s.is_empty()) { - return format!("{role}[label=\"{}\"]", label.replace('"', "\\\"")); - } - role.to_string() -} - -fn to_detail( - element: &Element, - app_bounds: &Rect, - depth: u32, - discovery: Discovery, -) -> ElementDetail { - let role = format!("{:?}", element.role); - ElementDetail { - discovery, - id: element.id.to_string(), - selector: selector_for(element, &role), - role, - label: element.title.clone().filter(|s| !s.is_empty()), - value: element.value.clone().filter(|s| !s.is_empty()), - identifier: element.identifier.clone().filter(|s| !s.is_empty()), - enabled: element.enabled, - focused: element.focused, - actions: element.actions.clone(), - bounds: element - .bounds - .as_ref() - .and_then(|bounds| NormalizedRect::from_screen(bounds, app_bounds)), - depth, - } -} - -fn flatten(element: &Element, app_bounds: &Rect, depth: u32, out: &mut Vec) { - out.push(to_detail(element, app_bounds, depth, Discovery::Recursive)); - for child in &element.children { - flatten(child, app_bounds, depth + 1, out); - } -} - -/// Start the accessibility worker thread and return its command channel. -#[cfg(target_os = "macos")] -pub fn spawn_ax_worker(udid: &str) -> Result> { - use accessibility_core::platform::ios_simulator::IOSSimulatorAccessibility; - - let mut reader = IOSSimulatorAccessibility::new(Some(udid))?; - let (tx, mut rx) = mpsc::unbounded_channel::(); - - std::thread::Builder::new() - .name("sim-ax".into()) - .spawn(move || { - while let Some(command) = rx.blocking_recv() { - match command { - AxCommand::Snapshot { scan, reply } => { - let _ = reply.send(snapshot(&mut reader, scan)); - } - AxCommand::HitTest { x, y, reply } => { - let _ = reply.send(hit_test(&mut reader, x, y)); - } - } - } - })?; - - Ok(tx) -} - -#[cfg(target_os = "macos")] -fn snapshot( - reader: &mut accessibility_core::platform::ios_simulator::IOSSimulatorAccessibility, - scan: bool, -) -> Result { - let tree = reader.get_tree(&TreeFilter::default())?; - // `get_screen_bounds` is only populated once a tree has been read, so it - // has to be queried after the fetch above. - let app_bounds = reader.get_screen_bounds()?; - - let mut elements: Vec = Vec::with_capacity(tree.element_count); - flatten(&tree.root, &app_bounds, 0, &mut elements); - - // Drop backdrops and anything with no usable geometry. Both are real parts - // of the tree but neither can be pointed at, and leaving them in makes the - // client's containment search pick them constantly. - elements.retain(|element| { - element.bounds.is_some_and(|bounds| { - !bounds.is_backdrop() && bounds.width > 0.0 && bounds.height > 0.0 - }) - }); - - // Everything the tree walk explained, so the sweep can skip it. - let mut coverage = CoverageGrid::new(); - for element in &elements { - if let Some(bounds) = element.bounds { - coverage.mark(&bounds); - } - } - let coverage_before = coverage.ratio(); - - let mut probes = None; - let mut coverage_after_scan = None; - if scan { - let swept = sweep(reader, &app_bounds, &mut coverage, &elements)?; - probes = Some(swept.probes); - elements.extend(swept.elements); - coverage_after_scan = Some(coverage.ratio()); - } - - Ok(AxSnapshot { - app_name: tree.app_name, - pid: tree.pid, - elements, - coverage: coverage_before, - coverage_after_scan, - probes, - is_landscape: app_bounds.size.width > app_bounds.size.height, - }) -} - -#[cfg(target_os = "macos")] -fn hit_test( - reader: &mut accessibility_core::platform::ios_simulator::IOSSimulatorAccessibility, - x: f64, - y: f64, -) -> Result> { - // The hit test wants macOS screen points, so the browser's normalized - // coordinates are mapped back through the app's bounds. - let app_bounds = reader - .get_screen_bounds() - .map_err(|_| anyhow!("no accessibility snapshot yet; fetch the tree first"))?; - - let screen_x = app_bounds.origin.x + x * app_bounds.size.width; - let screen_y = app_bounds.origin.y + y * app_bounds.size.height; - - let Some(element) = reader.element_at_point(screen_x, screen_y)? else { - return Ok(None); - }; - let detail = to_detail(&element, &app_bounds, 0, Discovery::Recursive); - - // Nothing pointable here. Reporting the backdrop would highlight the whole - // device; reporting nothing lets the caller leave the previous selection - // or clear it. - if detail.bounds.is_none_or(|bounds| bounds.is_backdrop()) { - return Ok(None); - } - Ok(Some(detail)) -} - -#[cfg(not(target_os = "macos"))] -pub fn spawn_ax_worker(_udid: &str) -> Result> { - anyhow::bail!("Simulator accessibility requires macOS") -} - -/// Spacing between sweep probes, in device points. -/// -/// idb uses 50; 40 is a little denser for phone-sized screens, where rows are -/// around 50 points tall and a coarser grid can step straight over one. Every -/// probe is a hit test, so this trades roughly linearly against scan time. -#[cfg(target_os = "macos")] -const SWEEP_STEP_POINTS: f64 = 40.0; - -/// Upper bound on probes, so a scan cannot run away on a large display. -#[cfg(target_os = "macos")] -const SWEEP_MAX_PROBES: usize = 400; - -#[cfg(target_os = "macos")] -struct SweepResult { - elements: Vec, - probes: usize, -} - -/// Hit test the regions the tree walk could not explain. -/// -/// This is the only way to reach content in another process — `WKWebView` and -/// Safari above all — because such content is individually addressable by -/// point but has no traversable hierarchy. The result is a flat set of point -/// samples, which is why each is tagged [`Discovery::PointGrid`]. -#[cfg(target_os = "macos")] -fn sweep( - reader: &mut accessibility_core::platform::ios_simulator::IOSSimulatorAccessibility, - app_bounds: &Rect, - coverage: &mut CoverageGrid, - known: &[ElementDetail], -) -> Result { - use std::collections::HashSet; - - // Elements the tree already reported, keyed by position, so a probe that - // lands on one does not report it twice under a different provenance. - let mut seen: HashSet = known - .iter() - .filter_map(|element| element.bounds.map(frame_key)) - .collect(); - - let mut elements = Vec::new(); - let mut probes = 0usize; - - let columns = (app_bounds.size.width / SWEEP_STEP_POINTS).floor().max(1.0) as usize; - let rows = (app_bounds.size.height / SWEEP_STEP_POINTS) - .floor() - .max(1.0) as usize; - - for row in 0..rows { - for column in 0..columns { - if probes >= SWEEP_MAX_PROBES { - return Ok(SweepResult { elements, probes }); - } - - // Probe cell centres so a point never lands exactly on a boundary. - let x = (column as f64 + 0.5) / columns as f64; - let y = (row as f64 + 0.5) / rows as f64; - - // Already explained by the tree; nothing to learn here. - if coverage.is_filled(x, y) { - continue; - } - probes += 1; - - let screen_x = app_bounds.origin.x + x * app_bounds.size.width; - let screen_y = app_bounds.origin.y + y * app_bounds.size.height; - let Ok(Some(element)) = reader.element_at_point(screen_x, screen_y) else { - continue; - }; - - let detail = to_detail(&element, app_bounds, 0, Discovery::PointGrid); - let Some(bounds) = detail.bounds else { - continue; - }; - if bounds.is_backdrop() || bounds.width <= 0.0 || bounds.height <= 0.0 { - continue; - } - // A 40 point grid lands on a large element many times over. - if !seen.insert(frame_key(bounds)) { - // Still mark it: further probes inside it are wasted. - coverage.mark(&bounds); - continue; - } - - coverage.mark(&bounds); - elements.push(detail); - } - } - - Ok(SweepResult { elements, probes }) -} - -/// Position key for deduplication, rounded so float noise does not defeat it. -#[cfg(target_os = "macos")] -fn frame_key(bounds: NormalizedRect) -> String { - format!( - "{:.4},{:.4},{:.4},{:.4}", - bounds.x, bounds.y, bounds.width, bounds.height - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use accessibility_core::accessibility::{Point, Size}; - - fn rect(x: f64, y: f64, w: f64, h: f64) -> Rect { - Rect::new(Point::new(x, y), Size::new(w, h)) - } - - #[test] - fn normalizes_against_app_bounds_not_the_screen() { - // Landscape: the app reports itself 874x402, and accessibility frames - // are already in that logical space. - let app = rect(0.0, 0.0, 874.0, 402.0); - let button = rect(754.5, 82.0, 27.0, 44.0); - let normalized = NormalizedRect::from_screen(&button, &app).expect("normalizes"); - assert!((normalized.x - 0.863).abs() < 0.001); - assert!((normalized.y - 0.204).abs() < 0.001); - assert!((normalized.width - 0.031).abs() < 0.001); - } - - #[test] - fn offsets_by_the_app_origin() { - let app = rect(100.0, 50.0, 400.0, 800.0); - let element = rect(300.0, 450.0, 200.0, 400.0); - let normalized = NormalizedRect::from_screen(&element, &app).expect("normalizes"); - assert!((normalized.x - 0.5).abs() < 1e-9); - assert!((normalized.y - 0.5).abs() < 1e-9); - } - - #[test] - fn degenerate_app_bounds_do_not_divide_by_zero() { - let app = rect(0.0, 0.0, 0.0, 0.0); - assert!(NormalizedRect::from_screen(&rect(1.0, 1.0, 2.0, 2.0), &app).is_none()); - } - - #[test] - fn full_screen_containers_are_backdrops() { - // Every app has these; highlighting one paints over the whole device. - assert!( - NormalizedRect { - x: 0.0, - y: 0.0, - width: 1.0, - height: 1.0 - } - .is_backdrop() - ); - assert!( - NormalizedRect { - x: 0.0, - y: 0.0, - width: 0.98, - height: 0.96 - } - .is_backdrop() - ); - } - - #[test] - fn ordinary_controls_are_not_backdrops() { - // A full-width settings row is large but perfectly pointable. - assert!( - !NormalizedRect { - x: 0.05, - y: 0.45, - width: 0.90, - height: 0.06 - } - .is_backdrop() - ); - // So is a tall sidebar. - assert!( - !NormalizedRect { - x: 0.0, - y: 0.0, - width: 0.25, - height: 1.0 - } - .is_backdrop() - ); - } -} +pub use accessibility_core::platform::ios_simulator::ax::*; diff --git a/packages/accessibility-serve/src/coverage.rs b/packages/accessibility-serve/src/coverage.rs index 3686a34..9ef905c 100644 --- a/packages/accessibility-serve/src/coverage.rs +++ b/packages/accessibility-serve/src/coverage.rs @@ -1,138 +1,3 @@ -//! Tracking which parts of the screen the accessibility tree explains. -//! -//! Two jobs. It measures how much of the display is accounted for by elements -//! the tree walk found, which is a direct health signal — a screen full of -//! content reporting 5% coverage means most of the UI is invisible to every -//! tree-based tool. And it tells the point-grid sweep where not to bother -//! probing, so discovery only pays for the unexplained regions. +//! Compatibility re-exports for accessibility coverage tracking. -use crate::ax::NormalizedRect; - -/// Cells along each axis. -/// -/// 32x64 over a normalized display is about 12x13 points per cell on a phone — -/// fine enough that a table row does not mark a neighbouring row as covered, -/// coarse enough to stay a cheap bitmap. -const COLUMNS: usize = 32; -const ROWS: usize = 64; - -pub struct CoverageGrid { - filled: [bool; COLUMNS * ROWS], -} - -impl Default for CoverageGrid { - fn default() -> Self { - Self::new() - } -} - -impl CoverageGrid { - pub fn new() -> Self { - Self { - filled: [false; COLUMNS * ROWS], - } - } - - /// Mark every cell a normalized rect touches. - pub fn mark(&mut self, rect: &NormalizedRect) { - // Backdrops cover everything and would leave nothing to discover. - if rect.is_backdrop() { - return; - } - let (min_col, max_col) = span(rect.x, rect.width, COLUMNS); - let (min_row, max_row) = span(rect.y, rect.height, ROWS); - for row in min_row..=max_row { - for column in min_col..=max_col { - self.filled[row * COLUMNS + column] = true; - } - } - } - - pub fn is_filled(&self, x: f64, y: f64) -> bool { - let column = index(x, COLUMNS); - let row = index(y, ROWS); - self.filled[row * COLUMNS + column] - } - - /// Fraction of the display explained so far, 0.0 to 1.0. - pub fn ratio(&self) -> f64 { - let filled = self.filled.iter().filter(|cell| **cell).count(); - filled as f64 / self.filled.len() as f64 - } -} - -fn index(value: f64, cells: usize) -> usize { - ((value.clamp(0.0, 1.0) * cells as f64) as usize).min(cells - 1) -} - -/// Inclusive cell range covered by an interval, clamped to the display. -fn span(start: f64, length: f64, cells: usize) -> (usize, usize) { - let low = index(start, cells); - let high = index(start + length.max(0.0), cells); - (low.min(high), high.max(low)) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn rect(x: f64, y: f64, width: f64, height: f64) -> NormalizedRect { - NormalizedRect { - x, - y, - width, - height, - } - } - - #[test] - fn empty_grid_explains_nothing() { - assert_eq!(CoverageGrid::new().ratio(), 0.0); - assert!(!CoverageGrid::new().is_filled(0.5, 0.5)); - } - - #[test] - fn marking_covers_the_rect_and_not_its_surroundings() { - let mut grid = CoverageGrid::new(); - grid.mark(&rect(0.4, 0.4, 0.2, 0.2)); - assert!(grid.is_filled(0.5, 0.5)); - assert!(!grid.is_filled(0.1, 0.1)); - assert!(!grid.is_filled(0.9, 0.9)); - } - - #[test] - fn backdrops_are_ignored() { - // Otherwise the Application node alone would report full coverage and - // the sweep would never probe anywhere. - let mut grid = CoverageGrid::new(); - grid.mark(&rect(0.0, 0.0, 1.0, 1.0)); - assert_eq!(grid.ratio(), 0.0); - } - - #[test] - fn ratio_grows_with_marked_area() { - let mut grid = CoverageGrid::new(); - let before = grid.ratio(); - grid.mark(&rect(0.0, 0.0, 0.5, 0.5)); - let after = grid.ratio(); - assert!(after > before); - // A quarter of the display, within one cell of rounding on each axis. - assert!((after - 0.25).abs() < 0.05, "ratio was {after}"); - } - - #[test] - fn out_of_range_rects_do_not_panic() { - let mut grid = CoverageGrid::new(); - // Rows can extend past the bottom of the screen; seen in real trees. - grid.mark(&rect(0.9, 0.95, 0.5, 0.5)); - grid.mark(&rect(-0.2, -0.2, 0.1, 0.1)); - assert!(grid.is_filled(0.99, 0.99)); - } - - #[test] - fn thin_rects_still_mark_a_cell() { - let mut grid = CoverageGrid::new(); - grid.mark(&rect(0.5, 0.5, 0.0, 0.0)); - assert!(grid.is_filled(0.5, 0.5)); - } -} +pub use accessibility_core::platform::ios_simulator::coverage::*; diff --git a/packages/accessibility-serve/src/input.rs b/packages/accessibility-serve/src/input.rs index f6ea75b..f8d28fc 100644 --- a/packages/accessibility-serve/src/input.rs +++ b/packages/accessibility-serve/src/input.rs @@ -1,345 +1,3 @@ -//! Input forwarding from the browser to the simulator's HID subsystem. -//! -//! All coordinates on this path are normalized 0..1 fractions of the *raw* -//! framebuffer. The browser un-rotates them before sending, so nothing here -//! needs to know about orientation, and no points/pixels/scale conversion is -//! involved anywhere. +//! Compatibility re-exports for iOS Simulator input orchestration. -use std::sync::mpsc::{self, RecvTimeoutError, Sender}; -use std::time::{Duration, Instant}; - -use anyhow::Result; -use serde::Deserialize; - -/// Touches below this fraction of the screen height are tagged as originating -/// from the bottom edge, which is what makes swipe-up-to-home work. Without -/// the edge hint iOS treats the drag as a normal in-app gesture. -pub const HOME_INDICATOR_BAND: f64 = 0.93; - -/// How far a wheel delta moves the virtual finger, as a multiple of the -/// delta itself. -const SCROLL_GAIN: f64 = 1.0; - -/// How close to an edge the virtual finger may get before the drag is -/// restarted from the middle of the screen. -const SCROLL_EDGE_MARGIN: f64 = 0.08; - -/// Quiet period after which a scroll gesture is lifted. -const SCROLL_IDLE: Duration = Duration::from_millis(100); - -/// How often the worker wakes to check for an expired scroll gesture. -const WORKER_TICK: Duration = Duration::from_millis(25); - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum TouchPhase { - Begin, - Move, - End, -} - -/// Raw-framebuffer edge a touch is flagged as coming from. -/// -/// Required for iOS to recognize system gestures such as swipe-up-to-home. -/// The client decides this, because only it knows the current orientation and -/// the framebuffer never rotates. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Default)] -#[serde(rename_all = "lowercase")] -pub enum TouchEdge { - #[default] - None, - Left, - Top, - Bottom, - Right, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum HardwareButton { - Home, - Lock, - Siri, - SideButton, - ApplePay, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, serde::Serialize)] -#[serde(rename_all = "snake_case")] -pub enum Orientation { - Portrait, - PortraitUpsideDown, - LandscapeLeft, - LandscapeRight, -} - -impl Orientation { - /// Whether the display is wider than it is tall in this orientation. - pub fn is_landscape(self) -> bool { - matches!( - self, - Orientation::LandscapeLeft | Orientation::LandscapeRight - ) - } -} - -/// A single input action to apply to the simulator. -#[derive(Debug, Clone, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum InputCommand { - Touch { - phase: TouchPhase, - x: f64, - y: f64, - #[serde(default)] - edge: TouchEdge, - }, - Button { - button: HardwareButton, - }, - /// A single key press by USB HID usage code, with optional held modifiers. - /// - /// Used for navigation and shortcuts; text goes through [`InputCommand::Text`] - /// so the character-to-key table lives in one place. - Key { - key_code: u32, - #[serde(default)] - modifiers: Vec, - }, - /// Type a string, expanded server-side into key presses. - Text { - text: String, - }, - /// A wheel or trackpad delta, as a fraction of the display. - Scroll { - dx: f64, - dy: f64, - x: f64, - y: f64, - }, - Rotate { - orientation: Orientation, - }, -} - -/// Turns a stream of wheel deltas into a touch drag. -/// -/// iOS has no notion of a scroll wheel, so scrolling has to be a finger. The -/// awkward part is that a real finger runs out of screen: once the virtual -/// contact point nears an edge it is lifted and re-planted in the middle, so -/// an unbounded wheel can keep producing motion. -#[derive(Default)] -struct ScrollGesture { - active: bool, - x: f64, - y: f64, - last_event: Option, -} - -impl ScrollGesture { - fn near_edge(&self) -> bool { - self.x < SCROLL_EDGE_MARGIN - || self.x > 1.0 - SCROLL_EDGE_MARGIN - || self.y < SCROLL_EDGE_MARGIN - || self.y > 1.0 - SCROLL_EDGE_MARGIN - } -} - -/// Start the HID worker thread and return its command channel. -/// -/// The worker owns the `SimulatorHID` because it is not `Sync`, and because -/// HID sends block on a dispatch queue round trip. It wakes periodically even -/// when idle so a scroll gesture can be lifted after the wheel stops. -#[cfg(target_os = "macos")] -pub fn spawn_input_worker(udid: &str) -> Result> { - use accessibility_ios_sys::{ - HardwareButton as SysButton, Orientation as SysOrientation, SimulatorHID, - TouchEdge as SysEdge, TouchPhase as SysPhase, - }; - - let hid = SimulatorHID::for_device(Some(udid))?; - let (tx, rx) = mpsc::channel::(); - - std::thread::Builder::new() - .name("sim-input".into()) - .spawn(move || { - let mut scroll = ScrollGesture::default(); - - loop { - match rx.recv_timeout(WORKER_TICK) { - Ok(command) => { - // Any direct touch interrupts an in-flight scroll, or - // the two gestures would fight over the same finger. - if !matches!(command, InputCommand::Scroll { .. }) && scroll.active { - let _ = hid.touch_normalized(scroll.x, scroll.y, SysPhase::End); - scroll = ScrollGesture::default(); - } - - let result = match command { - InputCommand::Touch { phase, x, y, edge } => { - let phase = match phase { - TouchPhase::Begin => SysPhase::Begin, - TouchPhase::Move => SysPhase::Move, - TouchPhase::End => SysPhase::End, - }; - let edge = match edge { - TouchEdge::None => SysEdge::None, - TouchEdge::Left => SysEdge::Left, - TouchEdge::Top => SysEdge::Top, - TouchEdge::Bottom => SysEdge::Bottom, - TouchEdge::Right => SysEdge::Right, - }; - hid.touch_normalized_edge(x, y, phase, edge) - } - InputCommand::Button { button } => { - let button = match button { - HardwareButton::Home => SysButton::Home, - HardwareButton::Lock => SysButton::Lock, - HardwareButton::Siri => SysButton::Siri, - HardwareButton::SideButton => SysButton::SideButton, - HardwareButton::ApplePay => SysButton::ApplePay, - }; - hid.press_button(button, 0) - } - InputCommand::Key { - key_code, - ref modifiers, - } => hid.send_key_with_modifiers(key_code, modifiers), - InputCommand::Text { ref text } => type_text(&hid, text), - InputCommand::Rotate { orientation } => { - hid.set_orientation(match orientation { - Orientation::Portrait => SysOrientation::Portrait, - Orientation::PortraitUpsideDown => { - SysOrientation::PortraitUpsideDown - } - Orientation::LandscapeLeft => SysOrientation::LandscapeLeft, - Orientation::LandscapeRight => SysOrientation::LandscapeRight, - }) - } - InputCommand::Scroll { dx, dy, x, y } => { - apply_scroll(&hid, &mut scroll, dx, dy, x, y) - } - }; - - if let Err(error) = result { - tracing::warn!("input event failed: {error}"); - } - } - Err(RecvTimeoutError::Timeout) => { - // Lift the finger once the wheel has gone quiet, - // otherwise the page keeps inertial-scrolling. - if scroll.active - && scroll - .last_event - .is_some_and(|at| at.elapsed() >= SCROLL_IDLE) - { - let _ = hid.touch_normalized(scroll.x, scroll.y, SysPhase::End); - scroll = ScrollGesture::default(); - } - } - Err(RecvTimeoutError::Disconnected) => break, - } - } - - if scroll.active { - let _ = hid.touch_normalized(scroll.x, scroll.y, SysPhase::End); - } - })?; - - Ok(tx) -} - -/// Expand text into key presses and send them. -/// -/// Rejects the whole string if any character is untypeable, so a partial or -/// subtly wrong string is never entered. -#[cfg(target_os = "macos")] -fn type_text(hid: &accessibility_ios_sys::SimulatorHID, text: &str) -> Result<()> { - for stroke in crate::keymap::keystrokes_for(text)? { - hid.send_key_with_modifiers(stroke.usage, &stroke.modifiers())?; - } - Ok(()) -} - -#[cfg(target_os = "macos")] -fn apply_scroll( - hid: &accessibility_ios_sys::SimulatorHID, - scroll: &mut ScrollGesture, - dx: f64, - dy: f64, - x: f64, - y: f64, -) -> Result<()> { - use accessibility_ios_sys::TouchPhase as SysPhase; - - if !scroll.active { - // Plant the finger under the pointer so the gesture lands on whatever - // the user is actually hovering. - scroll.x = x.clamp(0.0, 1.0); - scroll.y = y.clamp(0.0, 1.0); - scroll.active = true; - hid.touch_normalized(scroll.x, scroll.y, SysPhase::Begin)?; - } - - // Content follows the finger, so the finger moves opposite the wheel. - scroll.x = (scroll.x - dx * SCROLL_GAIN).clamp(0.0, 1.0); - scroll.y = (scroll.y - dy * SCROLL_GAIN).clamp(0.0, 1.0); - scroll.last_event = Some(Instant::now()); - - if scroll.near_edge() { - // Out of room: lift and re-plant in the middle so the next delta has - // somewhere to go. - hid.touch_normalized(scroll.x, scroll.y, SysPhase::End)?; - scroll.x = 0.5; - scroll.y = 0.5; - hid.touch_normalized(scroll.x, scroll.y, SysPhase::Begin)?; - return Ok(()); - } - - hid.touch_normalized(scroll.x, scroll.y, SysPhase::Move) -} - -#[cfg(not(target_os = "macos"))] -pub fn spawn_input_worker(_udid: &str) -> Result> { - anyhow::bail!("Simulator input requires macOS") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn gesture_detects_each_edge() { - for (x, y) in [(0.01, 0.5), (0.99, 0.5), (0.5, 0.01), (0.5, 0.99)] { - let gesture = ScrollGesture { - active: true, - x, - y, - last_event: None, - }; - assert!( - gesture.near_edge(), - "expected ({x}, {y}) to be near an edge" - ); - } - } - - #[test] - fn gesture_center_is_not_near_edge() { - let gesture = ScrollGesture { - active: true, - x: 0.5, - y: 0.5, - last_event: None, - }; - assert!(!gesture.near_edge()); - } - - #[test] - fn landscape_classification() { - assert!(Orientation::LandscapeLeft.is_landscape()); - assert!(Orientation::LandscapeRight.is_landscape()); - assert!(!Orientation::Portrait.is_landscape()); - assert!(!Orientation::PortraitUpsideDown.is_landscape()); - } -} +pub use accessibility_core::platform::ios_simulator::input::*; diff --git a/packages/accessibility-serve/src/keymap.rs b/packages/accessibility-serve/src/keymap.rs index f3d7f6a..be9ffa8 100644 --- a/packages/accessibility-serve/src/keymap.rs +++ b/packages/accessibility-serve/src/keymap.rs @@ -1,236 +1,3 @@ -//! Mapping text to simulator key presses. -//! -//! The simulator's keyboard takes USB HID usage codes and has no notion of a -//! shift flag, so a capital letter or a shifted symbol is a *sequence*: hold -//! Left Shift, press the base key, release both. This module owns that -//! translation so the browser does not have to duplicate the table. -//! -//! Scope is deliberately US ASCII. There is no layout awareness and no -//! Unicode: a character outside the table is reported as an error rather than -//! silently dropped or turned into the wrong key, which is how the previous -//! implementation lost every `@` and quietly lowercased every capital. +//! Compatibility re-exports for simulator keyboard mapping. -use anyhow::{Result, anyhow}; - -/// USB HID keyboard usage codes, re-exported for the input layer. -pub mod usage { - pub const RETURN: u32 = 40; - pub const ESCAPE: u32 = 41; - pub const BACKSPACE: u32 = 42; - pub const TAB: u32 = 43; - pub const RIGHT_ARROW: u32 = 79; - pub const LEFT_ARROW: u32 = 80; - pub const DOWN_ARROW: u32 = 81; - pub const UP_ARROW: u32 = 82; - pub const LEFT_CONTROL: u32 = 224; - pub const LEFT_SHIFT: u32 = 225; - pub const LEFT_ALT: u32 = 226; - pub const LEFT_GUI: u32 = 227; -} - -/// A single key press: the usage code and whether Shift is held for it. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct KeyStroke { - pub usage: u32, - pub shift: bool, -} - -impl KeyStroke { - const fn plain(usage: u32) -> Self { - Self { - usage, - shift: false, - } - } - - const fn shifted(usage: u32) -> Self { - Self { usage, shift: true } - } - - /// The modifier usages to hold while pressing this key. - pub fn modifiers(&self) -> Vec { - if self.shift { - vec![usage::LEFT_SHIFT] - } else { - Vec::new() - } - } -} - -/// Symbols reachable without Shift, in USB HID usage order. -const UNSHIFTED_SYMBOLS: &[(char, u32)] = &[ - ('-', 45), - ('=', 46), - ('[', 47), - (']', 48), - ('\\', 49), - (';', 51), - ('\'', 52), - ('`', 53), - (',', 54), - ('.', 55), - ('/', 56), -]; - -/// Symbols produced by holding Shift over another key's usage. -const SHIFTED_SYMBOLS: &[(char, u32)] = &[ - ('!', 30), - ('@', 31), - ('#', 32), - ('$', 33), - ('%', 34), - ('^', 35), - ('&', 36), - ('*', 37), - ('(', 38), - (')', 39), - ('_', 45), - ('+', 46), - ('{', 47), - ('}', 48), - ('|', 49), - (':', 51), - ('"', 52), - ('~', 53), - ('<', 54), - ('>', 55), - ('?', 56), -]; - -/// The key press that produces `character`, if one exists on a US keyboard. -pub fn keystroke_for(character: char) -> Option { - match character { - // Letters share a usage; case is the Shift modifier. - 'a'..='z' => Some(KeyStroke::plain(character as u32 - 'a' as u32 + 4)), - 'A'..='Z' => Some(KeyStroke::shifted(character as u32 - 'A' as u32 + 4)), - // Digits are not contiguous with zero: 1-9 are 30-38 and 0 is 39. - '1'..='9' => Some(KeyStroke::plain(character as u32 - '1' as u32 + 30)), - '0' => Some(KeyStroke::plain(39)), - '\n' | '\r' => Some(KeyStroke::plain(usage::RETURN)), - '\t' => Some(KeyStroke::plain(usage::TAB)), - ' ' => Some(KeyStroke::plain(44)), - _ => UNSHIFTED_SYMBOLS - .iter() - .find(|(c, _)| *c == character) - .map(|(_, usage)| KeyStroke::plain(*usage)) - .or_else(|| { - SHIFTED_SYMBOLS - .iter() - .find(|(c, _)| *c == character) - .map(|(_, usage)| KeyStroke::shifted(*usage)) - }), - } -} - -/// Translate a string into key presses. -/// -/// Fails on the first unmappable character rather than typing a partial or -/// wrong string. -pub fn keystrokes_for(text: &str) -> Result> { - text.chars() - .map(|character| { - keystroke_for(character).ok_or_else(|| { - anyhow!( - "cannot type {character:?}: only US-ASCII characters are supported \ - (no unicode or emoji)" - ) - }) - }) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - - fn type_text(text: &str) -> Vec<(u32, bool)> { - keystrokes_for(text) - .expect("typeable") - .into_iter() - .map(|k| (k.usage, k.shift)) - .collect() - } - - #[test] - fn letters_use_usb_hid_usages_not_hitoolbox() { - // The bug this module exists to fix: 'a' is usage 4, not HIToolbox 0. - assert_eq!(type_text("a"), vec![(4, false)]); - assert_eq!(type_text("b"), vec![(5, false)]); - assert_eq!(type_text("z"), vec![(29, false)]); - } - - #[test] - fn capitals_hold_shift_over_the_same_usage() { - assert_eq!(type_text("aA"), vec![(4, false), (4, true)]); - } - - #[test] - fn digits_are_not_contiguous_with_zero() { - assert_eq!(type_text("1"), vec![(30, false)]); - assert_eq!(type_text("9"), vec![(38, false)]); - assert_eq!(type_text("0"), vec![(39, false)]); - } - - #[test] - fn types_an_email_address() { - // The exact string the old implementation mangled into - // "testexample.com": @ and ? dropped, capitals lowercased. - let strokes = type_text("Test@Example.com?"); - assert_eq!(strokes.len(), 17, "every character must produce a key"); - assert_eq!(strokes[0], (23, true), "T is shifted t"); - assert_eq!(strokes[4], (31, true), "@ is shift-2"); - assert_eq!(strokes[16], (56, true), "? is shift-/"); - } - - #[test] - fn shifted_and_unshifted_symbols_share_usages() { - assert_eq!(type_text(";"), vec![(51, false)]); - assert_eq!(type_text(":"), vec![(51, true)]); - assert_eq!(type_text("/"), vec![(56, false)]); - assert_eq!(type_text("?"), vec![(56, true)]); - } - - #[test] - fn whitespace_maps_to_real_keys() { - assert_eq!(type_text(" "), vec![(44, false)]); - assert_eq!(type_text("\n"), vec![(usage::RETURN, false)]); - assert_eq!(type_text("\t"), vec![(usage::TAB, false)]); - } - - #[test] - fn unmappable_characters_fail_loudly() { - // Better a clear error than a silently wrong string. - for text in ["café", "hello 👋", "→"] { - let error = keystrokes_for(text).expect_err("should reject"); - assert!(error.to_string().contains("cannot type"), "{error}"); - } - } - - #[test] - fn failure_is_reported_before_anything_is_typed() { - // keystrokes_for collects into a Result, so a bad character partway - // through yields no partial output for the caller to send. - assert!(keystrokes_for("ok\u{1F600}bad").is_err()); - } - - #[test] - fn shift_is_the_only_modifier_for_text() { - assert_eq!( - keystroke_for('A').expect("mapped").modifiers(), - vec![usage::LEFT_SHIFT] - ); - assert!(keystroke_for('a').expect("mapped").modifiers().is_empty()); - } - - #[test] - fn every_printable_ascii_character_is_typeable() { - // If this ever regresses, some character silently becomes untypeable. - for byte in 0x20u8..0x7F { - let character = byte as char; - assert!( - keystroke_for(character).is_some(), - "no key for {character:?}" - ); - } - } -} +pub use accessibility_core::platform::ios_simulator::keymap::*; diff --git a/packages/accessibility-serve/src/lib.rs b/packages/accessibility-serve/src/lib.rs index f87f28c..dbc21a9 100644 --- a/packages/accessibility-serve/src/lib.rs +++ b/packages/accessibility-serve/src/lib.rs @@ -6,23 +6,35 @@ //! accessibility tree is exposed so the UI can inspect elements. pub mod avcc; +#[cfg(target_os = "macos")] pub mod ax; +#[cfg(target_os = "macos")] pub mod coverage; +#[cfg(target_os = "macos")] pub mod http; +#[cfg(target_os = "macos")] pub mod input; +#[cfg(target_os = "macos")] pub mod keymap; +#[cfg(target_os = "macos")] pub mod session; +#[cfg(target_os = "macos")] pub mod settings; +#[cfg(target_os = "macos")] pub mod webrtc_stream; use std::net::SocketAddr; +#[cfg(target_os = "macos")] use std::sync::Arc; -use anyhow::{Context, Result}; +#[cfg(target_os = "macos")] +use anyhow::Context; +use anyhow::Result; use accessibility_core::video::VideoConfig; -pub use session::SimSession; +#[cfg(target_os = "macos")] +pub use accessibility_core::platform::ios_simulator::SimSession; /// Which transport the web UI should try first. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] @@ -81,6 +93,7 @@ impl Default for ServeConfig { } /// Start capturing and serve until the process is interrupted. +#[cfg(target_os = "macos")] pub async fn serve(config: ServeConfig) -> Result<()> { let session = SimSession::start(config.udid.as_deref(), config.video) .context("failed to start simulator capture")?; @@ -113,3 +126,9 @@ pub async fn serve(config: ServeConfig) -> Result<()> { .await .context("server error") } + +/// iOS Simulator serving is only available on macOS. +#[cfg(not(target_os = "macos"))] +pub async fn serve(_config: ServeConfig) -> Result<()> { + anyhow::bail!("Serving an iOS Simulator requires macOS") +} diff --git a/packages/accessibility-serve/src/session.rs b/packages/accessibility-serve/src/session.rs index 43ca4af..5c62903 100644 --- a/packages/accessibility-serve/src/session.rs +++ b/packages/accessibility-serve/src/session.rs @@ -1,353 +1,3 @@ -//! Ownership of the per-device simulator resources. -//! -//! The simulator's Objective-C objects are not `Sync` and their calls block, so -//! each one lives on a dedicated thread behind a command channel rather than in -//! a mutex on the async runtime. -//! -//! Input and accessibility get *separate* threads on purpose: an accessibility -//! tree fetch can take hundreds of milliseconds, and pointer events queued -//! behind one would make the stream feel broken. +//! Compatibility re-exports for reusable iOS Simulator sessions. -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Instant; - -use anyhow::{Result, anyhow}; -use tokio::sync::{broadcast, mpsc, oneshot}; - -use accessibility_core::video::{ - EncodedFrame, FrameKind, Recording, RecordingConfig, VideoCapture, VideoConfig, -}; - -use crate::ax::{AxCommand, AxSnapshot, ElementDetail, spawn_ax_worker}; -use crate::input::{InputCommand, Orientation, spawn_input_worker}; -use crate::settings::{Setting, SettingKey}; - -/// How many encoded frames to buffer per subscriber. -/// -/// Small on purpose: for interactive video a dropped frame is better than a -/// late one, and a slow client should not be able to inflate memory. -const FRAME_BUFFER: usize = 16; - -/// Counters for diagnosing stream quality and pacing. -/// -/// Cheap enough to always collect: the interesting failures here (keyframe -/// storms, subscribers falling behind) are invisible without them and only -/// show up under load, which is exactly when you cannot attach a profiler. -#[derive(Default)] -pub struct StreamStats { - pub frames: AtomicU64, - pub keyframes: AtomicU64, - pub bytes: AtomicU64, - /// Keyframes asked for by a new subscriber, an RTCP PLI, or a lagging - /// receiver. A high rate here starves the stream of bitrate for delta - /// frames and is self-reinforcing. - pub keyframe_requests: AtomicU64, - /// Times a subscriber fell far enough behind to drop frames. - pub lag_events: AtomicU64, -} - -/// A snapshot of [`StreamStats`] with rates worked out. -#[derive(Debug, Clone, serde::Serialize)] -pub struct StatsReport { - pub uptime_secs: f64, - pub frames: u64, - pub keyframes: u64, - pub bytes: u64, - pub fps: f64, - pub mbps: f64, - pub bits_per_pixel: f64, - pub mean_frame_kb: f64, - pub keyframe_requests: u64, - pub lag_events: u64, - pub subscribers: usize, - /// Frames written to the current recording, or `None` when idle. - pub recording_frames: Option, - /// Capture resolution. - pub width: u32, - pub height: u32, - /// Resolution actually encoded, after downscaling. - pub encoded_width: u32, - pub encoded_height: u32, -} - -/// Geometry and identity of the device being served. -#[derive(Debug, Clone, serde::Serialize)] -pub struct DeviceInfo { - pub udid: String, - /// Raw framebuffer width. Does not change with orientation. - pub width: u32, - /// Raw framebuffer height. Does not change with orientation. - pub height: u32, - pub orientation: Orientation, -} - -pub struct SimSession { - device_udid: String, - capture: Box, - frames: broadcast::Sender, - /// Most recent parameter set, replayed to clients that join mid-stream. - latest_parameter_set: Arc>>, - stats: Arc, - started: Instant, - input: std::sync::mpsc::Sender, - ax: mpsc::UnboundedSender, - /// Last orientation we asked for. - /// - /// The framebuffer is always portrait-native — rotating the device - /// rotates the *content* inside a fixed-size surface — so orientation - /// cannot be recovered from the video and has to be tracked here. - orientation: std::sync::Mutex, -} - -impl SimSession { - /// Attach to a booted simulator and start capturing. - pub fn start(udid: Option<&str>, config: VideoConfig) -> Result> { - let (frames, _) = broadcast::channel(FRAME_BUFFER); - let latest_parameter_set = Arc::new(std::sync::Mutex::new(None)); - let stats = Arc::new(StreamStats::default()); - - let sink = { - let frames = frames.clone(); - let latest_parameter_set = Arc::clone(&latest_parameter_set); - let stats = Arc::clone(&stats); - Arc::new(move |frame: EncodedFrame| { - if frame.kind == FrameKind::ParameterSet { - *latest_parameter_set.lock().unwrap() = Some(frame.clone()); - } - stats.frames.fetch_add(1, Ordering::Relaxed); - stats - .bytes - .fetch_add(frame.data.len() as u64, Ordering::Relaxed); - if frame.kind == FrameKind::Keyframe { - stats.keyframes.fetch_add(1, Ordering::Relaxed); - } - // A send error just means nobody is watching yet. - let _ = frames.send(frame); - }) - }; - - let (capture, resolved_udid) = start_capture(udid, &config, sink)?; - let input = spawn_input_worker(&resolved_udid)?; - let ax = spawn_ax_worker(&resolved_udid)?; - - Ok(Arc::new(Self { - device_udid: resolved_udid, - capture, - frames, - latest_parameter_set, - stats, - started: Instant::now(), - input, - ax, - orientation: std::sync::Mutex::new(Orientation::Portrait), - })) - } - - pub fn subscribe(&self) -> broadcast::Receiver { - let receiver = self.frames.subscribe(); - #[allow(clippy::let_and_return)] - // A new subscriber cannot decode anything until the next keyframe, so - // ask for one immediately instead of making them wait out the interval. - self.capture.request_keyframe(); - receiver - } - - pub fn latest_parameter_set(&self) -> Option { - self.latest_parameter_set.lock().unwrap().clone() - } - - pub fn request_keyframe(&self) { - self.stats.keyframe_requests.fetch_add(1, Ordering::Relaxed); - self.capture.request_keyframe(); - } - - pub fn note_lag(&self) { - self.stats.lag_events.fetch_add(1, Ordering::Relaxed); - } - - pub fn stats(&self) -> StatsReport { - let elapsed = self.started.elapsed().as_secs_f64().max(1e-6); - let frames = self.stats.frames.load(Ordering::Relaxed); - let bytes = self.stats.bytes.load(Ordering::Relaxed); - let geometry = self.capture.geometry(); - let encoded = self.capture.encoded_geometry(); - // Bits per pixel only means anything against the encoded size. - let pixels = (encoded.width as f64) * (encoded.height as f64); - let fps = frames as f64 / elapsed; - - StatsReport { - uptime_secs: (elapsed * 10.0).round() / 10.0, - frames, - keyframes: self.stats.keyframes.load(Ordering::Relaxed), - bytes, - fps: (fps * 10.0).round() / 10.0, - mbps: ((bytes as f64 * 8.0 / elapsed / 1e6) * 100.0).round() / 100.0, - // The headline number: anything much under 0.1 will visibly - // block up on motion. - bits_per_pixel: if pixels > 0.0 && fps > 0.0 { - ((bytes as f64 * 8.0 / elapsed) / (pixels * fps) * 10000.0).round() / 10000.0 - } else { - 0.0 - }, - mean_frame_kb: if frames > 0 { - ((bytes as f64 / frames as f64 / 1024.0) * 100.0).round() / 100.0 - } else { - 0.0 - }, - keyframe_requests: self.stats.keyframe_requests.load(Ordering::Relaxed), - lag_events: self.stats.lag_events.load(Ordering::Relaxed), - subscribers: self.frames.receiver_count(), - recording_frames: self.capture.recording_frames(), - width: geometry.width, - height: geometry.height, - encoded_width: encoded.width, - encoded_height: encoded.height, - } - } - - pub fn device_info(&self) -> DeviceInfo { - let geometry = self.capture.geometry(); - DeviceInfo { - udid: self.device_udid.clone(), - width: geometry.width, - height: geometry.height, - orientation: self.orientation(), - } - } - - pub fn orientation(&self) -> Orientation { - *self.orientation.lock().unwrap() - } - - /// Rotate the device and remember the new orientation. - pub fn set_orientation(&self, orientation: Orientation) { - *self.orientation.lock().unwrap() = orientation; - self.send_input(InputCommand::Rotate { orientation }); - } - - /// Begin recording to a file beside the system temp directory. - /// - /// Runs a second encode of the same frames, so the live stream is - /// unaffected and the recording can use B-frames and its own resolution. - pub fn start_recording(&self, config: RecordingConfig) -> Result { - let path = std::env::temp_dir().join(format!( - "serve-sim-{}-{}.mp4", - self.device_udid, - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or_default() - )); - self.capture.start_recording(&path, &config)?; - Ok(path) - } - - pub fn stop_recording(&self) -> Result { - self.capture.stop_recording() - } - - pub fn recording_frames(&self) -> Option { - self.capture.recording_frames() - } - - pub fn settings(&self) -> Vec { - crate::settings::read_all(&self.device_udid) - } - - pub fn set_setting(&self, key: SettingKey, value: &str) -> Result { - crate::settings::write(&self.device_udid, key, value) - } - - /// Queue an input event. Fire-and-forget: pointer events must never block - /// the socket reader. - pub fn send_input(&self, command: InputCommand) { - if let InputCommand::Rotate { orientation } = command { - *self.orientation.lock().unwrap() = orientation; - } - let _ = self.input.send(command); - } - - /// Read the accessibility tree. - /// - /// `scan` additionally hit-tests the regions the tree walk cannot explain, - /// which is the only way to reach `WKWebView` and Safari content. It costs - /// a few hundred milliseconds, so it is opt-in. - pub async fn ax_snapshot(&self, scan: bool) -> Result { - let (tx, rx) = oneshot::channel(); - self.ax - .send(AxCommand::Snapshot { scan, reply: tx }) - .map_err(|_| anyhow!("accessibility worker stopped"))?; - let snapshot = rx - .await - .map_err(|_| anyhow!("accessibility worker stopped"))??; - self.reconcile_orientation(snapshot.is_landscape); - Ok(snapshot) - } - - /// Correct the tracked orientation against what the device reports. - /// - /// Orientation can change without us: the user can rotate from the - /// Simulator menu, an app can force an orientation, or the server can be - /// restarted while the device is already sideways. Accessibility bounds - /// are the only cheap signal, and they only reveal landscape vs portrait, - /// so a disagreement resolves to a sensible default of the right kind - /// rather than to an exact rotation. - fn reconcile_orientation(&self, is_landscape: bool) { - let mut orientation = self.orientation.lock().unwrap(); - if orientation.is_landscape() == is_landscape { - return; - } - *orientation = if is_landscape { - Orientation::LandscapeLeft - } else { - Orientation::Portrait - }; - } - - /// Best-effort orientation seed at startup. - /// - /// Without this the server would assume portrait and render a sideways - /// device whenever it attaches to an already-rotated simulator. - pub async fn seed_orientation(&self) { - if let Ok(snapshot) = self.ax_snapshot(false).await { - self.reconcile_orientation(snapshot.is_landscape); - } - } - - pub async fn ax_hit_test(&self, x: f64, y: f64) -> Result> { - let (tx, rx) = oneshot::channel(); - self.ax - .send(AxCommand::HitTest { x, y, reply: tx }) - .map_err(|_| anyhow!("accessibility worker stopped"))?; - rx.await - .map_err(|_| anyhow!("accessibility worker stopped"))? - } -} - -#[cfg(target_os = "macos")] -fn start_capture( - udid: Option<&str>, - config: &VideoConfig, - sink: accessibility_core::video::FrameSink, -) -> Result<(Box, String)> { - use accessibility_core::platform::ios_simulator::SimulatorVideoCapture; - - // Resolve the concrete UDID up front so the input and accessibility - // workers bind to the same device the video came from, even when the - // caller passed `None`. - let resolved = accessibility_ios_sys::SimFramebuffer::new(udid)? - .device_udid() - .to_string(); - let capture = SimulatorVideoCapture::start(&resolved, config, sink)?; - Ok((Box::new(capture), resolved)) -} - -#[cfg(not(target_os = "macos"))] -fn start_capture( - _udid: Option<&str>, - _config: &VideoConfig, - _sink: accessibility_core::video::FrameSink, -) -> Result<(Box, String)> { - anyhow::bail!("Serving an iOS Simulator requires macOS") -} +pub use accessibility_core::platform::ios_simulator::session::*; diff --git a/packages/accessibility-serve/src/settings.rs b/packages/accessibility-serve/src/settings.rs index a3bcd47..7e5f221 100644 --- a/packages/accessibility-serve/src/settings.rs +++ b/packages/accessibility-serve/src/settings.rs @@ -1,173 +1,3 @@ -//! Simulator-wide UI settings. -//! -//! These go through `simctl ui`, which is the same mechanism Xcode's Devices -//! window uses. Only the three options simctl actually implements are exposed: -//! appearance, increase contrast, and content size. -//! -//! The Devices window also offers reduce-motion, colour filters, transparency -//! and VoiceOver, but simctl has no verb for those — they require a helper -//! binary spawned *inside* the simulator that drives the private -//! libAccessibility setters. That is a meaningfully larger piece of work and -//! is deliberately not attempted here. +//! Compatibility re-exports for iOS Simulator settings. -use anyhow::{Context, Result, anyhow}; -use serde::{Deserialize, Serialize}; - -/// Content size categories, smallest to largest. -/// -/// The five `accessibility-*` entries are the extended range that only appears -/// once a user opts into larger accessibility text. -pub const CONTENT_SIZES: &[&str] = &[ - "extra-small", - "small", - "medium", - "large", - "extra-large", - "extra-extra-large", - "extra-extra-extra-large", - "accessibility-medium", - "accessibility-large", - "accessibility-extra-large", - "accessibility-extra-extra-large", - "accessibility-extra-extra-extra-large", -]; - -pub const APPEARANCES: &[&str] = &["light", "dark"]; -pub const TOGGLE: &[&str] = &["enabled", "disabled"]; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum SettingKey { - Appearance, - IncreaseContrast, - ContentSize, -} - -impl SettingKey { - /// The `simctl ui` subcommand for this setting. - fn verb(self) -> &'static str { - match self { - SettingKey::Appearance => "appearance", - SettingKey::IncreaseContrast => "increase_contrast", - SettingKey::ContentSize => "content_size", - } - } - - pub fn allowed_values(self) -> &'static [&'static str] { - match self { - SettingKey::Appearance => APPEARANCES, - SettingKey::IncreaseContrast => TOGGLE, - SettingKey::ContentSize => CONTENT_SIZES, - } - } - - pub fn all() -> [SettingKey; 3] { - [ - SettingKey::Appearance, - SettingKey::IncreaseContrast, - SettingKey::ContentSize, - ] - } -} - -/// One setting and its current value, as reported by the simulator. -#[derive(Debug, Clone, Serialize)] -pub struct Setting { - pub key: SettingKey, - /// Current value, or `unsupported`/`unknown` if the runtime says so. - pub value: String, - pub allowed: &'static [&'static str], -} - -/// Read every supported setting from the device. -pub fn read_all(udid: &str) -> Vec { - SettingKey::all() - .into_iter() - .map(|key| Setting { - key, - // A failed read is reported as unknown rather than failing the - // whole request; one unsupported option should not blank the UI. - value: read(udid, key).unwrap_or_else(|_| "unknown".to_string()), - allowed: key.allowed_values(), - }) - .collect() -} - -pub fn read(udid: &str, key: SettingKey) -> Result { - let output = simctl(&["ui", udid, key.verb()])?; - Ok(output.trim().to_string()) -} - -pub fn write(udid: &str, key: SettingKey, value: &str) -> Result { - // `content_size` also accepts increment/decrement, which are not in the - // reported value set but are the ergonomic way to drive it from a UI. - let stepping = - matches!(key, SettingKey::ContentSize) && matches!(value, "increment" | "decrement"); - - if !stepping && !key.allowed_values().contains(&value) { - return Err(anyhow!( - "'{value}' is not valid for {:?}; expected one of {}", - key, - key.allowed_values().join(", ") - )); - } - - simctl(&["ui", udid, key.verb(), value])?; - read(udid, key) -} - -fn simctl(args: &[&str]) -> Result { - let output = std::process::Command::new("xcrun") - .arg("simctl") - .args(args) - .output() - .context("failed to run xcrun simctl")?; - - if !output.status.success() { - return Err(anyhow!( - "simctl {} failed: {}", - args.join(" "), - String::from_utf8_lossy(&output.stderr).trim() - )); - } - Ok(String::from_utf8_lossy(&output.stdout).to_string()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn rejects_values_outside_the_allowed_set() { - let error = write("no-such-device", SettingKey::Appearance, "chartreuse") - .expect_err("invalid appearance should be rejected"); - // Rejected locally, without ever shelling out to simctl. - assert!(error.to_string().contains("chartreuse")); - } - - #[test] - fn content_size_accepts_stepping_verbs() { - // These are not reported values, so they must be allowed explicitly. - assert!(!CONTENT_SIZES.contains(&"increment")); - for value in ["increment", "decrement"] { - let error = write("no-such-device", SettingKey::ContentSize, value) - .expect_err("no such device"); - assert!( - !error.to_string().contains("is not valid"), - "{value} should reach simctl rather than being rejected" - ); - } - } - - #[test] - fn every_key_has_values_and_a_distinct_verb() { - let mut verbs = Vec::new(); - for key in SettingKey::all() { - assert!(!key.allowed_values().is_empty()); - verbs.push(key.verb()); - } - verbs.sort_unstable(); - verbs.dedup(); - assert_eq!(verbs.len(), SettingKey::all().len()); - } -} +pub use accessibility_core::platform::ios_simulator::settings::*;