From d8498cc3824319a863b7da0637619f69dc5044e6 Mon Sep 17 00:00:00 2001 From: blindfs Date: Wed, 6 May 2026 17:03:08 +0800 Subject: [PATCH 1/4] fix: prioritize AXPopover over focused_window() --- src/app_executor.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/app_executor.rs b/src/app_executor.rs index 5b9402c..3caa3c4 100644 --- a/src/app_executor.rs +++ b/src/app_executor.rs @@ -15,7 +15,7 @@ use crate::{ }; use accessibility::{AXUIElement, AXUIElementActions, AXUIElementAttributes}; use accessibility_sys::{ - kAXErrorAttributeUnsupported, kAXErrorCannotComplete, kAXFocusedAttribute, + kAXErrorAttributeUnsupported, kAXErrorCannotComplete, kAXFocusedAttribute, kAXPopoverRole, }; use core_foundation::{base::TCFType, boolean::CFBoolean, number::CFNumber, string::CFString}; use log::Level; @@ -473,7 +473,19 @@ impl AppExecutor { let (focused_window, window_frame) = if vis_level == VisibilityCheckingLevel::Loosest { (focused_app, screen_frame) } else { - let window = focused_app.focused_window().unwrap_or(focused_app); + let mut window = focused_app.focused_window(); + // NOTE: prioritize popover windows, e.g. Apple Music search + if let Ok(windows) = focused_app.windows() + && windows.len() > 1 + { + for win in windows.iter() { + if win.role().is_ok_and(|r| r == kAXPopoverRole) { + window = Ok(win.clone()); + break; + } + } + } + let window = window.unwrap_or(focused_app); let frame = window .get_frame() .and_then(|f| f.intersect(&screen_frame)) From 0000d640fca451cc11ec8644926b6cf35db9ce8b Mon Sep 17 00:00:00 2001 From: blindfs Date: Wed, 6 May 2026 20:09:33 +0800 Subject: [PATCH 2/4] fix: prioritize system alarm windows --- src/app_executor.rs | 32 ++++++++++++++++++++++---------- src/ax_element.rs | 13 ++++++++----- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/app_executor.rs b/src/app_executor.rs index 3caa3c4..1cc08dd 100644 --- a/src/app_executor.rs +++ b/src/app_executor.rs @@ -455,11 +455,29 @@ impl AppExecutor { } fn select_app_window(&mut self, vis_level: VisibilityCheckingLevel) -> Option { + let screen_frame = Frame::from_origion(self.screen_size); + + // NOTE: prioritize system alarms + if let Ok(app) = AXUIElement::application_with_bundle("com.apple.coreservices.uiagent") + && let Ok(window) = app.focused_window() + { + let frame = window.get_frame(screen_frame); + self.last_window_frame = frame; + self.is_electron = false; + + self.selected = Some(ElementOfInterest::new( + Some(window), + None, + RoleOfInterest::Generic, + frame, + )); + + return Some(frame); + } + let (pid, is_electron) = get_focused_pid()?; self.is_electron = is_electron; - let focused_app = AXUIElement::application(pid); - let screen_frame = Frame::from_origion(self.screen_size); // HACK: need this to bootstrap UI tree generation for some electron apps, // e.g. Discord @@ -486,10 +504,7 @@ impl AppExecutor { } } let window = window.unwrap_or(focused_app); - let frame = window - .get_frame() - .and_then(|f| f.intersect(&screen_frame)) - .unwrap_or(screen_frame); + let frame = window.get_frame(screen_frame); (window, frame) }; self.last_window_frame = window_frame; @@ -1083,10 +1098,7 @@ impl AppExecutor { .and_then(|ele| ele.parent().ok()) { let screen_frame = Frame::from_origion(self.screen_size); - let frame = parent_element - .get_frame() - .and_then(|f| f.intersect(&screen_frame)) - .unwrap_or(screen_frame); + let frame = parent_element.get_frame(screen_frame); self.selected = Some(ElementOfInterest { element: Some(parent_element), context: None, diff --git a/src/ax_element.rs b/src/ax_element.rs index ddb47dc..8d0dff7 100644 --- a/src/ax_element.rs +++ b/src/ax_element.rs @@ -340,7 +340,7 @@ pub trait GetAttribute { fn get_attribute(&self, attribute_name: &str) -> Option; fn get_attribute_string(&self, attribute_name: &str) -> Option; fn get_string_value_or_description(&self) -> Option; - fn get_frame(&self) -> Option; + fn get_frame(&self, default: Frame) -> Frame; fn get_dom_classes(&self) -> Option>; fn inspect(&self) -> String; fn is_clickable(&self) -> bool; @@ -368,7 +368,7 @@ impl GetAttribute for AXUIElement { .map(|cf| cf.to_string()) } - fn get_frame(&self) -> Option { + fn get_frame(&self, default_frame: Frame) -> Frame { let cf_array_in = CFArray::from_CFTypes(&[ CFString::new(kAXPositionAttribute), CFString::new(kAXSizeAttribute), @@ -385,7 +385,7 @@ impl GetAttribute for AXUIElement { ) }; - if err != kAXErrorSuccess || values_ref.is_null() { + let frame = if err != kAXErrorSuccess || values_ref.is_null() { None } else { let values_array: CFArray = @@ -401,10 +401,13 @@ impl GetAttribute for AXUIElement { .and_then(|size_ptr| cftype_to_rust_type::(*size_ptr, kAXValueTypeCGSize)); match (pos, size) { - (Some(p), Some(s)) => Some(Frame::new(p.x, p.y, p.x + s.width, p.y + s.height)), + (Some(p), Some(s)) => { + Frame::new(p.x, p.y, p.x + s.width, p.y + s.height).intersect(&default_frame) + } _ => None, } - } + }; + frame.unwrap_or(default_frame) } fn inspect(&self) -> String { From 785a7d76aab1e402978481e956b2146d889d2a7f Mon Sep 17 00:00:00 2001 From: blindfs Date: Thu, 7 May 2026 10:01:46 +0800 Subject: [PATCH 3/4] fix: false negative of full-width scroll-off-y, e.g. Brave google search --- src/ax_element.rs | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/ax_element.rs b/src/ax_element.rs index 8d0dff7..4f6bc01 100644 --- a/src/ax_element.rs +++ b/src/ax_element.rs @@ -120,9 +120,13 @@ impl ElementBasicAttributes { { return false; } - self.role - .to_lowercase() - .contains(&target.role.to_lowercase()) + let role = self.role.to_lowercase(); + for r in target.role.to_lowercase().split('|') { + if role.contains(r) { + return true; + } + } + false } } @@ -411,11 +415,13 @@ impl GetAttribute for AXUIElement { } fn inspect(&self) -> String { + let Some(fp) = ElementBasicAttributes::from(self) else { + return "Unknown".into(); + }; + let mut msg = String::new(); - if let Ok(r) = self.role() { - msg.push_str(&format!("Role: {}\n", r)); - } + msg.push_str(&format!("Role: {}\n", fp.role)); if let Ok(t) = self.title() { msg.push_str(&format!("title: {}\n", t)); @@ -433,6 +439,13 @@ impl GetAttribute for AXUIElement { msg.push_str(&format!("value: {:?}\n", v)); } + if let Some(f) = fp.frame { + let CGPoint { x, y } = f.top_left; + msg.push_str(&format!("pos: x: {x}, y: {y}\n")); + let (w, h) = f.size(); + msg.push_str(&format!("size: width: {w}, height: {h}\n")); + } + msg // for attr in &self.attribute_names().unwrap() { // println!( @@ -575,12 +588,13 @@ pub fn traverse_elements( }; // WARN: Performance critical! Exclude electron elements scrolled off y axis, - // but should avoid false negatives of ancestors for some menu items, - // e.g. (Discord right click menu) if ele_fp.frame.is_some_and(|f| { - let (_, h) = f.size(); + let (w, h) = f.size(); (h == 0.0 && f.bottom_right.y == window_frame.bottom_right.y) - || (h == 1.0 && f.top_left.y == window_frame.top_left.y) + // NOTE: keep full width elements, e.g. Brave google search + || (h == 1.0 && f.top_left.y == window_frame.top_left.y && w != window_frame.size().0) + // NOTE: should avoid false negatives of ancestors for some menu items, + // e.g. (Discord right click menu) }) && vis_level != VisibilityCheckingLevel::Loosest { return; From d4eba9af92897137d695fe0c5a152f17669ef024 Mon Sep 17 00:00:00 2001 From: blindfs Date: Thu, 7 May 2026 10:19:36 +0800 Subject: [PATCH 4/4] fix: electron right click menu initial wait --- src/app_executor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app_executor.rs b/src/app_executor.rs index 1cc08dd..2ed4778 100644 --- a/src/app_executor.rs +++ b/src/app_executor.rs @@ -481,7 +481,7 @@ impl AppExecutor { // HACK: need this to bootstrap UI tree generation for some electron apps, // e.g. Discord - if is_electron && pid != self.last_pid { + if is_electron && (pid != self.last_pid || vis_level == VisibilityCheckingLevel::Loosest) { let _ = focused_app.role(); std::thread::sleep(Duration::from_millis(self.config.electron_initial_wait_ms)); }