diff --git a/src/app_executor.rs b/src/app_executor.rs
index 5b9402c..2ed4778 100644
--- a/src/app_executor.rs
+++ b/src/app_executor.rs
@@ -15,7 +15,7 @@ use crate::{
};
use accessibility::{AXUIElement, AXUIElementActions, AXUIElementAttributes};
use accessibility_sys::{
- kAXErrorAttributeUnsupported, kAXErrorCannotComplete, kAXFocusedAttribute,
+ kAXErrorAttributeUnsupported, kAXErrorCannotComplete, kAXFocusedAttribute, kAXPopoverRole,
};
use core_foundation::{base::TCFType, boolean::CFBoolean, number::CFNumber, string::CFString};
use log::Level;
@@ -455,15 +455,33 @@ impl AppExecutor {
}
fn select_app_window(&mut self, vis_level: VisibilityCheckingLevel) -> Option {
+ let screen_frame = Frame::from_origion(self.screen_size);
+
+ // NOTE: prioritize system alarms
+ if let Ok(app) = AXUIElement::application_with_bundle("com.apple.coreservices.uiagent")
+ && let Ok(window) = app.focused_window()
+ {
+ let frame = window.get_frame(screen_frame);
+ self.last_window_frame = frame;
+ self.is_electron = false;
+
+ self.selected = Some(ElementOfInterest::new(
+ Some(window),
+ None,
+ RoleOfInterest::Generic,
+ frame,
+ ));
+
+ return Some(frame);
+ }
+
let (pid, is_electron) = get_focused_pid()?;
self.is_electron = is_electron;
-
let focused_app = AXUIElement::application(pid);
- let screen_frame = Frame::from_origion(self.screen_size);
// HACK: need this to bootstrap UI tree generation for some electron apps,
// e.g. Discord
- if is_electron && pid != self.last_pid {
+ if is_electron && (pid != self.last_pid || vis_level == VisibilityCheckingLevel::Loosest) {
let _ = focused_app.role();
std::thread::sleep(Duration::from_millis(self.config.electron_initial_wait_ms));
}
@@ -473,11 +491,20 @@ impl AppExecutor {
let (focused_window, window_frame) = if vis_level == VisibilityCheckingLevel::Loosest {
(focused_app, screen_frame)
} else {
- let window = focused_app.focused_window().unwrap_or(focused_app);
- let frame = window
- .get_frame()
- .and_then(|f| f.intersect(&screen_frame))
- .unwrap_or(screen_frame);
+ let mut window = focused_app.focused_window();
+ // NOTE: prioritize popover windows, e.g. Apple Music search
+ if let Ok(windows) = focused_app.windows()
+ && windows.len() > 1
+ {
+ for win in windows.iter() {
+ if win.role().is_ok_and(|r| r == kAXPopoverRole) {
+ window = Ok(win.clone());
+ break;
+ }
+ }
+ }
+ let window = window.unwrap_or(focused_app);
+ let frame = window.get_frame(screen_frame);
(window, frame)
};
self.last_window_frame = window_frame;
@@ -1071,10 +1098,7 @@ impl AppExecutor {
.and_then(|ele| ele.parent().ok())
{
let screen_frame = Frame::from_origion(self.screen_size);
- let frame = parent_element
- .get_frame()
- .and_then(|f| f.intersect(&screen_frame))
- .unwrap_or(screen_frame);
+ let frame = parent_element.get_frame(screen_frame);
self.selected = Some(ElementOfInterest {
element: Some(parent_element),
context: None,
diff --git a/src/ax_element.rs b/src/ax_element.rs
index ddb47dc..4f6bc01 100644
--- a/src/ax_element.rs
+++ b/src/ax_element.rs
@@ -120,9 +120,13 @@ impl ElementBasicAttributes {
{
return false;
}
- self.role
- .to_lowercase()
- .contains(&target.role.to_lowercase())
+ let role = self.role.to_lowercase();
+ for r in target.role.to_lowercase().split('|') {
+ if role.contains(r) {
+ return true;
+ }
+ }
+ false
}
}
@@ -340,7 +344,7 @@ pub trait GetAttribute {
fn get_attribute(&self, attribute_name: &str) -> Option;
fn get_attribute_string(&self, attribute_name: &str) -> Option;
fn get_string_value_or_description(&self) -> Option;
- fn get_frame(&self) -> Option;
+ fn get_frame(&self, default: Frame) -> Frame;
fn get_dom_classes(&self) -> Option>;
fn inspect(&self) -> String;
fn is_clickable(&self) -> bool;
@@ -368,7 +372,7 @@ impl GetAttribute for AXUIElement {
.map(|cf| cf.to_string())
}
- fn get_frame(&self) -> Option {
+ fn get_frame(&self, default_frame: Frame) -> Frame {
let cf_array_in = CFArray::from_CFTypes(&[
CFString::new(kAXPositionAttribute),
CFString::new(kAXSizeAttribute),
@@ -385,7 +389,7 @@ impl GetAttribute for AXUIElement {
)
};
- if err != kAXErrorSuccess || values_ref.is_null() {
+ let frame = if err != kAXErrorSuccess || values_ref.is_null() {
None
} else {
let values_array: CFArray =
@@ -401,18 +405,23 @@ impl GetAttribute for AXUIElement {
.and_then(|size_ptr| cftype_to_rust_type::(*size_ptr, kAXValueTypeCGSize));
match (pos, size) {
- (Some(p), Some(s)) => Some(Frame::new(p.x, p.y, p.x + s.width, p.y + s.height)),
+ (Some(p), Some(s)) => {
+ Frame::new(p.x, p.y, p.x + s.width, p.y + s.height).intersect(&default_frame)
+ }
_ => None,
}
- }
+ };
+ frame.unwrap_or(default_frame)
}
fn inspect(&self) -> String {
+ let Some(fp) = ElementBasicAttributes::from(self) else {
+ return "Unknown".into();
+ };
+
let mut msg = String::new();
- if let Ok(r) = self.role() {
- msg.push_str(&format!("Role: {}\n", r));
- }
+ msg.push_str(&format!("Role: {}\n", fp.role));
if let Ok(t) = self.title() {
msg.push_str(&format!("title: {}\n", t));
@@ -430,6 +439,13 @@ impl GetAttribute for AXUIElement {
msg.push_str(&format!("value: {:?}\n", v));
}
+ if let Some(f) = fp.frame {
+ let CGPoint { x, y } = f.top_left;
+ msg.push_str(&format!("pos: x: {x}, y: {y}\n"));
+ let (w, h) = f.size();
+ msg.push_str(&format!("size: width: {w}, height: {h}\n"));
+ }
+
msg
// for attr in &self.attribute_names().unwrap() {
// println!(
@@ -572,12 +588,13 @@ pub fn traverse_elements(
};
// WARN: Performance critical! Exclude electron elements scrolled off y axis,
- // but should avoid false negatives of ancestors for some menu items,
- // e.g. (Discord right click menu)
if ele_fp.frame.is_some_and(|f| {
- let (_, h) = f.size();
+ let (w, h) = f.size();
(h == 0.0 && f.bottom_right.y == window_frame.bottom_right.y)
- || (h == 1.0 && f.top_left.y == window_frame.top_left.y)
+ // NOTE: keep full width elements, e.g. Brave google search
+ || (h == 1.0 && f.top_left.y == window_frame.top_left.y && w != window_frame.size().0)
+ // NOTE: should avoid false negatives of ancestors for some menu items,
+ // e.g. (Discord right click menu)
}) && vis_level != VisibilityCheckingLevel::Loosest
{
return;