From b97bb730a2af496f829c4e6511ff66b5a17fa9f5 Mon Sep 17 00:00:00 2001 From: blindfs Date: Mon, 4 May 2026 12:49:24 +0800 Subject: [PATCH 1/8] feat: skip meanless intermediate nodes in element exploring mode fix --- src/app_executor.rs | 5 +++- src/ax_element.rs | 73 ++++++++++++++++++++++++++++++--------------- src/key_listener.rs | 3 -- src/util.rs | 7 +++++ 4 files changed, 60 insertions(+), 28 deletions(-) diff --git a/src/app_executor.rs b/src/app_executor.rs index ba5ef0d..c90c0df 100644 --- a/src/app_executor.rs +++ b/src/app_executor.rs @@ -341,7 +341,10 @@ impl AppExecutor { (focused_app, screen_frame) } else { let window = focused_app.focused_window().unwrap_or(focused_app); - let frame = window.get_frame().unwrap_or(screen_frame); + let frame = window + .get_frame() + .and_then(|f| f.intersect(&screen_frame)) + .unwrap_or(screen_frame); (window, frame) }; diff --git a/src/ax_element.rs b/src/ax_element.rs index 6a64400..bd7e661 100644 --- a/src/ax_element.rs +++ b/src/ax_element.rs @@ -232,6 +232,7 @@ impl ElementCache { { return; } + RoleOfInterest::TextField => (), // Check text before size, keep small texts _ if context.is_some() => { // Skip elements with empty/nonsense text @@ -508,7 +509,6 @@ pub fn traverse_elements( // Get child elements 1 level lower // for false negatives aggressively filtered by the visibility checker if *target == Target::ChildElement { - cache.clear(); if let Ok(children) = element.visible_children().or_else(|_| element.children()) { for child in &children { // NOTE: Some apps, like App Store, have circular referencing @@ -516,33 +516,56 @@ pub fn traverse_elements( continue; } if let Some(child_fp) = ElementBasicAttributes::from(&child) - // HACK: exclude electron elements scrolled off y axis - && child_fp.frame.is_some_and(|f| { - let (_, h) = f.size(); - h != 1.0 && h != 0.0 - }) - && child_fp.visible_frame(parent_frame).is_some() + && let Some(c_f) = child_fp.frame + && let Some(inter) = child_fp.visible_frame(window_frame) { - cache.add(&child, None, RoleOfInterest::Generic, child_fp.frame); + // HACK: exclude electron elements scrolled off y axis + let (c_w, c_h) = c_f.size(); + if c_h == 1.0 || c_h == 0.0 { + continue; + } + + // NOTE: recur into temp nodes with nonsense frames, + // or dominating child elements, most of the time, + // they're meaningless. + if inter.contains(window_frame) || c_w <= 1.0 || c_h <= 1.0 || { + let (i_w, i_h) = inter.size(); + let (w_w, w_h) = window_frame.size(); + i_w > 0.9 * w_w && i_h > 0.9 * w_h + } { + traverse_elements( + &child, + parent_frame, + window_frame, + cache, + target, + vis_level, + ); + } else { + cache.add( + &child, + None, + RoleOfInterest::Generic, + child_fp.frame.and_then(|f| f.intersect(window_frame)), + ); + } } } } // Skip element levels where only 1 item available - if cache.cache.len() == 1 - && let Some(ElementOfInterest { - element: Some(element), - frame, - .. - }) = cache.cache.first() - { - traverse_elements( - &element.clone(), - &frame.clone(), - window_frame, - cache, - target, - vis_level, - ); + if cache.cache.len() == 1 { + let temp_eoi = cache.cache[0].clone(); + cache.clear(); + if let Some(element) = temp_eoi.element { + traverse_elements( + &element, + &temp_eoi.frame, + window_frame, + cache, + target, + vis_level, + ); + } } return; @@ -675,7 +698,9 @@ pub fn traverse_elements( ); } Target::Text => { - if let Some(value) = element.get_attribute_string(kAXValueAttribute) { + if let Some(value) = element.get_attribute_string(kAXValueAttribute) + && !value.is_empty() + { cache.add( element, Some(value), diff --git a/src/key_listener.rs b/src/key_listener.rs index 7098f6a..64a81d6 100644 --- a/src/key_listener.rs +++ b/src/key_listener.rs @@ -84,9 +84,6 @@ impl Display for MenuItem { } } -// TODO: Config sub-menu to -// 1. Reload config -// 2. Toggle aggressive visibility check pub const DASH_BOARD_MENU_ITEMS: [MenuItem; 9] = [ MenuItem::new("󰦨 Text", 'T', AppSignal::Activate(Target::Text)), MenuItem::new("󰳽 Press", 'P', AppSignal::Activate(Target::Clickable)), diff --git a/src/util.rs b/src/util.rs index 296320d..cfa369b 100644 --- a/src/util.rs +++ b/src/util.rs @@ -174,6 +174,13 @@ impl Frame { self.bottom_right.y.max(other.bottom_right.y), ) } + + pub fn contains(&self, other: &Frame) -> bool { + self.top_left.x <= other.top_left.x + && self.top_left.y <= other.top_left.y + && self.bottom_right.x >= other.bottom_right.x + && self.bottom_right.y >= other.bottom_right.y + } } fn estimate_font_height(s: &str, frame: &Frame) -> f64 { From db767f9ffeb31c20b63fa7e61833bdab513eb368 Mon Sep 17 00:00:00 2001 From: blindfs Date: Mon, 4 May 2026 14:25:40 +0800 Subject: [PATCH 2/8] fix: better visibility checking trade-off --- src/app_executor.rs | 20 +++++--- src/ax_element.rs | 120 +++++++++++++++++++++----------------------- src/config.rs | 7 +++ 3 files changed, 76 insertions(+), 71 deletions(-) diff --git a/src/app_executor.rs b/src/app_executor.rs index c90c0df..6d83788 100644 --- a/src/app_executor.rs +++ b/src/app_executor.rs @@ -8,7 +8,7 @@ use crate::{ ax_element::{ ElementCache, ElementOfInterest, GetAttribute, SetAttribute, Target, traverse_elements, }, - config::{GlyphlowConfig, RoleOfInterest, WorkFlow, WorkFlowAction}, + config::{GlyphlowConfig, RoleOfInterest, VisibilityCheckingLevel, WorkFlow, WorkFlowAction}, drawer::GlyphlowDrawingLayer, os_util::get_focused_pid, util::{Frame, HintBox, estimate_frame_for_text, hint_boxes_from_frames, select_range_helper}, @@ -321,7 +321,7 @@ impl AppExecutor { ) } - fn select_app_window(&mut self) -> Option { + fn select_app_window(&mut self, vis_level: VisibilityCheckingLevel) -> Option { let (pid, is_electron) = get_focused_pid()?; self.is_electron = is_electron; @@ -336,8 +336,8 @@ impl AppExecutor { } self.last_pid = pid; - // HACK: some electron apps put right click pop-up menus in different windows - let (focused_window, window_frame) = if self.target == Target::MenuItem { + // HACK: menu items may go out of focused window + let (focused_window, window_frame) = if vis_level == VisibilityCheckingLevel::Loosest { (focused_app, screen_frame) } else { let window = focused_app.focused_window().unwrap_or(focused_app); @@ -366,8 +366,14 @@ impl AppExecutor { _ => target, }; + let vis_level = match target { + // NOTE: loose visibility checking for specific targets + Target::MenuItem | Target::Custom(_) => VisibilityCheckingLevel::Loosest, + _ => self.config.visibility_checking_level, + }; + if self.selected.is_none() { - self.select_app_window(); + self.select_app_window(vis_level); } self.clear_cache(); @@ -384,7 +390,7 @@ impl AppExecutor { frame, &mut self.element_cache, &target, - self.config.visibility_checking_level, + vis_level, ); } } @@ -1156,7 +1162,7 @@ impl AppExecutor { } else { // Defaults to the window &self - .select_app_window() + .select_app_window(self.config.visibility_checking_level) .unwrap_or_else(|| Frame::from_origion(self.screen_size)) }; if screen_shot(frame).await { diff --git a/src/ax_element.rs b/src/ax_element.rs index bd7e661..eb1e121 100644 --- a/src/ax_element.rs +++ b/src/ax_element.rs @@ -8,11 +8,11 @@ use accessibility::{AXAttribute, AXUIElement, AXUIElementAttributes}; use accessibility_sys::{ AXUIElementCopyMultipleAttributeValues, AXValueCreate, AXValueGetValue, AXValueRef, kAXButtonRole, kAXCellRole, kAXCheckBoxRole, kAXComboBoxRole, kAXDescriptionAttribute, - kAXErrorSuccess, kAXGroupRole, kAXHiddenAttribute, kAXImageRole, kAXMenuItemRole, + kAXErrorSuccess, kAXGroupRole, kAXHiddenAttribute, kAXImageRole, kAXListRole, kAXMenuItemRole, kAXPopUpButtonRole, kAXPositionAttribute, kAXPressAction, kAXRoleAttribute, kAXRowRole, - kAXScrollBarRole, kAXSelectedTextRangeAttribute, kAXSizeAttribute, kAXStaticTextRole, - kAXTextAreaRole, kAXTextFieldRole, kAXTitleAttribute, kAXValueAttribute, kAXValueTypeCFRange, - kAXValueTypeCGPoint, kAXValueTypeCGSize, kAXWindowRole, + kAXScrollAreaRole, kAXScrollBarRole, kAXSelectedTextRangeAttribute, kAXSizeAttribute, + kAXStaticTextRole, kAXTextAreaRole, kAXTextFieldRole, kAXTitleAttribute, kAXValueAttribute, + kAXValueTypeCFRange, kAXValueTypeCGPoint, kAXValueTypeCGSize, kAXWindowRole, }; use core_foundation::{ array::{CFArray, CFArrayRef}, @@ -506,52 +506,53 @@ pub fn traverse_elements( return; }; + // HACK: Performance critical! Exclude electron elements scrolled off y axis, + // but should avoid false negatives of ancestors for some menu items, + // e.g. (Discord right click menu) + if ele_fp.frame.is_some_and(|f| { + let (_, h) = f.size(); + (h == 0.0 && f.bottom_right.y == parent_frame.bottom_right.y) + || (h == 1.0 && f.top_left.y == parent_frame.top_left.y) + }) && vis_level != VisibilityCheckingLevel::Loosest + { + return; + } + // Get child elements 1 level lower // for false negatives aggressively filtered by the visibility checker if *target == Target::ChildElement { - if let Ok(children) = element.visible_children().or_else(|_| element.children()) { - for child in &children { - // NOTE: Some apps, like App Store, have circular referencing - if *child == *element { - continue; - } - if let Some(child_fp) = ElementBasicAttributes::from(&child) - && let Some(c_f) = child_fp.frame - && let Some(inter) = child_fp.visible_frame(window_frame) - { - // HACK: exclude electron elements scrolled off y axis - let (c_w, c_h) = c_f.size(); - if c_h == 1.0 || c_h == 0.0 { - continue; - } - - // NOTE: recur into temp nodes with nonsense frames, - // or dominating child elements, most of the time, - // they're meaningless. - if inter.contains(window_frame) || c_w <= 1.0 || c_h <= 1.0 || { - let (i_w, i_h) = inter.size(); - let (w_w, w_h) = window_frame.size(); - i_w > 0.9 * w_w && i_h > 0.9 * w_h - } { - traverse_elements( - &child, - parent_frame, - window_frame, - cache, - target, - vis_level, - ); - } else { - cache.add( - &child, - None, - RoleOfInterest::Generic, - child_fp.frame.and_then(|f| f.intersect(window_frame)), - ); - } + let Ok(children) = element.visible_children().or_else(|_| element.children()) else { + return; + }; + for child in &children { + // NOTE: Some apps, like App Store, have circular referencing + if *child == *element { + continue; + } + if let Some(child_fp) = ElementBasicAttributes::from(&child) + && let Some(c_f) = child_fp.frame + && let Some(inter) = child_fp.visible_frame(window_frame) + { + // NOTE: recur into temp nodes with nonsense frames, + // or dominating child elements, most of the time, they're meaningless. + let (c_w, c_h) = c_f.size(); + if inter.contains(window_frame) || c_w <= 1.0 || c_h <= 1.0 || { + let (i_w, i_h) = inter.size(); + let (w_w, w_h) = window_frame.size(); + i_w > 0.9 * w_w && i_h > 0.9 * w_h + } { + traverse_elements(&child, parent_frame, window_frame, cache, target, vis_level); + } else { + cache.add( + &child, + None, + RoleOfInterest::Generic, + child_fp.frame.and_then(|f| f.intersect(window_frame)), + ); } } } + // Skip element levels where only 1 item available if cache.cache.len() == 1 { let temp_eoi = cache.cache[0].clone(); @@ -559,7 +560,7 @@ pub fn traverse_elements( if let Some(element) = temp_eoi.element { traverse_elements( &element, - &temp_eoi.frame, + parent_frame, window_frame, cache, target, @@ -576,29 +577,18 @@ pub fn traverse_elements( return; }; - // HACK: exclude electron elements scrolled off y axis, - // but some menu items' ancestors (Discord) are of zero height - let vis_level = match target { - // NOTE: loose visibility checking for specific targets - Target::MenuItem | Target::Custom(_) => VisibilityCheckingLevel::Loose, - _ if ele_fp.frame.is_some_and(|f| { - let (_, h) = f.size(); - h == 1.0 || h == 0.0 - }) => - { - return; - } - _ => vis_level, - }; - match vis_level { VisibilityCheckingLevel::Medium => { + // NOTE: `parent_frame` should be monotonically decreasing, + // and always included in `window_frame` new_frame = ele_fp .frame .and_then(|f| f.intersect(window_frame)) .unwrap_or(*parent_frame); } - VisibilityCheckingLevel::Loose => new_frame = *parent_frame, + VisibilityCheckingLevel::Loose | VisibilityCheckingLevel::Loosest => { + new_frame = *parent_frame + } _ => (), } @@ -681,11 +671,13 @@ pub fn traverse_elements( } _ => (), }, - kAXWindowRole => { + kAXWindowRole | kAXListRole | kAXScrollAreaRole | "AXWebArea" + if vis_level != VisibilityCheckingLevel::Loosest => + { // NOTE: For AXApplication the frame is usually None, defaults to full screen. // Need to narrow down to window frame at this place. - if let Some(win_frame) = ele_fp.frame { - new_frame = win_frame; + if let Some(area_frame) = ele_fp.frame { + new_frame = area_frame.intersect(parent_frame).unwrap_or(*parent_frame); }; } kAXComboBoxRole | kAXTextFieldRole | kAXTextAreaRole => match target { diff --git a/src/config.rs b/src/config.rs index f74482b..5d78ba8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -292,8 +292,15 @@ pub struct KeyBinding { #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Copy)] pub enum VisibilityCheckingLevel { + /// As long as element frame intersects with whole screen, + /// served for limited targets + Loosest, + /// As long as element frame intersects with window frame Loose, + /// Element frame should intersect with its own parent, + /// as well as the window frame Medium, + /// Element frame should intersect with all its ancestors Strict, } From ac5867e3161a24d561a1359de464c698601e3064 Mon Sep 17 00:00:00 2001 From: blindfs Date: Mon, 4 May 2026 14:30:22 +0800 Subject: [PATCH 3/8] fix: exploring mode override on center collision --- src/ax_element.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/ax_element.rs b/src/ax_element.rs index eb1e121..40cd72d 100644 --- a/src/ax_element.rs +++ b/src/ax_element.rs @@ -257,10 +257,7 @@ impl ElementCache { // NOTE: de-duplication for DOM elements let new_ele = ElementOfInterest::new(Some(element.clone()), context, role.clone(), frame); - // Keep all nodes with Target::ChildElement/GenericNode, as it's basically a debugging mode - if role != RoleOfInterest::Generic - && let Some(idx) = self.seen_center.get(¢er) - { + if let Some(idx) = self.seen_center.get(¢er) { self.cache[*idx] = new_ele; } else { self.seen_center.insert(center, self.cache.len()); From a179bb2a646150dd3a5160dbbdda1c0f79fae491 Mon Sep 17 00:00:00 2001 From: blindfs Date: Mon, 4 May 2026 14:51:30 +0800 Subject: [PATCH 4/8] fix: regression: zero sized shadow menu items failing the Apple Intelligence workflow --- src/ax_element.rs | 8 +++++--- src/config.rs | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ax_element.rs b/src/ax_element.rs index 40cd72d..9b4678c 100644 --- a/src/ax_element.rs +++ b/src/ax_element.rs @@ -220,7 +220,10 @@ impl ElementCache { let (w, h) = frame.size(); match role { // NOTE: some roles to keep - RoleOfInterest::Generic | RoleOfInterest::ScrollBar | RoleOfInterest::CustomTarget => {} + RoleOfInterest::Generic | RoleOfInterest::ScrollBar => {} + // HACK: some menu items (like Apple Intelligence writing tools) + // may have zero sized shadows, skip them to keep the workflow going + RoleOfInterest::CustomTarget if w != 0.0 && h != 0.0 => {} RoleOfInterest::Image if w.min(h) < self.image_min_size => { return; } @@ -503,7 +506,7 @@ pub fn traverse_elements( return; }; - // HACK: Performance critical! Exclude electron elements scrolled off y axis, + // WARN: Performance critical! Exclude electron elements scrolled off y axis, // but should avoid false negatives of ancestors for some menu items, // e.g. (Discord right click menu) if ele_fp.frame.is_some_and(|f| { @@ -597,7 +600,6 @@ pub fn traverse_elements( cache.add(element, None, RoleOfInterest::CustomTarget, ele_fp.frame); }; - // TODO: Fine-grained control #[allow(non_upper_case_globals)] match ele_fp.role.as_str() { // TODO: DOM Class List based image searching for icon button diff --git a/src/config.rs b/src/config.rs index 5d78ba8..51be510 100644 --- a/src/config.rs +++ b/src/config.rs @@ -293,7 +293,7 @@ pub struct KeyBinding { #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Copy)] pub enum VisibilityCheckingLevel { /// As long as element frame intersects with whole screen, - /// served for limited targets + /// reserved for limited targets Loosest, /// As long as element frame intersects with window frame Loose, From f2f976a6998024dd589d0cb9d365f0d8b1a0043a Mon Sep 17 00:00:00 2001 From: blindfs Date: Mon, 4 May 2026 15:06:04 +0800 Subject: [PATCH 5/8] feat: new workflow action `done` to deactivate --- src/app_executor.rs | 8 ++++++-- src/config.rs | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/app_executor.rs b/src/app_executor.rs index 6d83788..acb8dd6 100644 --- a/src/app_executor.rs +++ b/src/app_executor.rs @@ -765,7 +765,7 @@ impl AppExecutor { .config .text_actions .get(idx) - .expect("Internal Error: text action index: {idx} out of bounds."); + .expect("Internal Error: text action index out of bounds."); let args = action .args .iter() @@ -826,7 +826,7 @@ impl AppExecutor { .workflows .get(idx) .cloned() - .expect("Internal Error: text workflow index: {idx} out of bounds."); + .expect("Internal Error: text workflow index out of bounds."); for (act_idx, act) in workflow.actions.iter().enumerate() { // Check starting_role, nothing happens if not match @@ -836,6 +836,10 @@ impl AppExecutor { // Actions don't need a selected element match act { + WorkFlowAction::Done => { + self.deactivate(); + continue; + } WorkFlowAction::Sleep(ms) => { std::thread::sleep(Duration::from_millis(*ms)); continue; diff --git a/src/config.rs b/src/config.rs index 51be510..ca2a5aa 100644 --- a/src/config.rs +++ b/src/config.rs @@ -35,6 +35,7 @@ pub struct CustomTarget { #[derive(Serialize, Deserialize, Debug, Clone)] pub enum WorkFlowAction { + Done, SelectAll, Focus, Press, From de84fb7ea2376b9a824111803925a6b7f3b96e62 Mon Sep 17 00:00:00 2001 From: blindfs Date: Mon, 4 May 2026 16:08:53 +0800 Subject: [PATCH 6/8] fix: apple music AXSectionList regression --- src/ax_element.rs | 82 +++++++++++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 32 deletions(-) diff --git a/src/ax_element.rs b/src/ax_element.rs index 9b4678c..83719fd 100644 --- a/src/ax_element.rs +++ b/src/ax_element.rs @@ -7,12 +7,13 @@ use crate::{ use accessibility::{AXAttribute, AXUIElement, AXUIElementAttributes}; use accessibility_sys::{ AXUIElementCopyMultipleAttributeValues, AXValueCreate, AXValueGetValue, AXValueRef, - kAXButtonRole, kAXCellRole, kAXCheckBoxRole, kAXComboBoxRole, kAXDescriptionAttribute, - kAXErrorSuccess, kAXGroupRole, kAXHiddenAttribute, kAXImageRole, kAXListRole, kAXMenuItemRole, - kAXPopUpButtonRole, kAXPositionAttribute, kAXPressAction, kAXRoleAttribute, kAXRowRole, - kAXScrollAreaRole, kAXScrollBarRole, kAXSelectedTextRangeAttribute, kAXSizeAttribute, - kAXStaticTextRole, kAXTextAreaRole, kAXTextFieldRole, kAXTitleAttribute, kAXValueAttribute, - kAXValueTypeCFRange, kAXValueTypeCGPoint, kAXValueTypeCGSize, kAXWindowRole, + kAXButtonRole, kAXCellRole, kAXCheckBoxRole, kAXComboBoxRole, kAXContentListSubrole, + kAXDescriptionAttribute, kAXErrorSuccess, kAXGroupRole, kAXHiddenAttribute, kAXImageRole, + kAXListRole, kAXMenuItemRole, kAXPopUpButtonRole, kAXPositionAttribute, kAXPressAction, + kAXRoleAttribute, kAXRowRole, kAXScrollAreaRole, kAXScrollBarRole, + kAXSelectedTextRangeAttribute, kAXSizeAttribute, kAXStaticTextRole, kAXTextAreaRole, + kAXTextFieldRole, kAXTitleAttribute, kAXValueAttribute, kAXValueTypeCFRange, + kAXValueTypeCGPoint, kAXValueTypeCGSize, kAXWindowRole, }; use core_foundation::{ array::{CFArray, CFArrayRef}, @@ -511,8 +512,8 @@ pub fn traverse_elements( // e.g. (Discord right click menu) if ele_fp.frame.is_some_and(|f| { let (_, h) = f.size(); - (h == 0.0 && f.bottom_right.y == parent_frame.bottom_right.y) - || (h == 1.0 && f.top_left.y == parent_frame.top_left.y) + (h == 0.0 && f.bottom_right.y == window_frame.bottom_right.y) + || (h == 1.0 && f.top_left.y == window_frame.top_left.y) }) && vis_level != VisibilityCheckingLevel::Loosest { return; @@ -541,7 +542,7 @@ pub fn traverse_elements( let (w_w, w_h) = window_frame.size(); i_w > 0.9 * w_w && i_h > 0.9 * w_h } { - traverse_elements(&child, parent_frame, window_frame, cache, target, vis_level); + traverse_elements(&child, &c_f, window_frame, cache, target, vis_level); } else { cache.add( &child, @@ -560,7 +561,10 @@ pub fn traverse_elements( if let Some(element) = temp_eoi.element { traverse_elements( &element, - parent_frame, + &temp_eoi + .frame + .intersect(window_frame) + .unwrap_or(*parent_frame), window_frame, cache, target, @@ -573,24 +577,30 @@ pub fn traverse_elements( } // If invisible, return early - let Some(mut new_frame) = ele_fp.visible_frame(parent_frame) else { - return; - }; - - match vis_level { - VisibilityCheckingLevel::Medium => { - // NOTE: `parent_frame` should be monotonically decreasing, - // and always included in `window_frame` - new_frame = ele_fp - .frame - .and_then(|f| f.intersect(window_frame)) - .unwrap_or(*parent_frame); - } + // NOTE: `parent_frame` should be monotonically decreasing, + // and always included in `window_frame` + let new_frame = match vis_level { VisibilityCheckingLevel::Loose | VisibilityCheckingLevel::Loosest => { - new_frame = *parent_frame + let Some(new_frame) = ele_fp.visible_frame(window_frame) else { + return; + }; + new_frame } - _ => (), - } + // Check intersection with parent frame + _ => { + let Some(new_frame) = ele_fp.visible_frame(parent_frame) else { + return; + }; + if vis_level == VisibilityCheckingLevel::Strict { + new_frame + } else { + ele_fp + .frame + .and_then(|f| f.intersect(window_frame)) + .unwrap_or(*parent_frame) + } + } + }; // Try matching custom target first if let Target::Custom(ct) = target @@ -600,6 +610,8 @@ pub fn traverse_elements( cache.add(element, None, RoleOfInterest::CustomTarget, ele_fp.frame); }; + let mut window_frame = *window_frame; + #[allow(non_upper_case_globals)] match ele_fp.role.as_str() { // TODO: DOM Class List based image searching for icon button @@ -670,13 +682,19 @@ pub fn traverse_elements( } _ => (), }, - kAXWindowRole | kAXListRole | kAXScrollAreaRole | "AXWebArea" + // NOTE: narrow down to window frame at "Window-ish" nodes. + // This is useful for y axis scroll-off detection of electron apps + kAXWindowRole | kAXScrollAreaRole | "AXWebArea" if vis_level != VisibilityCheckingLevel::Loosest => { - // NOTE: For AXApplication the frame is usually None, defaults to full screen. - // Need to narrow down to window frame at this place. - if let Some(area_frame) = ele_fp.frame { - new_frame = area_frame.intersect(parent_frame).unwrap_or(*parent_frame); + if let Some(area_frame) = ele_fp.frame.and_then(|f| f.intersect(&window_frame)) { + window_frame = area_frame; + }; + } + // NOTE: don't do it for AXSectionList, e.g. Apple Music + kAXListRole if element.subrole().is_ok_and(|r| r == kAXContentListSubrole) => { + if let Some(area_frame) = ele_fp.frame.and_then(|f| f.intersect(&window_frame)) { + window_frame = area_frame; }; } kAXComboBoxRole | kAXTextFieldRole | kAXTextAreaRole => match target { @@ -777,7 +795,7 @@ pub fn traverse_elements( if *child == *element { continue; } - traverse_elements(&child, &new_frame, window_frame, cache, target, vis_level); + traverse_elements(&child, &new_frame, &window_frame, cache, target, vis_level); } } } From 73c19d796851d0a78f02e0e682dcae6a1500d245 Mon Sep 17 00:00:00 2001 From: blindfs Date: Mon, 4 May 2026 17:26:35 +0800 Subject: [PATCH 7/8] refactor: left/right click for generic elements are now implemented as workflows --- src/app_executor.rs | 217 ++++++++++++++++++++------------------------ src/config.rs | 23 ++++- src/key_listener.rs | 13 +-- 3 files changed, 121 insertions(+), 132 deletions(-) diff --git a/src/app_executor.rs b/src/app_executor.rs index acb8dd6..aefef8d 100644 --- a/src/app_executor.rs +++ b/src/app_executor.rs @@ -24,6 +24,7 @@ use objc2_core_foundation::CGSize; use objc2_quartz_core::CALayer; use rdev::{Button, EventType, simulate}; use std::{ + collections::VecDeque, path::PathBuf, sync::{Arc, Mutex}, time::Duration, @@ -94,6 +95,8 @@ pub struct AppExecutor { last_pid: i32, /// For multi-selection multi_selection: MultiSeletionState, + /// Something to finish after filtering + pending_workflow_actions: VecDeque, } impl AppExecutor { @@ -131,6 +134,7 @@ impl AppExecutor { is_electron: false, last_pid: 0, multi_selection: MultiSeletionState::default(), + pending_workflow_actions: VecDeque::new(), } } @@ -154,6 +158,7 @@ impl AppExecutor { self.clear_cache(); self.clear_drawing(); self.selected = None; + self.pending_workflow_actions.clear(); self.set_mode(Mode::Idle); } @@ -451,34 +456,6 @@ impl AppExecutor { Self::simulate_event(&EventType::ButtonRelease(button)); } - fn click_on_selected(&self) { - if let Some(ElementOfInterest { frame, .. }) = self.selected.as_ref() { - let (x, y) = frame.center(); - Self::simulate_click(x, y, false); - } - } - - fn right_click_menu_on_selected(&mut self) { - if let Some(ElementOfInterest { element, frame, .. }) = self.selected.as_ref() { - let center = frame.center(); - let (x, y) = center; - if let Some(element) = element { - self.right_click_menu_on_element(element, center) - } else { - Self::simulate_click(x, y, true); - } - // HACK: wait for the right click menu to draw. - std::thread::sleep(Duration::from_millis(self.config.menu_wait_ms)); - self.selected = None; - self.activate(Target::MenuItem); - } else { - self.notify( - "Trying to perform a right click with nothing selected.", - Level::Error, - ); - } - } - fn focus_on_element(element: &AXUIElement) { element.set_attribute_by_name(kAXFocusedAttribute, CFBoolean::true_value().as_CFType()); } @@ -645,6 +622,7 @@ impl AppExecutor { } Target::Custom(_) => { self.selected = Some(eoi.clone()); + self.execute_pending_workflow_actions(); } Target::ImageOCR => self.perform_ocr_on_frame(*frame).await, Target::Text => { @@ -812,106 +790,110 @@ impl AppExecutor { fn is_workflow_valid(&self, wf: &WorkFlow) -> bool { match wf.starting_role { RoleOfInterest::Empty => self.selected.is_none(), - RoleOfInterest::Generic => self.selected.is_some(), + RoleOfInterest::Generic => self.selected.as_ref().is_some_and(|s| s.element.is_some()), _ => self .selected .as_ref() - .is_some_and(|s| s.role == wf.starting_role), + .is_some_and(|s| s.element.is_some() && s.role == wf.starting_role), } } - async fn execute_workflow(&mut self, idx: usize) { - let workflow = self - .config - .workflows - .get(idx) - .cloned() - .expect("Internal Error: text workflow index out of bounds."); - - for (act_idx, act) in workflow.actions.iter().enumerate() { - // Check starting_role, nothing happens if not match - if act_idx == 0 && !self.is_workflow_valid(&workflow) { - return; + /// Returns true if there're pending actions to finish + fn execute_workflow_action(&mut self, act: &WorkFlowAction) -> bool { + // Actions don't need a selected element + match act { + WorkFlowAction::Sleep(ms) => { + std::thread::sleep(Duration::from_millis(*ms)); + return false; } - - // Actions don't need a selected element - match act { - WorkFlowAction::Done => { - self.deactivate(); - continue; - } - WorkFlowAction::Sleep(ms) => { - std::thread::sleep(Duration::from_millis(*ms)); - continue; + WorkFlowAction::SearchFor(ct) => { + self.selected = None; + self.activate(Target::Custom(ct.clone())); + if self.element_cache.cache.len() == 1 { + self.clear_drawing(); + self.selected = Some(self.element_cache.cache[0].clone()); + } else if self.element_cache.cache.len() > 1 { + return true; } - WorkFlowAction::SearchFor(ct) => { - self.selected = None; - self.activate(Target::Custom(ct.clone())); - if self.element_cache.cache.len() == 1 { - self.quick_follow().await; - } else if self.element_cache.cache.len() > 1 { - self.notify_then_deactivate( - "Multiple elements found.\nOperation canceled.\nPlease run manually", - Level::Warn, - ); - return; - } else { - return; - } - continue; + return false; + } + WorkFlowAction::KeyCombo(kb) => { + self.set_simulating_key(true); + for k in kb.keys.iter() { + Self::simulate_event(&EventType::KeyPress(*k)); + std::thread::sleep(Duration::from_millis(20)); } - WorkFlowAction::KeyCombo(kb) => { - self.set_simulating_key(true); - for k in kb.keys.iter() { - Self::simulate_event(&EventType::KeyPress(*k)); - std::thread::sleep(Duration::from_millis(20)); - } - for k in kb.keys.iter().rev() { - Self::simulate_event(&EventType::KeyRelease(*k)); - } - self.set_simulating_key(false); - continue; + for k in kb.keys.iter().rev() { + Self::simulate_event(&EventType::KeyRelease(*k)); } - _ => (), + self.set_simulating_key(false); + return false; } + _ => (), + } - // Actions that require a selected element - let Some(ElementOfInterest { - element: Some(element), - context, - role, - frame, - .. - }) = self.selected.as_ref() - else { - self.notify_then_deactivate( - &format!("Running a workflow action with no element selected. {act:?} at idx {act_idx}"), - Level::Error, - ); + // Actions that require a selected element + let Some(ElementOfInterest { + element: Some(element), + context, + role, + frame, + .. + }) = self.selected.as_ref() + else { + self.notify_then_deactivate( + &format!("Running a workflow action with no element selected. {act:?}"), + Level::Error, + ); + return true; + }; + + match act { + WorkFlowAction::Focus => { + Self::focus_on_element(element); + } + WorkFlowAction::Press => { + let center = frame.center(); + self.press_on_element(element, role, center); + } + WorkFlowAction::ShowMenu => { + let center = frame.center(); + self.right_click_menu_on_element(element, center); + } + WorkFlowAction::SelectAll => { + let len = context + .clone() + .map(|txt| txt.encode_utf16().count()) + .unwrap_or(0) as isize; + element.set_selected_range(0, len); + } + _ => (), + } + false + } + + fn execute_pending_workflow_actions(&mut self) { + while let Some(act) = self.pending_workflow_actions.pop_front() { + if self.execute_workflow_action(&act) { return; }; + } + self.clear_drawing(); + self.notify_then_deactivate("Done", Level::Trace); + } - match act { - WorkFlowAction::Focus => { - Self::focus_on_element(element); - } - WorkFlowAction::Press => { - let center = frame.center(); - self.press_on_element(element, role, center); - } - WorkFlowAction::ShowMenu => { - let center = frame.center(); - self.right_click_menu_on_element(element, center); - } - WorkFlowAction::SelectAll => { - let len = context - .clone() - .map(|txt| txt.encode_utf16().count()) - .unwrap_or(0) as isize; - element.set_selected_range(0, len); - } - _ => (), - } + fn execute_workflow(&mut self, idx: usize) { + let workflow = self + .config + .workflows + .get(idx) + .cloned() + .expect("Internal Error: text workflow index out of bounds."); + + // Silently quit if workflow is not valid for current selected element + if self.is_workflow_valid(&workflow) { + self.pending_workflow_actions = workflow.actions.into(); + self.execute_pending_workflow_actions(); } } @@ -956,15 +938,8 @@ impl AppExecutor { AppSignal::DeActivate => { self.deactivate(); } - AppSignal::Press => { - self.click_on_selected(); - self.deactivate(); - } - AppSignal::ShowMenu => { - self.right_click_menu_on_selected(); - } AppSignal::RunWorkFlow(idx) => { - self.execute_workflow(idx).await; + self.execute_workflow(idx); } AppSignal::ToggleMultiSelection => match self.target { Target::Text | Target::ImageOCR => { diff --git a/src/config.rs b/src/config.rs index ca2a5aa..50dfdcc 100644 --- a/src/config.rs +++ b/src/config.rs @@ -35,7 +35,6 @@ pub struct CustomTarget { #[derive(Serialize, Deserialize, Debug, Clone)] pub enum WorkFlowAction { - Done, SelectAll, Focus, Press, @@ -224,6 +223,8 @@ impl AlphabeticKey for Key { Key::KeyY => 'Y', Key::KeyZ => 'Z', Key::Backspace | Key::Delete => '-', + Key::LeftBracket => '[', + Key::RightBracket => ']', _ => ' ', } } @@ -392,6 +393,26 @@ fn default_workflows() -> Vec { WorkFlowAction::Press, ], }, + WorkFlow { + key: '[', + display: "󰳽 Press [Left Click]".into(), + starting_role: RoleOfInterest::Generic, + actions: vec![WorkFlowAction::Press], + }, + WorkFlow { + key: ']', + display: " Menu [Right Click]".into(), + starting_role: RoleOfInterest::Generic, + actions: vec![ + WorkFlowAction::ShowMenu, + WorkFlowAction::Sleep(150), + WorkFlowAction::SearchFor(CustomTarget { + role: "MenuItem".into(), + ..Default::default() + }), + WorkFlowAction::Press, + ], + }, ] } fn default_scroll_distance() -> f64 { diff --git a/src/key_listener.rs b/src/key_listener.rs index 64a81d6..232c65e 100644 --- a/src/key_listener.rs +++ b/src/key_listener.rs @@ -57,8 +57,6 @@ pub enum AppSignal { ReadClipboard, ScreenShot, FrameOCR, - Press, - ShowMenu, } #[derive(Debug, PartialEq)] @@ -119,7 +117,7 @@ pub const SCROLLBAR_MENU_ITEMS: [MenuItem; 4] = [ ), ]; -pub const TEXT_ACTION_MENU_ITEMS: [MenuItem; 5] = [ +pub const TEXT_ACTION_MENU_ITEMS: [MenuItem; 3] = [ MenuItem::new("⮺ Copy", 'C', AppSignal::TextAction(TextAction::Copy)), MenuItem::new( "◫ Dictionary", @@ -127,15 +125,10 @@ pub const TEXT_ACTION_MENU_ITEMS: [MenuItem; 5] = [ AppSignal::TextAction(TextAction::Dictionary), ), MenuItem::new("󰃻 Split", 'S', AppSignal::TextAction(TextAction::Split)), - MenuItem::new("󰳽 Press [Left Click]", 'P', AppSignal::Press), - MenuItem::new(" Menu [Right Click]", 'M', AppSignal::ShowMenu), ]; -pub const IMAGE_ACTION_MENU_ITEMS: [MenuItem; 3] = [ - MenuItem::new("󱄺 Image OCR", 'O', AppSignal::FrameOCR), - MenuItem::new("󰳽 Press [Left Click]", 'P', AppSignal::Press), - MenuItem::new(" Menu [Right Click]", 'M', AppSignal::ShowMenu), -]; +pub const IMAGE_ACTION_MENU_ITEMS: [MenuItem; 1] = + [MenuItem::new("󱄺 Image OCR", 'O', AppSignal::FrameOCR)]; #[derive(Debug, PartialEq)] pub enum Mode { From 6f474cf3fc45d0594bba84b98a273bc5960f0d11 Mon Sep 17 00:00:00 2001 From: blindfs Date: Mon, 4 May 2026 18:14:08 +0800 Subject: [PATCH 8/8] fix: checkbox skip size check --- src/ax_element.rs | 4 ++-- src/config.rs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ax_element.rs b/src/ax_element.rs index 83719fd..b0d4ed4 100644 --- a/src/ax_element.rs +++ b/src/ax_element.rs @@ -221,7 +221,7 @@ impl ElementCache { let (w, h) = frame.size(); match role { // NOTE: some roles to keep - RoleOfInterest::Generic | RoleOfInterest::ScrollBar => {} + RoleOfInterest::Generic | RoleOfInterest::ScrollBar | RoleOfInterest::CheckBox => {} // HACK: some menu items (like Apple Intelligence writing tools) // may have zero sized shadows, skip them to keep the workflow going RoleOfInterest::CustomTarget if w != 0.0 && h != 0.0 => {} @@ -726,7 +726,7 @@ pub fn traverse_elements( }, kAXCheckBoxRole => match target { Target::Clickable => { - cache.add(element, None, RoleOfInterest::Button, ele_fp.frame); + cache.add(element, None, RoleOfInterest::CheckBox, ele_fp.frame); } Target::Text => { if let Ok(value) = element.description().map(|v| v.to_string()) { diff --git a/src/config.rs b/src/config.rs index 50dfdcc..3c3f011 100644 --- a/src/config.rs +++ b/src/config.rs @@ -11,6 +11,7 @@ use std::path::PathBuf; #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] pub enum RoleOfInterest { Button, + CheckBox, Generic, Empty, Image,