diff --git a/docs/agent_control.md b/docs/agent_control.md index 714dd0a..ec75826 100644 --- a/docs/agent_control.md +++ b/docs/agent_control.md @@ -205,9 +205,9 @@ case-insensitive), `handle` (HWND), `process` (process name without `.exe`), | `find` | Find a **UI Automation element** by name / automation id / class. | window target, `by` (`name`\|`automationid`\|`classname`), `value`, `timeout` | | `wait-for` | Same as `find`, but waits up to a timeout for the element to appear (default 5 s). | window target, `by`, `value`, `timeout` | | `invoke` | Drive a UI Automation element pattern (incl. select for SelectionItem); far more reliable than coordinate clicks. | window target, `by`, `value`, `action` (`invoke`\|`toggle`\|`setvalue`\|`setfocus`\|`expand`\|`collapse`\|`select`), `text` | -| `drag` | Press → move along a path → release, dispatched **atomically** (nothing interleaves). Each endpoint is a UI Automation element (dragged from/to its centre) or an absolute screen pixel; add `path` waypoints for a curved/multi-stop drag. Covers window resize/move by chrome, sliders, marquee-select, drag-reorder. | window target (needed when an endpoint is an element); `from`/`to` each `{ by, value }` or `{ x, y }`; optional `path` `[{ x, y }, …]` | +| `drag` | Press → move along a path → release, dispatched **atomically** (nothing interleaves). Each endpoint is a UI Automation element (dragged from/to its centre) or a pixel endpoint: with a window target, endpoint `{ x, y }` is window-relative (window top-left + offset, matching `capture` coordinates); without a window target, `{ x, y }` is absolute screen pixels. `path` waypoints remain absolute screen pixels. Covers window resize/move by chrome, sliders, marquee-select, drag-reorder. | window target (required for element endpoints; optional for window-relative pixel endpoints); `from`/`to` each `{ by, value }` or `{ x, y }`; optional `path` `[{ x, y }, …]` | | `launch` | Launch an app directly (path + args + working dir); gated. Returns pid + primary window handle/info and explicit intent flags (`startedNewProcess`, `attachedToExisting`; attachment also returns `launchedProcessId`/`windowProcessId`). Preferred over Win+R composition. | `path` (required), `arguments`, `workingDirectory`, `timeout` | -| `click` | Click at a point (a UI Automation element's centre or an absolute screen pixel); move+click is dispatched **atomically**. For element types `invoke` can't drive, or when you must target a pixel. Prefer `invoke` for ordinary buttons/menus. | window target (needed when `at` is an element); `at` = `{ by, value }` or `{ x, y }`; `button` (`left`\|`right`\|`middle`, default `left`); `count` (`1`\|`2`, default `1`) | +| `click` | Click at a point (a UI Automation element's centre or a pixel endpoint); move+click is dispatched **atomically**. With a window target, `at: { x, y }` is window-relative (window top-left + offset, matching `capture` coordinates). Without a window target, `{ x, y }` is absolute screen pixels. For element types `invoke` can't drive, or when you must target a pixel. Prefer `invoke` for ordinary buttons/menus. | window target (required for element endpoints; optional for window-relative pixel endpoints); `at` = `{ by, value }` or `{ x, y }`; `button` (`left`\|`right`\|`middle`, default `left`); `count` (`1`\|`2`, default `1`) | | `focus` | Give a window (and optionally a control in it) real **keyboard focus** so `send_command`/`chars` keystrokes reach it. Foregrounds the window, clicks the control (a real click focuses custom-drawn surfaces a bare `setfocus` misses; e.g. a MAUI GraphicsView), then **verifies**. Fails `foreground` if the window won't activate, `focus` if no control takes focus. Use before firing an app's own keyboard shortcut at a specific surface. | window target; optional `at` = `{ by, value }` or `{ x, y }` (omit to just foreground + confirm focus) | | `record` | Record a window or region to an **animated GIF** over time (start/stop or a bounded one-shot). | window target, or region `x`/`y`/`width`/`height`; `action` (`start`\|`stop`\|`oneshot`), `fps`, `durationMs`, `maxWidth`, `file` | diff --git a/src/Agent/AgentInstructions.md b/src/Agent/AgentInstructions.md index 473e464..bd1b190 100644 --- a/src/Agent/AgentInstructions.md +++ b/src/Agent/AgentInstructions.md @@ -31,7 +31,7 @@ For low-cost observation passes, downscale with `maxWidth` or `scale` and prefer `returnImage:false`) to get metadata + artifact path without inline base64 or an MCP image block. Browser chrome is UIA-targetable; in-page web content is not — verify field values with `get-text` on a window-relative region (`x`/`y`/`width`/`height` from `query` bounds) before reaching for `capture`; use -screen-absolute pixel `click` when needed (re-observe after layout shifts). `get-text` returns extracted +pixel `click` when needed (`click at:{x,y}` is window-relative to the captured window bounds when you set a window target, otherwise screen-absolute; re-observe after layout shifts). `get-text` returns extracted text plus line/word counts; `ocr-no-text` means OCR could not read the region (try a larger area or `capture`); `ocr-blank` means the pixels were a flat fill (like `capture-blank`). Use `capture` for a single state check when the tree and OCR can't answer; use `record` ONLY to show CHANGE @@ -52,7 +52,8 @@ raise `maxNodes` or target a deeper window so you don't reason over a partial tr non-fatal; errorCategory tells you how to recover. The bounds `query`/`find` report are ABSOLUTE screen pixels; the `displays` tool reports every monitor's pixel bounds and DPI/scale (and the union virtualBounds) so you can interpret those bounds across multiple/scaled monitors and place pixel clicks or -drags without measuring the screen yourself. +drags without measuring the screen yourself. Window-relative click/drag endpoints are interpreted in that +same per-monitor pixel space after adding the target window's top-left. 3. ACT: prefer `invoke` (by name/automationId/classname; action invoke|toggle|setvalue|setfocus|expand| collapse|select) over coordinate clicks; it is far more reliable. To put text in a native field, prefer @@ -71,16 +72,17 @@ that action; re-finding it will never help; pick a different action or `click` i expand|collapse|select). To DRAG (resize a window by its sizing border, move one by its title bar, drag a slider/handle, marquee-select, or reorder; there is no `invoke` for these), use the `drag` tool: give a `from` and a `to`, each either an element `{ by, value }` -in the target window (dragged from/to its centre) or an absolute screen pixel `{ x, y }`, plus optional +in the target window (dragged from/to its centre) or a pixel `{ x, y }` (window-relative when a +window target is set, otherwise screen-absolute), plus optional `path` waypoints for a curved or multi-stop drag. The whole press→move→release is dispatched ATOMICALLY, so prefer it over hand-rolling `mouse:lbd`/`mouse:mt`/`mouse:lbu` (which can interleave with other commands). -Coords are absolute screen pixels (the same space `query`/`find` bounds report), so you can drag straight +Without a window target, coords are absolute screen pixels (the same space `query`/`find` bounds report), so you can drag straight from one control's bounds to another's. Re-`query` afterward: a moved/resized window's controls are at new bounds. For window-level move/resize, use the `window` tool: `action:"move"`/`"resize"` with target coordinates and optional `animate:true` to make the window appear to be dragged rather than instantly teleported. `window` also handles `minimize`, `maximize`, `restore`, and `foreground`. -To CLICK a point `invoke` can't reach (a custom-drawn cell, a canvas/map coordinate, or a barepixel), use the `click` tool: give `at` as an element `{ by, value }` (clicked at its centre) or an -absolute screen pixel `{ x, y }`, with optional `button` (left|right|middle) and `count` +To CLICK a point `invoke` can't reach (a custom-drawn cell, a canvas/map coordinate, or a barepixel), use the `click` tool: give `at` as an element `{ by, value }` (clicked at its centre) or a +pixel `{ x, y }` (window-relative when a window target is set, otherwise absolute screen pixels), with optional `button` (left|right|middle) and `count` (2 = double-click); the move+click is dispatched atomically. Still prefer `invoke` for ordinary buttons and menu items; it doesn't depend on the control being on-screen and unobscured. System file dialogs (Open, Save Print Output As) are separate windows; `wait-for`/`query` by title, reuse the returned `handle`, and act on that diff --git a/src/Agent/ToolCatalog.cs b/src/Agent/ToolCatalog.cs index 11d8617..3ba46fb 100644 --- a/src/Agent/ToolCatalog.cs +++ b/src/Agent/ToolCatalog.cs @@ -433,8 +433,8 @@ private static JsonObject BuildInvokeSchema() { private static JsonObject BuildDragSchema() { JsonObject dragProps = WindowTargetProps(); - dragProps["from"] = EndpointSchema("Drag start: an element ({ by, value }) in the target window, or a pixel ({ x, y })."); - dragProps["to"] = EndpointSchema("Drag end: an element ({ by, value }) in the target window, or a pixel ({ x, y })."); + dragProps["from"] = EndpointSchema("Drag start: an element ({ by, value }) in the target window, or a pixel ({ x, y }). With a window target and a pixel endpoint, x/y are window-relative (window top-left + offset); without a window target they are absolute screen pixels."); + dragProps["to"] = EndpointSchema("Drag end: an element ({ by, value }) in the target window, or a pixel ({ x, y }). With a window target and a pixel endpoint, x/y are window-relative (window top-left + offset); without a window target they are absolute screen pixels."); dragProps["path"] = new JsonObject { ["type"] = "array", ["description"] = "Optional intermediate waypoints (absolute screen pixels) between from and to.", @@ -447,17 +447,17 @@ private static JsonObject BuildDragSchema() { }, }; return Tool("drag", - "Press → move along a path → release, dispatched atomically (no interleaving). Endpoints are an element (by/value, dragged from/to its centre) or an absolute screen pixel; add path waypoints for a curved/multi-stop drag. Covers window resize/move by chrome, sliders, marquee select, drag-reorder. Give a window target when either endpoint is an element.", + "Press → move along a path → release, dispatched atomically (no interleaving). Endpoints are an element (by/value, dragged from/to its centre) or a pixel: with a window target, pixel endpoints are window-relative (matching capture coordinates); without one, they are absolute screen pixels. Add path waypoints (always absolute pixels) for a curved/multi-stop drag. Covers window resize/move by chrome, sliders, marquee select, drag-reorder. Give a window target when either endpoint is an element.", dragProps, ["from", "to"]); } private static JsonObject BuildClickSchema() { JsonObject clickProps = WindowTargetProps(); - clickProps["at"] = EndpointSchema("Where to click: an element ({ by, value }) in the target window (its centre) or an absolute screen pixel ({ x, y })."); + clickProps["at"] = EndpointSchema("Where to click: an element ({ by, value }) in the target window (its centre) or a pixel ({ x, y }). With a window target and a pixel endpoint, x/y are window-relative (window top-left + offset); without a window target they are absolute screen pixels."); clickProps["button"] = PropSchema("string", "Button: left | right | middle (default left)"); clickProps["count"] = PropSchema("integer", "Click count: 1 = single, 2 = double (default 1)"); return Tool("click", - "Click at a point; an element (by/value, clicked at its centre) or an absolute screen pixel (the space query/find bounds report). Move+click is dispatched atomically. Prefer invoke for buttons/menus; use click for element types invoke cannot drive or when you must target a pixel. Give a window target when 'at' is an element.", + "Click at a point; an element (by/value, clicked at its centre) or a pixel endpoint. With a window target, pixel endpoints are window-relative (window top-left + offset, matching capture coordinates); without one they are absolute screen pixels (the space query/find bounds report). Move+click is dispatched atomically. Prefer invoke for buttons/menus; use click for element types invoke cannot drive or when you must target a pixel. Give a window target when 'at' is an element or when you want window-relative pixels.", clickProps, ["at"]); } diff --git a/src/Commands/ClickCommand.cs b/src/Commands/ClickCommand.cs index 2f57191..da32676 100644 --- a/src/Commands/ClickCommand.cs +++ b/src/Commands/ClickCommand.cs @@ -11,9 +11,10 @@ namespace MCEControl; /// /// Agent actuation command: a single mouse click at a target point (issue #122). The point is either an /// absolute screen pixel (x/y) or a UI Automation element in the target window (value, -/// resolved to the centre of its bounds); the same pixel space query/find report, so an -/// agent can click a control it just observed without converting to normalized coordinates itself. The -/// move-then-click is dispatched atomically by so it cannot +/// resolved to the centre of its bounds). When a window target is supplied and the endpoint is pixel +/// coordinates, x/y are interpreted as window-relative and translated to absolute +/// screen pixels via the target window's outer-rect origin. The move-then-click is dispatched atomically by +/// so it cannot /// interleave with another command's mouse input (the hazard #113 warns about when a caller hand-rolls /// mt/lbc). Gated by and audited /// (structurally, via ). Disabled by default (security). @@ -41,8 +42,9 @@ public static List BuiltInCommands { protected override string AuditDetails() => $"click at (by={By} value='{Value}' {X},{Y}) button={Button} count={Count} window handle={Handle} title='{Window}' process='{Process}'"; - // A window is only needed to resolve an element endpoint; a pure pixel click needs none. - protected override bool RequiresWindowTarget => !string.IsNullOrEmpty(Value); + // A window is needed for element endpoints, and also for pixel endpoints when a target selector was + // provided (pixels become window-relative in that case). + protected override bool RequiresWindowTarget => HasWindowTarget || !string.IsNullOrEmpty(Value); protected override CommandResult ExecuteCore(WindowInfo? target) { IntPtr hwnd = target is null ? IntPtr.Zero : new IntPtr(target.Handle); @@ -70,13 +72,26 @@ protected override CommandResult ExecuteCore(WindowInfo? target) { /// /// Resolves the click point to an absolute screen pixel: the centre of an element (/ /// ) when is set, else the literal (, - /// ). Returns false with a structured when the element - /// can't be resolved (not found, ambiguous, stale window, elevated target; #261). + /// ) when no window target is set, else the target window's outer-rect origin plus + /// (, ) when a window target is set. Returns false with a structured + /// when the element can't be resolved (not found, ambiguous, stale window, + /// elevated target; #261), or when a targeted window disappears before its origin can be read. /// private bool TryResolvePoint(IntPtr hwnd, out (int X, int Y) point, out CommandResult? failure) { failure = null; if (string.IsNullOrEmpty(Value)) { - point = (X, Y); + if (hwnd == IntPtr.Zero) { + point = (X, Y); + return true; + } + if (!TryGetWindowOrigin(hwnd, out (int X, int Y) origin)) { + point = default; + failure = CommandResult.Fail(Cmd, + "The target window disappeared before click coordinates could be resolved. Re-query and retry.", + "window-closed", "stale-element"); + return false; + } + point = OffsetByWindowOrigin((X, Y), origin); return true; } string by = string.IsNullOrEmpty(By) ? "name" : By; diff --git a/src/Commands/DragCommand.cs b/src/Commands/DragCommand.cs index 2e6886e..e4f58a5 100644 --- a/src/Commands/DragCommand.cs +++ b/src/Commands/DragCommand.cs @@ -12,7 +12,9 @@ namespace MCEControl; /// Agent actuation command: a first-class press → move-path → release drag (issue #123). Each endpoint /// is either an absolute screen pixel (fromX/fromY, toX/toY) or a UI Automation element in /// the target window (fromValue/toValue, resolved to its bounds' centre), with optional -/// intermediate waypoints (). The whole gesture is dispatched atomically by +/// intermediate waypoints (). When a window target is supplied and an endpoint is +/// pixel coordinates, that endpoint is interpreted as window-relative and translated to absolute +/// screen pixels via the target window's outer-rect origin. The whole gesture is dispatched atomically by /// so it cannot interleave with another command's mouse input /// (the hazard #113 warns about when a caller hand-rolls lbd/mt/lbu). Gated by /// and audited (structurally, via @@ -46,8 +48,9 @@ public static List BuiltInCommands { protected override string AuditDetails() => $"drag from (by={FromBy} value='{FromValue}' {FromX},{FromY}) to (by={ToBy} value='{ToValue}' {ToX},{ToY}) window handle={Handle} title='{Window}' process='{Process}'"; - // A window is only needed to resolve element endpoints; a pure coordinate drag needs none. - protected override bool RequiresWindowTarget => !string.IsNullOrEmpty(FromValue) || !string.IsNullOrEmpty(ToValue); + // A window is needed for element endpoints, and also for pixel endpoints when a target selector was + // provided (pixel endpoints become window-relative in that case). + protected override bool RequiresWindowTarget => HasWindowTarget || !string.IsNullOrEmpty(FromValue) || !string.IsNullOrEmpty(ToValue); protected override CommandResult ExecuteCore(WindowInfo? target) { IntPtr hwnd = target is null ? IntPtr.Zero : new IntPtr(target.Handle); @@ -79,16 +82,29 @@ protected override CommandResult ExecuteCore(WindowInfo? target) { /// /// Resolves one drag endpoint to an absolute screen pixel: an element (by/value, centre of its /// bounds) when is set, else the literal (, - /// ). Returns false with a structured when an - /// element endpoint has no window or can't be resolved (not found, ambiguous, stale window, - /// elevated target; #261). ("Drag start"/"Drag end") is prefixed onto - /// the detail of EVERY failure category (via ) so a two-endpoint drag - /// always says which end failed (#262 review). + /// ) when no window target is set, else the target window's outer-rect + /// offset plus (, ) when a window target is set. Returns + /// false with a structured when an element endpoint has no window or + /// can't be resolved (not found, ambiguous, stale window, elevated target; #261), or when a targeted + /// window disappears before its origin can be read. ("Drag + /// start"/"Drag end") is prefixed onto the detail of EVERY failure category (via + /// ) so a two-endpoint drag always says which end failed (#262 review). /// private static bool TryResolvePoint(IntPtr hwnd, string by, string value, int x, int y, string endpoint, string cmd, out (int X, int Y) point, out CommandResult? failure) { failure = null; if (string.IsNullOrEmpty(value)) { - point = (x, y); + if (hwnd == IntPtr.Zero) { + point = (x, y); + return true; + } + if (!TryGetWindowOrigin(hwnd, out (int X, int Y) origin)) { + point = default; + failure = LabelEndpoint(endpoint, CommandResult.Fail(cmd, + "The target window disappeared before drag coordinates could be resolved. Re-query and retry.", + "window-closed", "stale-element")); + return false; + } + point = OffsetByWindowOrigin((x, y), origin); return true; } if (hwnd == IntPtr.Zero) { diff --git a/src/Commands/WindowTargetingAgentCommand.cs b/src/Commands/WindowTargetingAgentCommand.cs index 3da5529..e7477d6 100644 --- a/src/Commands/WindowTargetingAgentCommand.cs +++ b/src/Commands/WindowTargetingAgentCommand.cs @@ -1,6 +1,7 @@ // Copyright © Kindel, LLC - http://www.kindel.com // Published under the MIT License - Source on GitHub: https://github.com/tig/mcec +using System; using System.Xml.Serialization; namespace MCEControl; @@ -115,6 +116,29 @@ protected sealed override CommandResult ExecuteCore() { _ => null, }; + /// + /// Resolves a window's outer-rect origin in absolute screen pixels. Uses + /// so coordinate endpoints inferred from a window + /// capture can be replayed without hand-adding monitor offsets. + /// + protected static bool TryGetWindowOrigin(IntPtr hwnd, out (int X, int Y) origin) { + origin = default; + if (hwnd == IntPtr.Zero) { + return false; + } + + if (!AgentNativeMethods.GetWindowRect(hwnd, out NativeRect rect)) { + return false; + } + + origin = (rect.Left, rect.Top); + return true; + } + + /// Adds a window origin to a window-relative point to get an absolute screen pixel. + protected static (int X, int Y) OffsetByWindowOrigin((int X, int Y) point, (int X, int Y) windowOrigin) => + (windowOrigin.X + point.X, windowOrigin.Y + point.Y); + // No Clone override: the shared selectors are value/string-typed, so the base // MemberwiseClone-based Command.Clone (#207) copies them by construction. } diff --git a/tests/MCEControl.xUnit/Commands/UiaFailureProbeCommand.cs b/tests/MCEControl.xUnit/Commands/UiaFailureProbeCommand.cs index 31a57f4..9bd6cac 100644 --- a/tests/MCEControl.xUnit/Commands/UiaFailureProbeCommand.cs +++ b/tests/MCEControl.xUnit/Commands/UiaFailureProbeCommand.cs @@ -17,4 +17,7 @@ public sealed class UiaFailureProbeCommand : WindowTargetingAgentCommand { UiaFindFailureFor(cmd, by, value, outcome); public static CommandResult? Failure(string cmd, UiaFailureKind kind) => UiaFailureFor(cmd, kind); + + public static (int X, int Y) OffsetPoint((int X, int Y) point, (int X, int Y) origin) => + OffsetByWindowOrigin(point, origin); } diff --git a/tests/MCEControl.xUnit/Commands/WindowTargetingAgentCommandTests.cs b/tests/MCEControl.xUnit/Commands/WindowTargetingAgentCommandTests.cs index cf0da4b..9fe8b5b 100644 --- a/tests/MCEControl.xUnit/Commands/WindowTargetingAgentCommandTests.cs +++ b/tests/MCEControl.xUnit/Commands/WindowTargetingAgentCommandTests.cs @@ -53,4 +53,18 @@ public void UiaFindFailureFor_FoundOrCleanMiss_IsNoFailure() { Assert.Null(UiaFailureProbeCommand.FindFailure("find", "name", "OK", found)); Assert.Null(UiaFailureProbeCommand.FindFailure("find", "name", "OK", UiaFindOutcome.NotFound)); } + + [Fact] + public void OffsetByWindowOrigin_AddsWindowOriginToWindowRelativePoint() { + (int X, int Y) point = UiaFailureProbeCommand.OffsetPoint((412, 88), (1923, 45)); + + Assert.Equal((2335, 133), point); + } + + [Fact] + public void OffsetByWindowOrigin_PreservesNegativeOffsets() { + (int X, int Y) point = UiaFailureProbeCommand.OffsetPoint((-8, 0), (3200, 120)); + + Assert.Equal((3192, 120), point); + } } diff --git a/tests/MCEControl.xUnit/Services/AgentServerTests.cs b/tests/MCEControl.xUnit/Services/AgentServerTests.cs index 3f66a55..2805e1c 100644 --- a/tests/MCEControl.xUnit/Services/AgentServerTests.cs +++ b/tests/MCEControl.xUnit/Services/AgentServerTests.cs @@ -195,6 +195,7 @@ public void Dispatch_ToolsList_ClickTool_DeclaresAtEndpoint() { Assert.True(props.ContainsKey("at")); Assert.True(props.ContainsKey("button")); Assert.True(props.ContainsKey("count")); + Assert.Contains("window-relative", props["at"]!["description"]!.GetValue(), StringComparison.Ordinal); List required = []; foreach (JsonNode? r in schema["required"]!.AsArray()) { @@ -352,6 +353,8 @@ public void Dispatch_ToolsList_DragTool_DeclaresFromToEndpoints() { Assert.True(props.ContainsKey("from")); Assert.True(props.ContainsKey("to")); Assert.True(props.ContainsKey("path")); + Assert.Contains("window-relative", props["from"]!["description"]!.GetValue(), StringComparison.Ordinal); + Assert.Contains("window-relative", props["to"]!["description"]!.GetValue(), StringComparison.Ordinal); List required = []; foreach (JsonNode? r in schema["required"]!.AsArray()) {