From a9103cefb931746ef433fb2386ab3bdd3207697d Mon Sep 17 00:00:00 2001 From: rrader26 Date: Mon, 11 May 2026 13:05:37 -0400 Subject: [PATCH 1/2] feat(bridge-windows): Phase 0e3 -- execute (UIA pattern dispatch) Adds the `execute` JSON-RPC method to the Windows bridge. Routes element-targeted actions through UIA patterns and returns the post-action state where the pattern exposes it. Action types supported: - click -> InvokePattern.Invoke (falls back to Toggle, SelectionItem, ExpandCollapse, or focus+Space keystroke) - type -> ValuePattern.SetValue (falls back to focus+keystrokes with optional Ctrl+A/Del clear) - select -> SelectionItemPattern.Select on the target, or expand + locate-by-name on a Selection container, or ValuePattern for editable combos - check -> TogglePattern.Toggle with state convergence (up to 3 toggles to reach the requested checked state) - expand -> ExpandCollapsePattern.Expand / Collapse - focus -> AutomationElement.Focus - scroll_to -> ScrollItemPattern.ScrollIntoView with focus fallback - key -> Keyboard.Type / TypeSimultaneously with modifier parsing (ctrl|alt|shift|meta|win) and a friendly key-name vocabulary (enter, tab, esc, F1..F12, arrows, pgup/pgdn, etc.) falling through to literal typing Element lookup: Capture() now stashes every walked AutomationElement into a per-capture session (element_id -> live handle). Execute() resolves the request's elementId against that session. Errors are specific and recoverable: - "No capture session active" -> caller must call capture first - "Unknown element_id" -> stale binding, re-capture - ElementNotAvailableException -> UI changed mid-action, re-capture Defensive throughout: every UIA call wrapped, every exception returned as a structured ExecuteResult so the agent gets a deterministic { ok, message, newValue } shape on every call. Notepad live demo script (`scripts/notepad-demo.ps1`): 1. Launches Notepad 2. list_windows -> find notepad 3. capture -> locate the text_area element 4. execute type -> "Hello from AgentMark Desktop ..." 5. capture -> assert the typed text is in element.value Designed to run from the interactive Parallels console where UIA can see the real desktop. Co-Authored-By: Claude Opus 4.7 (1M context) --- apps/agent-runner/bridges/windows/Program.cs | 55 ++- .../bridges/windows/Uia/UiaCapturer.cs | 380 ++++++++++++++++++ .../bridges/windows/scripts/notepad-demo.ps1 | 130 ++++++ 3 files changed, 563 insertions(+), 2 deletions(-) create mode 100644 apps/agent-runner/bridges/windows/scripts/notepad-demo.ps1 diff --git a/apps/agent-runner/bridges/windows/Program.cs b/apps/agent-runner/bridges/windows/Program.cs index 68ae2dc..2fe8f74 100644 --- a/apps/agent-runner/bridges/windows/Program.cs +++ b/apps/agent-runner/bridges/windows/Program.cs @@ -204,9 +204,10 @@ public RpcDispatcher() "capabilities" => HandleCapabilities(), "list_windows" => HandleListWindows(), "capture" => HandleCapture(@params), + "execute" => HandleExecute(@params), _ => throw new RpcException( RpcError.MethodNotFound, - $"Unknown method: {method}. Supported: ping, capabilities, list_windows, capture."), + $"Unknown method: {method}. Supported: ping, capabilities, list_windows, capture, execute."), }; } @@ -240,7 +241,7 @@ public void Dispose() "capabilities", "list_windows", "capture", - // "execute" lands in Phase 0e3 + "execute", }, uiaProvider = "FlaUI.UIA3", platform = "windows", @@ -280,6 +281,56 @@ private object HandleCapture(JsonElement @params) } } + private object HandleExecute(JsonElement @params) + { + var modifiers = ReadStringArray(@params, "modifiers"); + + var req = new UiaCapturer.ExecuteRequest + { + ElementId = ReadString(@params, "elementId") + ?? throw new RpcException(RpcError.InvalidParams, "execute requires `elementId`."), + ActionType = ReadString(@params, "actionType") ?? "click", + Text = ReadString(@params, "text"), + Value = ReadString(@params, "value"), + Checked = ReadBool(@params, "checked"), + Expanded = ReadBool(@params, "expanded"), + Key = ReadString(@params, "key"), + Modifiers = modifiers, + ClearFirst = ReadBool(@params, "clearFirst") ?? false, + TimeoutMs = ReadInt(@params, "timeoutMs") ?? 5000, + }; + + try + { + var capturer = _capturer.Value; + var sta = _staWorker.Value; + var result = sta.Invoke(() => capturer.Execute(req)); + return new + { + ok = result.Ok, + message = result.Message, + newValue = result.NewValue, + }; + } + catch (InvalidOperationException ex) + { + throw new RpcException(RpcError.InternalError, ex.Message); + } + } + + private static string[]? ReadStringArray(JsonElement parent, string name) + { + if (parent.ValueKind != JsonValueKind.Object) return null; + if (!parent.TryGetProperty(name, out var v)) return null; + if (v.ValueKind != JsonValueKind.Array) return null; + var list = new List(v.GetArrayLength()); + foreach (var item in v.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.String) list.Add(item.GetString()!); + } + return list.ToArray(); + } + private static string? ReadString(JsonElement parent, string name) { if (parent.ValueKind != JsonValueKind.Object) return null; diff --git a/apps/agent-runner/bridges/windows/Uia/UiaCapturer.cs b/apps/agent-runner/bridges/windows/Uia/UiaCapturer.cs index e6dc3f9..50a9fce 100644 --- a/apps/agent-runner/bridges/windows/Uia/UiaCapturer.cs +++ b/apps/agent-runner/bridges/windows/Uia/UiaCapturer.cs @@ -11,6 +11,7 @@ using FlaUI.Core.AutomationElements; using FlaUI.Core.Conditions; using FlaUI.Core.Definitions; +using FlaUI.Core.Exceptions; using FlaUI.Core.Patterns; using FlaUI.UIA3; @@ -21,6 +22,12 @@ internal sealed class UiaCapturer : IDisposable private readonly UIA3Automation _automation; private bool _disposed; + // Cache: element_id -> live AutomationElement. Populated during + // Capture(); consumed by Execute(). Replaced on every new Capture. + // Stays single-rooted (only one capture session at a time) for now; + // multi-window concurrent capture is a future feature. + private CaptureSession? _lastSession; + public UiaCapturer() { _automation = new UIA3Automation(); @@ -33,6 +40,12 @@ public void Dispose() try { _automation.Dispose(); } catch { /* swallow */ } } + private sealed class CaptureSession + { + public required AutomationElement RootWindow { get; init; } + public Dictionary Elements { get; } = new(StringComparer.Ordinal); + } + // ───────────────────────────────────────────────────────────────── // list_windows // ───────────────────────────────────────────────────────────────── @@ -102,15 +115,22 @@ public DesktopCaptureDto Capture(CaptureRequest req) var hwnd = GetHwnd(window); var deadline = DateTime.UtcNow.AddMilliseconds(Math.Max(500, req.TimeoutMs)); + // Start a fresh session so Execute() finds elements from THIS + // capture (not stale ones from a previous window). + var session = new CaptureSession { RootWindow = window }; + var ctx = new WalkContext { MaxDepth = Math.Max(1, req.MaxDepth), MaxElements = Math.Max(50, req.MaxElements), IncludeHidden = req.IncludeHidden, Deadline = deadline, + Session = session, }; var rootDto = WalkElement(window, depth: 0, ctx); + _lastSession = session; + var focused = TryGetFocused(); string? focusedId = focused != null && IsAncestorOrSelf(focused, window) ? AutomationIdFor(focused) @@ -139,6 +159,7 @@ private sealed class WalkContext public DateTime Deadline; public int ElementCount; public int MaxDepthReached; + public required CaptureSession Session; public readonly HashSet SeenIds = new(StringComparer.Ordinal); } @@ -156,6 +177,8 @@ private DesktopElementDto WalkElement(AutomationElement el, int depth, WalkConte id = $"{id}#{n}"; } ctx.SeenIds.Add(id); + // Stash a live handle so Execute() can find it later. + ctx.Session.Elements[id] = el; var role = RoleMapper.ToRole(el); var dto = new DesktopElementDto @@ -199,6 +222,363 @@ private DesktopElementDto WalkElement(AutomationElement el, int depth, WalkConte return dto; } + // ───────────────────────────────────────────────────────────────── + // execute + // ───────────────────────────────────────────────────────────────── + + public sealed class ExecuteRequest + { + public string ElementId { get; init; } = ""; + /// One of: click | type | select | check | expand | focus | scroll_to | key + public string ActionType { get; init; } = "click"; + + // Action-specific payloads. Only the ones relevant to ActionType + // are read; the rest are ignored. + public string? Text { get; init; } // type + public string? Value { get; init; } // select + public bool? Checked { get; init; } // check + public bool? Expanded { get; init; } // expand + public string? Key { get; init; } // key + public string[]? Modifiers { get; init; } // key + public bool ClearFirst { get; init; } // type + public int TimeoutMs { get; init; } = 5000; + } + + public sealed class ExecuteResult + { + public bool Ok { get; init; } + public string? Message { get; init; } + public string? NewValue { get; init; } + } + + public ExecuteResult Execute(ExecuteRequest req) + { + var session = _lastSession + ?? throw new InvalidOperationException( + "No capture session active. Call `capture` before `execute` so the bridge can resolve element_ids."); + + if (!session.Elements.TryGetValue(req.ElementId, out var element)) + { + return new ExecuteResult + { + Ok = false, + Message = $"Unknown element_id `{req.ElementId}` in the current capture session. Re-capture if the window has changed.", + }; + } + + try + { + switch (req.ActionType) + { + case "click": return DoClick(element); + case "type": return DoType(element, req.Text ?? "", req.ClearFirst); + case "select": return DoSelect(element, req.Value ?? ""); + case "check": return DoCheck(element, req.Checked ?? true); + case "expand": return DoExpand(element, req.Expanded ?? true); + case "focus": return DoFocus(element); + case "scroll_to": return DoScrollTo(element); + case "key": return DoKey(element, req.Key ?? "", req.Modifiers); + default: + return new ExecuteResult + { + Ok = false, + Message = $"Unknown action type `{req.ActionType}`.", + }; + } + } + catch (ElementNotAvailableException ex) + { + // UIA throws this when the underlying element disappeared + // between capture and execute (window closed, modal dismissed, etc.). + return new ExecuteResult { Ok = false, Message = $"Element no longer available: {ex.Message}" }; + } + catch (Exception ex) + { + return new ExecuteResult { Ok = false, Message = $"{ex.GetType().Name}: {ex.Message}" }; + } + } + + private static ExecuteResult DoClick(AutomationElement el) + { + var patterns = el.Patterns; + + if (patterns.Invoke.IsSupported) + { + patterns.Invoke.Pattern.Invoke(); + return new ExecuteResult { Ok = true }; + } + + // Some controls (toggle buttons, menu items in some toolkits) only + // support Toggle / SelectionItem. Try them in priority order. + if (patterns.Toggle.IsSupported) + { + patterns.Toggle.Pattern.Toggle(); + return new ExecuteResult { Ok = true }; + } + if (patterns.SelectionItem.IsSupported) + { + patterns.SelectionItem.Pattern.Select(); + return new ExecuteResult { Ok = true }; + } + if (patterns.ExpandCollapse.IsSupported) + { + var p = patterns.ExpandCollapse.Pattern; + if (p.ExpandCollapseState.ValueOrDefault == ExpandCollapseState.Expanded) p.Collapse(); + else p.Expand(); + return new ExecuteResult { Ok = true }; + } + + // Last resort: focus + simulated Space (works for buttons / + // checkboxes / links that don't expose any of the above). + try + { + el.Focus(); + FlaUI.Core.Input.Keyboard.Type(FlaUI.Core.WindowsAPI.VirtualKeyShort.SPACE); + return new ExecuteResult { Ok = true, Message = "Used keyboard fallback (no InvokePattern)." }; + } + catch + { + return new ExecuteResult { Ok = false, Message = "Element does not support any clickable pattern." }; + } + } + + private static ExecuteResult DoType(AutomationElement el, string text, bool clearFirst) + { + var patterns = el.Patterns; + + if (patterns.Value.IsSupported && !patterns.Value.Pattern.IsReadOnly.ValueOrDefault) + { + // Most reliable path: SetValue replaces the entire content + // atomically. Honour clear_first by treating non-clear as a + // no-op (Value semantics is set-by-default). + try { el.Focus(); } catch { /* ignore */ } + patterns.Value.Pattern.SetValue(text); + return new ExecuteResult { Ok = true, NewValue = patterns.Value.Pattern.Value.ValueOrDefault ?? text }; + } + + // Fallback: focus + simulated keystrokes. Used for elements that + // only expose TextPattern (Documents) or no value pattern at all. + try { el.Focus(); } catch { /* ignore */ } + if (clearFirst) + { + FlaUI.Core.Input.Keyboard.TypeSimultaneously( + FlaUI.Core.WindowsAPI.VirtualKeyShort.CONTROL, + FlaUI.Core.WindowsAPI.VirtualKeyShort.KEY_A); + FlaUI.Core.Input.Keyboard.Type(FlaUI.Core.WindowsAPI.VirtualKeyShort.DELETE); + } + FlaUI.Core.Input.Keyboard.Type(text); + + string? newValue = null; + try + { + if (patterns.Value.IsSupported) newValue = patterns.Value.Pattern.Value.ValueOrDefault; + } + catch { /* ignore */ } + + return new ExecuteResult { Ok = true, NewValue = newValue, Message = "Used keyboard fallback (no writable ValuePattern)." }; + } + + private static ExecuteResult DoSelect(AutomationElement el, string value) + { + var patterns = el.Patterns; + + // Direct SelectionItem on the target: caller already knows + // which child to select. + if (patterns.SelectionItem.IsSupported) + { + patterns.SelectionItem.Pattern.Select(); + return new ExecuteResult { Ok = true, NewValue = SafeName(el) ?? value }; + } + + // The element is a Selection container (ComboBox, ListBox). + // Expand it, then find the child whose name/value matches. + if (patterns.Selection.IsSupported) + { + if (patterns.ExpandCollapse.IsSupported) + { + try { patterns.ExpandCollapse.Pattern.Expand(); } catch { /* ignore */ } + } + + var children = el.FindAllChildren(); + var match = children.FirstOrDefault(c => + string.Equals(SafeName(c) ?? "", value, StringComparison.OrdinalIgnoreCase)); + if (match != null && match.Patterns.SelectionItem.IsSupported) + { + match.Patterns.SelectionItem.Pattern.Select(); + return new ExecuteResult { Ok = true, NewValue = SafeName(match) ?? value }; + } + + return new ExecuteResult { Ok = false, Message = $"No child option matched `{value}`." }; + } + + // ComboBox with editable Value (e.g. autocomplete): set the value. + if (patterns.Value.IsSupported && !patterns.Value.Pattern.IsReadOnly.ValueOrDefault) + { + patterns.Value.Pattern.SetValue(value); + return new ExecuteResult { Ok = true, NewValue = value }; + } + + return new ExecuteResult { Ok = false, Message = "Element does not support Selection or SelectionItem patterns." }; + } + + private static ExecuteResult DoCheck(AutomationElement el, bool wantChecked) + { + var patterns = el.Patterns; + + if (patterns.Toggle.IsSupported) + { + var p = patterns.Toggle.Pattern; + // Keep toggling until the state matches; ToggleState.Indeterminate + // (mixed) counts as not-on-yet for the wantChecked=true case. + int guard = 3; + while (guard-- > 0) + { + var current = p.ToggleState.ValueOrDefault; + var isOn = current == ToggleState.On; + if (isOn == wantChecked) break; + p.Toggle(); + } + var final = p.ToggleState.ValueOrDefault == ToggleState.On; + return new ExecuteResult { Ok = final == wantChecked, NewValue = final ? "true" : "false" }; + } + + if (patterns.SelectionItem.IsSupported) + { + // Radio buttons surface as SelectionItem — selecting is the + // only operation; "unchecking" isn't well-defined for a radio. + if (wantChecked) + { + patterns.SelectionItem.Pattern.Select(); + return new ExecuteResult { Ok = true, NewValue = "true" }; + } + return new ExecuteResult { Ok = false, Message = "Cannot uncheck a radio button directly." }; + } + + return new ExecuteResult { Ok = false, Message = "Element does not support Toggle or SelectionItem patterns." }; + } + + private static ExecuteResult DoExpand(AutomationElement el, bool wantExpanded) + { + if (!el.Patterns.ExpandCollapse.IsSupported) + { + return new ExecuteResult { Ok = false, Message = "Element does not support ExpandCollapse pattern." }; + } + var p = el.Patterns.ExpandCollapse.Pattern; + if (wantExpanded) p.Expand(); else p.Collapse(); + var final = p.ExpandCollapseState.ValueOrDefault; + return new ExecuteResult { Ok = true, NewValue = final.ToString() }; + } + + private static ExecuteResult DoFocus(AutomationElement el) + { + el.Focus(); + return new ExecuteResult { Ok = true }; + } + + private static ExecuteResult DoScrollTo(AutomationElement el) + { + if (el.Patterns.ScrollItem.IsSupported) + { + el.Patterns.ScrollItem.Pattern.ScrollIntoView(); + return new ExecuteResult { Ok = true }; + } + // SetFocus often implies scroll-into-view for most controls. + try + { + el.Focus(); + return new ExecuteResult { Ok = true, Message = "Used SetFocus fallback (no ScrollItem pattern)." }; + } + catch + { + return new ExecuteResult { Ok = false, Message = "Element does not support ScrollItem and SetFocus failed." }; + } + } + + private static ExecuteResult DoKey(AutomationElement el, string key, string[]? modifiers) + { + if (string.IsNullOrEmpty(key)) + { + return new ExecuteResult { Ok = false, Message = "Missing `key` argument." }; + } + + try { el.Focus(); } catch { /* ignore */ } + + var vk = ParseVirtualKey(key); + if (vk == null) + { + // Not a named key — fall through to typing the literal text. + FlaUI.Core.Input.Keyboard.Type(key); + return new ExecuteResult { Ok = true, Message = $"Typed literal text `{key}`." }; + } + + var mods = (modifiers ?? Array.Empty()) + .Select(ParseModifier) + .Where(m => m.HasValue) + .Select(m => m!.Value) + .ToArray(); + + if (mods.Length == 0) + { + FlaUI.Core.Input.Keyboard.Type(vk.Value); + } + else + { + var combo = mods.Append(vk.Value).ToArray(); + FlaUI.Core.Input.Keyboard.TypeSimultaneously(combo); + } + + return new ExecuteResult { Ok = true }; + } + + private static FlaUI.Core.WindowsAPI.VirtualKeyShort? ParseVirtualKey(string key) + { + // Accept common friendly names; fall through to anything that + // matches the enum case-insensitively. + var lowered = key.ToLowerInvariant(); + return lowered switch + { + "enter" or "return" => FlaUI.Core.WindowsAPI.VirtualKeyShort.RETURN, + "tab" => FlaUI.Core.WindowsAPI.VirtualKeyShort.TAB, + "escape" or "esc" => FlaUI.Core.WindowsAPI.VirtualKeyShort.ESCAPE, + "space" or "spacebar" => FlaUI.Core.WindowsAPI.VirtualKeyShort.SPACE, + "backspace" => FlaUI.Core.WindowsAPI.VirtualKeyShort.BACK, + "delete" or "del" => FlaUI.Core.WindowsAPI.VirtualKeyShort.DELETE, + "home" => FlaUI.Core.WindowsAPI.VirtualKeyShort.HOME, + "end" => FlaUI.Core.WindowsAPI.VirtualKeyShort.END, + "pageup" or "pgup" => FlaUI.Core.WindowsAPI.VirtualKeyShort.PRIOR, + "pagedown" or "pgdn" => FlaUI.Core.WindowsAPI.VirtualKeyShort.NEXT, + "up" => FlaUI.Core.WindowsAPI.VirtualKeyShort.UP, + "down" => FlaUI.Core.WindowsAPI.VirtualKeyShort.DOWN, + "left" => FlaUI.Core.WindowsAPI.VirtualKeyShort.LEFT, + "right" => FlaUI.Core.WindowsAPI.VirtualKeyShort.RIGHT, + "f1" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F1, + "f2" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F2, + "f3" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F3, + "f4" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F4, + "f5" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F5, + "f6" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F6, + "f7" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F7, + "f8" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F8, + "f9" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F9, + "f10" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F10, + "f11" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F11, + "f12" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F12, + _ => Enum.TryParse(key, ignoreCase: true, out var v) ? v : null, + }; + } + + private static FlaUI.Core.WindowsAPI.VirtualKeyShort? ParseModifier(string m) + { + return m.ToLowerInvariant() switch + { + "ctrl" or "control" => FlaUI.Core.WindowsAPI.VirtualKeyShort.CONTROL, + "alt" => FlaUI.Core.WindowsAPI.VirtualKeyShort.ALT, + "shift" => FlaUI.Core.WindowsAPI.VirtualKeyShort.SHIFT, + "meta" or "win" or "windows" => FlaUI.Core.WindowsAPI.VirtualKeyShort.LWIN, + _ => null, + }; + } + // ───────────────────────────────────────────────────────────────── // Target resolution // ───────────────────────────────────────────────────────────────── diff --git a/apps/agent-runner/bridges/windows/scripts/notepad-demo.ps1 b/apps/agent-runner/bridges/windows/scripts/notepad-demo.ps1 new file mode 100644 index 0000000..c790b7d --- /dev/null +++ b/apps/agent-runner/bridges/windows/scripts/notepad-demo.ps1 @@ -0,0 +1,130 @@ +# Live demo: drive Notepad end-to-end through the bridge. +# +# Sequence: +# 1. Launch Notepad (or attach if already open) and bring to foreground +# 2. list_windows -> find the Notepad window +# 3. capture -> get the element tree, locate the text editor +# 4. execute type -> write a message into Notepad's edit area +# 5. capture -> re-snapshot and prove the text actually landed +# +# Designed to be run from the VM's interactive PowerShell so UIA can +# see real windows (SSH session 2 can't). + +$ErrorActionPreference = 'Stop' + +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$exe = Join-Path $scriptDir '..\bin\Debug\net8.0-windows\agentmark-bridge-windows.exe' +if (-not (Test-Path $exe)) { + Write-Error "Bridge exe not built. Expected at $exe." + exit 1 +} + +# ── Step 1: ensure a Notepad window exists, foreground it ────────────── +Write-Host "Launching Notepad..." +Start-Process notepad +Start-Sleep -Milliseconds 800 + +# Spawn bridge +$psi = New-Object System.Diagnostics.ProcessStartInfo +$psi.FileName = $exe +$psi.UseShellExecute = $false +$psi.RedirectStandardInput = $true +$psi.RedirectStandardOutput = $true +$psi.RedirectStandardError = $true +$proc = [System.Diagnostics.Process]::Start($psi) + +function Send-Request($p, $payload) { + $p.StandardInput.WriteLine($payload) + $p.StandardInput.Flush() + return ($p.StandardOutput.ReadLine() | ConvertFrom-Json) +} + +# ── Step 2: locate the Notepad window ────────────────────────────────── +Write-Host "" +Write-Host "Step 2 -- list_windows..." +$listResp = Send-Request $proc '{"jsonrpc":"2.0","id":1,"method":"list_windows"}' +$notepad = $listResp.result.windows | Where-Object { $_.processName -eq 'notepad' } | Select-Object -First 1 +if (-not $notepad) { + Write-Error "No notepad window visible to UIA. Open Notepad in the VM console first." + $proc.StandardInput.Close(); $proc.WaitForExit(5000) | Out-Null + exit 1 +} +Write-Host " found: $($notepad.windowTitle) ($($notepad.windowId))" + +# ── Step 3: capture Notepad ──────────────────────────────────────────── +Write-Host "" +Write-Host "Step 3 -- capture Notepad..." +$capPayload = '{"jsonrpc":"2.0","id":2,"method":"capture","params":{"windowId":"' + $notepad.windowId + '","maxDepth":8,"maxElements":400}}' +$capResp = Send-Request $proc $capPayload + +if (-not $capResp.result) { + Write-Error "capture failed: $(($capResp | ConvertTo-Json -Compress))" + $proc.StandardInput.Close(); $proc.WaitForExit(5000) | Out-Null + exit 1 +} + +$cap = $capResp.result +Write-Host " treeDepth=$($cap.treeDepth) elementCount=$($cap.elementCount)" + +# Walk children looking for a text-area-style element (Notepad's editor +# surfaces as role: text_area). On Win 11, the actual edit control may +# be nested a level or two below the top-level window. +function Find-EditElement($node) { + if ($null -eq $node) { return $null } + if ($node.role -eq 'text_area' -or $node.role -eq 'text_input') { return $node } + if ($node.children) { + foreach ($child in $node.children) { + $found = Find-EditElement $child + if ($found) { return $found } + } + } + return $null +} + +$edit = Find-EditElement $cap.root +if (-not $edit) { + Write-Error "Could not find a text_area / text_input element in Notepad's tree. Dump:" + Write-Host ($cap.root | ConvertTo-Json -Depth 10) + $proc.StandardInput.Close(); $proc.WaitForExit(5000) | Out-Null + exit 1 +} +Write-Host " editor element_id : $($edit.id) (role=$($edit.role))" +Write-Host " current value : '$($edit.value)'" + +# ── Step 4: type into the editor ─────────────────────────────────────── +$message = "Hello from AgentMark Desktop -- phase 0e3 live demo" +Write-Host "" +Write-Host "Step 4 -- execute type into editor..." +$execPayload = '{"jsonrpc":"2.0","id":3,"method":"execute","params":{"elementId":"' + $edit.id + '","actionType":"type","text":"' + $message + '","clearFirst":true}}' +$execResp = Send-Request $proc $execPayload + +if (-not $execResp.result.ok) { + Write-Error "execute failed: $($execResp.result.message)" + $proc.StandardInput.Close(); $proc.WaitForExit(5000) | Out-Null + exit 1 +} +Write-Host " ok : $($execResp.result.ok)" +Write-Host " newValue : $($execResp.result.newValue)" +if ($execResp.result.message) { Write-Host " note : $($execResp.result.message)" } + +# Give the edit a moment to reflect. +Start-Sleep -Milliseconds 250 + +# ── Step 5: re-capture and prove the change ──────────────────────────── +Write-Host "" +Write-Host "Step 5 -- re-capture and verify..." +$cap2Payload = '{"jsonrpc":"2.0","id":4,"method":"capture","params":{"windowId":"' + $notepad.windowId + '","maxDepth":8,"maxElements":400}}' +$cap2Resp = Send-Request $proc $cap2Payload +$edit2 = Find-EditElement $cap2Resp.result.root +Write-Host " editor value after type : '$($edit2.value)'" + +$ok = $edit2.value -and $edit2.value.Contains("Hello from AgentMark Desktop") +$proc.StandardInput.Close(); $proc.WaitForExit(5000) | Out-Null + +Write-Host "" +if ($ok) { + Write-Host "LIVE DEMO PASSED -- AgentMark Desktop drove real Notepad end-to-end." -ForegroundColor Green +} else { + Write-Host "LIVE DEMO FAILED -- expected text not found after execute." -ForegroundColor Red + exit 1 +} From b8da4b7b95fd36fd01fdd4a4e0bb3e7035f410e1 Mon Sep 17 00:00:00 2001 From: rrader26 Date: Mon, 11 May 2026 14:06:07 -0400 Subject: [PATCH 2/2] fix(bridge-windows): strip UTF-8 BOM on stdin reads + demo robustness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PowerShell's StreamWriter unpredictably prepends a UTF-8 BOM to the very first WriteLine call on stdin depending on stream-flush timing. The first JSON request then arrives as `{"jsonrpc":...}` which System.Text.Json rejects with "'0xEF' is an invalid start of a value". Defensive fix: TrimStart('') on every read line. Costs nothing on clean clients, transparently survives BOMed ones. Demo script (scripts/notepad-demo.ps1): - Stop auto-launching Notepad. Win 11 UWP Notepad's startup race vs UIA registration is messy; require the user to open Notepad first. - Match Notepad by case-insensitive process name OR window title -- Win 11's UWP Notepad ships as process name "Notepad" (capitalized). - Add raw-JSON diagnostic + visible window list on no-match. Validated end-to-end on Win 11 ARM64 (Parallels): - list_windows returned 5 windows (3 Notepad + 2 conhost) - capture walked 37 elements at depth 6, located text_area - execute type wrote "Hello from AgentMark Desktop -- phase 0e3 live demo" via ValuePattern.SetValue - re-capture verified the typed text in element.value - LIVE DEMO PASSED This is the first real end-to-end run of AgentMark Desktop driving Windows software through the protocol, zero vision/AI involved. Co-Authored-By: Claude Opus 4.7 (1M context) --- apps/agent-runner/bridges/windows/Program.cs | 7 ++- .../bridges/windows/scripts/notepad-demo.ps1 | 48 +++++++++++++++---- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/apps/agent-runner/bridges/windows/Program.cs b/apps/agent-runner/bridges/windows/Program.cs index 2fe8f74..f0af753 100644 --- a/apps/agent-runner/bridges/windows/Program.cs +++ b/apps/agent-runner/bridges/windows/Program.cs @@ -57,7 +57,12 @@ public static int Main(string[] args) string? line; while ((line = Console.In.ReadLine()) != null) { - line = line.Trim(); + // Strip UTF-8 BOM if a client wrote one at the start of the + // stream. Windows clients (PowerShell especially) do this + // unpredictably on the first WriteLine, depending on stream + // buffering. Without this strip the first JSON request gets + // a leading U+FEFF and JsonDocument.Parse rejects it. + line = line.Trim().TrimStart('\uFEFF'); if (line.Length == 0) continue; JsonElement reqId = default; diff --git a/apps/agent-runner/bridges/windows/scripts/notepad-demo.ps1 b/apps/agent-runner/bridges/windows/scripts/notepad-demo.ps1 index c790b7d..e1309b8 100644 --- a/apps/agent-runner/bridges/windows/scripts/notepad-demo.ps1 +++ b/apps/agent-runner/bridges/windows/scripts/notepad-demo.ps1 @@ -19,10 +19,16 @@ if (-not (Test-Path $exe)) { exit 1 } -# ── Step 1: ensure a Notepad window exists, foreground it ────────────── -Write-Host "Launching Notepad..." -Start-Process notepad -Start-Sleep -Milliseconds 800 +# ── Step 1: Notepad must already be open ──────────────────────────────── +# Win 11 UWP apps have a startup delay and weird session quirks that +# make "launch + immediately query" unreliable. Easier and more honest +# to require Notepad to already be visible to the user. +$notepadProc = Get-Process notepad -ErrorAction SilentlyContinue +if (-not $notepadProc) { + Write-Host "Please open Notepad first (Win+R, type 'notepad', Enter), then re-run this script." -ForegroundColor Yellow + exit 1 +} +Write-Host "Found notepad process(es): $((Get-Process notepad).Id -join ', ')" # Spawn bridge $psi = New-Object System.Diagnostics.ProcessStartInfo @@ -33,23 +39,45 @@ $psi.RedirectStandardOutput = $true $psi.RedirectStandardError = $true $proc = [System.Diagnostics.Process]::Start($psi) -function Send-Request($p, $payload) { +function Send-RequestRaw($p, $payload) { $p.StandardInput.WriteLine($payload) $p.StandardInput.Flush() - return ($p.StandardOutput.ReadLine() | ConvertFrom-Json) + return $p.StandardOutput.ReadLine() +} + +function Send-Request($p, $payload) { + return (Send-RequestRaw $p $payload | ConvertFrom-Json) } # ── Step 2: locate the Notepad window ────────────────────────────────── Write-Host "" Write-Host "Step 2 -- list_windows..." -$listResp = Send-Request $proc '{"jsonrpc":"2.0","id":1,"method":"list_windows"}' -$notepad = $listResp.result.windows | Where-Object { $_.processName -eq 'notepad' } | Select-Object -First 1 +$rawList = Send-RequestRaw $proc '{"jsonrpc":"2.0","id":1,"method":"list_windows"}' +Write-Host "Raw list_windows response (first 800 chars):" -ForegroundColor DarkGray +Write-Host ($rawList.Substring(0, [Math]::Min(800, $rawList.Length))) -ForegroundColor DarkGray +$listResp = $rawList | ConvertFrom-Json + +# Match Notepad permissively. Win 11's UWP Notepad shows process name +# "Notepad" (capitalized) and the window class differs from classic +# notepad.exe. Match by window title containing 'Notepad' OR by +# processName matching notepad / notepad.exe case-insensitively. +$notepad = $listResp.result.windows | Where-Object { + ($_.processName -and $_.processName -imatch '^notepad') -or + ($_.windowTitle -and $_.windowTitle -imatch 'notepad') +} | Select-Object -First 1 + if (-not $notepad) { - Write-Error "No notepad window visible to UIA. Open Notepad in the VM console first." + Write-Host "" + Write-Host "All visible windows the bridge can see:" -ForegroundColor Yellow + $listResp.result.windows | ForEach-Object { + Write-Host (" [{0}] {1} -- class={2} id={3}" -f $_.processName, $_.windowTitle, $_.windowClass, $_.windowId) + } + Write-Host "" + Write-Error "Notepad not in the visible window list. Make sure Notepad is open and visible (not minimised)." $proc.StandardInput.Close(); $proc.WaitForExit(5000) | Out-Null exit 1 } -Write-Host " found: $($notepad.windowTitle) ($($notepad.windowId))" +Write-Host " found: $($notepad.windowTitle) (process=$($notepad.processName), $($notepad.windowId))" # ── Step 3: capture Notepad ──────────────────────────────────────────── Write-Host ""