diff --git a/apps/agent-runner/bridges/windows/Program.cs b/apps/agent-runner/bridges/windows/Program.cs index 68ae2dc..f0af753 100644 --- a/apps/agent-runner/bridges/windows/Program.cs +++ b/apps/agent-runner/bridges/windows/Program.cs @@ -57,7 +57,12 @@ public static int Main(string[] args) string? line; while ((line = Console.In.ReadLine()) != null) { - line = line.Trim(); + // Strip UTF-8 BOM if a client wrote one at the start of the + // stream. Windows clients (PowerShell especially) do this + // unpredictably on the first WriteLine, depending on stream + // buffering. Without this strip the first JSON request gets + // a leading U+FEFF and JsonDocument.Parse rejects it. + line = line.Trim().TrimStart('\uFEFF'); if (line.Length == 0) continue; JsonElement reqId = default; @@ -204,9 +209,10 @@ public RpcDispatcher() "capabilities" => HandleCapabilities(), "list_windows" => HandleListWindows(), "capture" => HandleCapture(@params), + "execute" => HandleExecute(@params), _ => throw new RpcException( RpcError.MethodNotFound, - $"Unknown method: {method}. Supported: ping, capabilities, list_windows, capture."), + $"Unknown method: {method}. Supported: ping, capabilities, list_windows, capture, execute."), }; } @@ -240,7 +246,7 @@ public void Dispose() "capabilities", "list_windows", "capture", - // "execute" lands in Phase 0e3 + "execute", }, uiaProvider = "FlaUI.UIA3", platform = "windows", @@ -280,6 +286,56 @@ private object HandleCapture(JsonElement @params) } } + private object HandleExecute(JsonElement @params) + { + var modifiers = ReadStringArray(@params, "modifiers"); + + var req = new UiaCapturer.ExecuteRequest + { + ElementId = ReadString(@params, "elementId") + ?? throw new RpcException(RpcError.InvalidParams, "execute requires `elementId`."), + ActionType = ReadString(@params, "actionType") ?? "click", + Text = ReadString(@params, "text"), + Value = ReadString(@params, "value"), + Checked = ReadBool(@params, "checked"), + Expanded = ReadBool(@params, "expanded"), + Key = ReadString(@params, "key"), + Modifiers = modifiers, + ClearFirst = ReadBool(@params, "clearFirst") ?? false, + TimeoutMs = ReadInt(@params, "timeoutMs") ?? 5000, + }; + + try + { + var capturer = _capturer.Value; + var sta = _staWorker.Value; + var result = sta.Invoke(() => capturer.Execute(req)); + return new + { + ok = result.Ok, + message = result.Message, + newValue = result.NewValue, + }; + } + catch (InvalidOperationException ex) + { + throw new RpcException(RpcError.InternalError, ex.Message); + } + } + + private static string[]? ReadStringArray(JsonElement parent, string name) + { + if (parent.ValueKind != JsonValueKind.Object) return null; + if (!parent.TryGetProperty(name, out var v)) return null; + if (v.ValueKind != JsonValueKind.Array) return null; + var list = new List(v.GetArrayLength()); + foreach (var item in v.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.String) list.Add(item.GetString()!); + } + return list.ToArray(); + } + private static string? ReadString(JsonElement parent, string name) { if (parent.ValueKind != JsonValueKind.Object) return null; diff --git a/apps/agent-runner/bridges/windows/Uia/UiaCapturer.cs b/apps/agent-runner/bridges/windows/Uia/UiaCapturer.cs index e6dc3f9..50a9fce 100644 --- a/apps/agent-runner/bridges/windows/Uia/UiaCapturer.cs +++ b/apps/agent-runner/bridges/windows/Uia/UiaCapturer.cs @@ -11,6 +11,7 @@ using FlaUI.Core.AutomationElements; using FlaUI.Core.Conditions; using FlaUI.Core.Definitions; +using FlaUI.Core.Exceptions; using FlaUI.Core.Patterns; using FlaUI.UIA3; @@ -21,6 +22,12 @@ internal sealed class UiaCapturer : IDisposable private readonly UIA3Automation _automation; private bool _disposed; + // Cache: element_id -> live AutomationElement. Populated during + // Capture(); consumed by Execute(). Replaced on every new Capture. + // Stays single-rooted (only one capture session at a time) for now; + // multi-window concurrent capture is a future feature. + private CaptureSession? _lastSession; + public UiaCapturer() { _automation = new UIA3Automation(); @@ -33,6 +40,12 @@ public void Dispose() try { _automation.Dispose(); } catch { /* swallow */ } } + private sealed class CaptureSession + { + public required AutomationElement RootWindow { get; init; } + public Dictionary Elements { get; } = new(StringComparer.Ordinal); + } + // ───────────────────────────────────────────────────────────────── // list_windows // ───────────────────────────────────────────────────────────────── @@ -102,15 +115,22 @@ public DesktopCaptureDto Capture(CaptureRequest req) var hwnd = GetHwnd(window); var deadline = DateTime.UtcNow.AddMilliseconds(Math.Max(500, req.TimeoutMs)); + // Start a fresh session so Execute() finds elements from THIS + // capture (not stale ones from a previous window). + var session = new CaptureSession { RootWindow = window }; + var ctx = new WalkContext { MaxDepth = Math.Max(1, req.MaxDepth), MaxElements = Math.Max(50, req.MaxElements), IncludeHidden = req.IncludeHidden, Deadline = deadline, + Session = session, }; var rootDto = WalkElement(window, depth: 0, ctx); + _lastSession = session; + var focused = TryGetFocused(); string? focusedId = focused != null && IsAncestorOrSelf(focused, window) ? AutomationIdFor(focused) @@ -139,6 +159,7 @@ private sealed class WalkContext public DateTime Deadline; public int ElementCount; public int MaxDepthReached; + public required CaptureSession Session; public readonly HashSet SeenIds = new(StringComparer.Ordinal); } @@ -156,6 +177,8 @@ private DesktopElementDto WalkElement(AutomationElement el, int depth, WalkConte id = $"{id}#{n}"; } ctx.SeenIds.Add(id); + // Stash a live handle so Execute() can find it later. + ctx.Session.Elements[id] = el; var role = RoleMapper.ToRole(el); var dto = new DesktopElementDto @@ -199,6 +222,363 @@ private DesktopElementDto WalkElement(AutomationElement el, int depth, WalkConte return dto; } + // ───────────────────────────────────────────────────────────────── + // execute + // ───────────────────────────────────────────────────────────────── + + public sealed class ExecuteRequest + { + public string ElementId { get; init; } = ""; + /// One of: click | type | select | check | expand | focus | scroll_to | key + public string ActionType { get; init; } = "click"; + + // Action-specific payloads. Only the ones relevant to ActionType + // are read; the rest are ignored. + public string? Text { get; init; } // type + public string? Value { get; init; } // select + public bool? Checked { get; init; } // check + public bool? Expanded { get; init; } // expand + public string? Key { get; init; } // key + public string[]? Modifiers { get; init; } // key + public bool ClearFirst { get; init; } // type + public int TimeoutMs { get; init; } = 5000; + } + + public sealed class ExecuteResult + { + public bool Ok { get; init; } + public string? Message { get; init; } + public string? NewValue { get; init; } + } + + public ExecuteResult Execute(ExecuteRequest req) + { + var session = _lastSession + ?? throw new InvalidOperationException( + "No capture session active. Call `capture` before `execute` so the bridge can resolve element_ids."); + + if (!session.Elements.TryGetValue(req.ElementId, out var element)) + { + return new ExecuteResult + { + Ok = false, + Message = $"Unknown element_id `{req.ElementId}` in the current capture session. Re-capture if the window has changed.", + }; + } + + try + { + switch (req.ActionType) + { + case "click": return DoClick(element); + case "type": return DoType(element, req.Text ?? "", req.ClearFirst); + case "select": return DoSelect(element, req.Value ?? ""); + case "check": return DoCheck(element, req.Checked ?? true); + case "expand": return DoExpand(element, req.Expanded ?? true); + case "focus": return DoFocus(element); + case "scroll_to": return DoScrollTo(element); + case "key": return DoKey(element, req.Key ?? "", req.Modifiers); + default: + return new ExecuteResult + { + Ok = false, + Message = $"Unknown action type `{req.ActionType}`.", + }; + } + } + catch (ElementNotAvailableException ex) + { + // UIA throws this when the underlying element disappeared + // between capture and execute (window closed, modal dismissed, etc.). + return new ExecuteResult { Ok = false, Message = $"Element no longer available: {ex.Message}" }; + } + catch (Exception ex) + { + return new ExecuteResult { Ok = false, Message = $"{ex.GetType().Name}: {ex.Message}" }; + } + } + + private static ExecuteResult DoClick(AutomationElement el) + { + var patterns = el.Patterns; + + if (patterns.Invoke.IsSupported) + { + patterns.Invoke.Pattern.Invoke(); + return new ExecuteResult { Ok = true }; + } + + // Some controls (toggle buttons, menu items in some toolkits) only + // support Toggle / SelectionItem. Try them in priority order. + if (patterns.Toggle.IsSupported) + { + patterns.Toggle.Pattern.Toggle(); + return new ExecuteResult { Ok = true }; + } + if (patterns.SelectionItem.IsSupported) + { + patterns.SelectionItem.Pattern.Select(); + return new ExecuteResult { Ok = true }; + } + if (patterns.ExpandCollapse.IsSupported) + { + var p = patterns.ExpandCollapse.Pattern; + if (p.ExpandCollapseState.ValueOrDefault == ExpandCollapseState.Expanded) p.Collapse(); + else p.Expand(); + return new ExecuteResult { Ok = true }; + } + + // Last resort: focus + simulated Space (works for buttons / + // checkboxes / links that don't expose any of the above). + try + { + el.Focus(); + FlaUI.Core.Input.Keyboard.Type(FlaUI.Core.WindowsAPI.VirtualKeyShort.SPACE); + return new ExecuteResult { Ok = true, Message = "Used keyboard fallback (no InvokePattern)." }; + } + catch + { + return new ExecuteResult { Ok = false, Message = "Element does not support any clickable pattern." }; + } + } + + private static ExecuteResult DoType(AutomationElement el, string text, bool clearFirst) + { + var patterns = el.Patterns; + + if (patterns.Value.IsSupported && !patterns.Value.Pattern.IsReadOnly.ValueOrDefault) + { + // Most reliable path: SetValue replaces the entire content + // atomically. Honour clear_first by treating non-clear as a + // no-op (Value semantics is set-by-default). + try { el.Focus(); } catch { /* ignore */ } + patterns.Value.Pattern.SetValue(text); + return new ExecuteResult { Ok = true, NewValue = patterns.Value.Pattern.Value.ValueOrDefault ?? text }; + } + + // Fallback: focus + simulated keystrokes. Used for elements that + // only expose TextPattern (Documents) or no value pattern at all. + try { el.Focus(); } catch { /* ignore */ } + if (clearFirst) + { + FlaUI.Core.Input.Keyboard.TypeSimultaneously( + FlaUI.Core.WindowsAPI.VirtualKeyShort.CONTROL, + FlaUI.Core.WindowsAPI.VirtualKeyShort.KEY_A); + FlaUI.Core.Input.Keyboard.Type(FlaUI.Core.WindowsAPI.VirtualKeyShort.DELETE); + } + FlaUI.Core.Input.Keyboard.Type(text); + + string? newValue = null; + try + { + if (patterns.Value.IsSupported) newValue = patterns.Value.Pattern.Value.ValueOrDefault; + } + catch { /* ignore */ } + + return new ExecuteResult { Ok = true, NewValue = newValue, Message = "Used keyboard fallback (no writable ValuePattern)." }; + } + + private static ExecuteResult DoSelect(AutomationElement el, string value) + { + var patterns = el.Patterns; + + // Direct SelectionItem on the target: caller already knows + // which child to select. + if (patterns.SelectionItem.IsSupported) + { + patterns.SelectionItem.Pattern.Select(); + return new ExecuteResult { Ok = true, NewValue = SafeName(el) ?? value }; + } + + // The element is a Selection container (ComboBox, ListBox). + // Expand it, then find the child whose name/value matches. + if (patterns.Selection.IsSupported) + { + if (patterns.ExpandCollapse.IsSupported) + { + try { patterns.ExpandCollapse.Pattern.Expand(); } catch { /* ignore */ } + } + + var children = el.FindAllChildren(); + var match = children.FirstOrDefault(c => + string.Equals(SafeName(c) ?? "", value, StringComparison.OrdinalIgnoreCase)); + if (match != null && match.Patterns.SelectionItem.IsSupported) + { + match.Patterns.SelectionItem.Pattern.Select(); + return new ExecuteResult { Ok = true, NewValue = SafeName(match) ?? value }; + } + + return new ExecuteResult { Ok = false, Message = $"No child option matched `{value}`." }; + } + + // ComboBox with editable Value (e.g. autocomplete): set the value. + if (patterns.Value.IsSupported && !patterns.Value.Pattern.IsReadOnly.ValueOrDefault) + { + patterns.Value.Pattern.SetValue(value); + return new ExecuteResult { Ok = true, NewValue = value }; + } + + return new ExecuteResult { Ok = false, Message = "Element does not support Selection or SelectionItem patterns." }; + } + + private static ExecuteResult DoCheck(AutomationElement el, bool wantChecked) + { + var patterns = el.Patterns; + + if (patterns.Toggle.IsSupported) + { + var p = patterns.Toggle.Pattern; + // Keep toggling until the state matches; ToggleState.Indeterminate + // (mixed) counts as not-on-yet for the wantChecked=true case. + int guard = 3; + while (guard-- > 0) + { + var current = p.ToggleState.ValueOrDefault; + var isOn = current == ToggleState.On; + if (isOn == wantChecked) break; + p.Toggle(); + } + var final = p.ToggleState.ValueOrDefault == ToggleState.On; + return new ExecuteResult { Ok = final == wantChecked, NewValue = final ? "true" : "false" }; + } + + if (patterns.SelectionItem.IsSupported) + { + // Radio buttons surface as SelectionItem — selecting is the + // only operation; "unchecking" isn't well-defined for a radio. + if (wantChecked) + { + patterns.SelectionItem.Pattern.Select(); + return new ExecuteResult { Ok = true, NewValue = "true" }; + } + return new ExecuteResult { Ok = false, Message = "Cannot uncheck a radio button directly." }; + } + + return new ExecuteResult { Ok = false, Message = "Element does not support Toggle or SelectionItem patterns." }; + } + + private static ExecuteResult DoExpand(AutomationElement el, bool wantExpanded) + { + if (!el.Patterns.ExpandCollapse.IsSupported) + { + return new ExecuteResult { Ok = false, Message = "Element does not support ExpandCollapse pattern." }; + } + var p = el.Patterns.ExpandCollapse.Pattern; + if (wantExpanded) p.Expand(); else p.Collapse(); + var final = p.ExpandCollapseState.ValueOrDefault; + return new ExecuteResult { Ok = true, NewValue = final.ToString() }; + } + + private static ExecuteResult DoFocus(AutomationElement el) + { + el.Focus(); + return new ExecuteResult { Ok = true }; + } + + private static ExecuteResult DoScrollTo(AutomationElement el) + { + if (el.Patterns.ScrollItem.IsSupported) + { + el.Patterns.ScrollItem.Pattern.ScrollIntoView(); + return new ExecuteResult { Ok = true }; + } + // SetFocus often implies scroll-into-view for most controls. + try + { + el.Focus(); + return new ExecuteResult { Ok = true, Message = "Used SetFocus fallback (no ScrollItem pattern)." }; + } + catch + { + return new ExecuteResult { Ok = false, Message = "Element does not support ScrollItem and SetFocus failed." }; + } + } + + private static ExecuteResult DoKey(AutomationElement el, string key, string[]? modifiers) + { + if (string.IsNullOrEmpty(key)) + { + return new ExecuteResult { Ok = false, Message = "Missing `key` argument." }; + } + + try { el.Focus(); } catch { /* ignore */ } + + var vk = ParseVirtualKey(key); + if (vk == null) + { + // Not a named key — fall through to typing the literal text. + FlaUI.Core.Input.Keyboard.Type(key); + return new ExecuteResult { Ok = true, Message = $"Typed literal text `{key}`." }; + } + + var mods = (modifiers ?? Array.Empty()) + .Select(ParseModifier) + .Where(m => m.HasValue) + .Select(m => m!.Value) + .ToArray(); + + if (mods.Length == 0) + { + FlaUI.Core.Input.Keyboard.Type(vk.Value); + } + else + { + var combo = mods.Append(vk.Value).ToArray(); + FlaUI.Core.Input.Keyboard.TypeSimultaneously(combo); + } + + return new ExecuteResult { Ok = true }; + } + + private static FlaUI.Core.WindowsAPI.VirtualKeyShort? ParseVirtualKey(string key) + { + // Accept common friendly names; fall through to anything that + // matches the enum case-insensitively. + var lowered = key.ToLowerInvariant(); + return lowered switch + { + "enter" or "return" => FlaUI.Core.WindowsAPI.VirtualKeyShort.RETURN, + "tab" => FlaUI.Core.WindowsAPI.VirtualKeyShort.TAB, + "escape" or "esc" => FlaUI.Core.WindowsAPI.VirtualKeyShort.ESCAPE, + "space" or "spacebar" => FlaUI.Core.WindowsAPI.VirtualKeyShort.SPACE, + "backspace" => FlaUI.Core.WindowsAPI.VirtualKeyShort.BACK, + "delete" or "del" => FlaUI.Core.WindowsAPI.VirtualKeyShort.DELETE, + "home" => FlaUI.Core.WindowsAPI.VirtualKeyShort.HOME, + "end" => FlaUI.Core.WindowsAPI.VirtualKeyShort.END, + "pageup" or "pgup" => FlaUI.Core.WindowsAPI.VirtualKeyShort.PRIOR, + "pagedown" or "pgdn" => FlaUI.Core.WindowsAPI.VirtualKeyShort.NEXT, + "up" => FlaUI.Core.WindowsAPI.VirtualKeyShort.UP, + "down" => FlaUI.Core.WindowsAPI.VirtualKeyShort.DOWN, + "left" => FlaUI.Core.WindowsAPI.VirtualKeyShort.LEFT, + "right" => FlaUI.Core.WindowsAPI.VirtualKeyShort.RIGHT, + "f1" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F1, + "f2" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F2, + "f3" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F3, + "f4" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F4, + "f5" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F5, + "f6" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F6, + "f7" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F7, + "f8" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F8, + "f9" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F9, + "f10" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F10, + "f11" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F11, + "f12" => FlaUI.Core.WindowsAPI.VirtualKeyShort.F12, + _ => Enum.TryParse(key, ignoreCase: true, out var v) ? v : null, + }; + } + + private static FlaUI.Core.WindowsAPI.VirtualKeyShort? ParseModifier(string m) + { + return m.ToLowerInvariant() switch + { + "ctrl" or "control" => FlaUI.Core.WindowsAPI.VirtualKeyShort.CONTROL, + "alt" => FlaUI.Core.WindowsAPI.VirtualKeyShort.ALT, + "shift" => FlaUI.Core.WindowsAPI.VirtualKeyShort.SHIFT, + "meta" or "win" or "windows" => FlaUI.Core.WindowsAPI.VirtualKeyShort.LWIN, + _ => null, + }; + } + // ───────────────────────────────────────────────────────────────── // Target resolution // ───────────────────────────────────────────────────────────────── diff --git a/apps/agent-runner/bridges/windows/scripts/notepad-demo.ps1 b/apps/agent-runner/bridges/windows/scripts/notepad-demo.ps1 new file mode 100644 index 0000000..e1309b8 --- /dev/null +++ b/apps/agent-runner/bridges/windows/scripts/notepad-demo.ps1 @@ -0,0 +1,158 @@ +# Live demo: drive Notepad end-to-end through the bridge. +# +# Sequence: +# 1. Launch Notepad (or attach if already open) and bring to foreground +# 2. list_windows -> find the Notepad window +# 3. capture -> get the element tree, locate the text editor +# 4. execute type -> write a message into Notepad's edit area +# 5. capture -> re-snapshot and prove the text actually landed +# +# Designed to be run from the VM's interactive PowerShell so UIA can +# see real windows (SSH session 2 can't). + +$ErrorActionPreference = 'Stop' + +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$exe = Join-Path $scriptDir '..\bin\Debug\net8.0-windows\agentmark-bridge-windows.exe' +if (-not (Test-Path $exe)) { + Write-Error "Bridge exe not built. Expected at $exe." + exit 1 +} + +# ── Step 1: Notepad must already be open ──────────────────────────────── +# Win 11 UWP apps have a startup delay and weird session quirks that +# make "launch + immediately query" unreliable. Easier and more honest +# to require Notepad to already be visible to the user. +$notepadProc = Get-Process notepad -ErrorAction SilentlyContinue +if (-not $notepadProc) { + Write-Host "Please open Notepad first (Win+R, type 'notepad', Enter), then re-run this script." -ForegroundColor Yellow + exit 1 +} +Write-Host "Found notepad process(es): $((Get-Process notepad).Id -join ', ')" + +# Spawn bridge +$psi = New-Object System.Diagnostics.ProcessStartInfo +$psi.FileName = $exe +$psi.UseShellExecute = $false +$psi.RedirectStandardInput = $true +$psi.RedirectStandardOutput = $true +$psi.RedirectStandardError = $true +$proc = [System.Diagnostics.Process]::Start($psi) + +function Send-RequestRaw($p, $payload) { + $p.StandardInput.WriteLine($payload) + $p.StandardInput.Flush() + return $p.StandardOutput.ReadLine() +} + +function Send-Request($p, $payload) { + return (Send-RequestRaw $p $payload | ConvertFrom-Json) +} + +# ── Step 2: locate the Notepad window ────────────────────────────────── +Write-Host "" +Write-Host "Step 2 -- list_windows..." +$rawList = Send-RequestRaw $proc '{"jsonrpc":"2.0","id":1,"method":"list_windows"}' +Write-Host "Raw list_windows response (first 800 chars):" -ForegroundColor DarkGray +Write-Host ($rawList.Substring(0, [Math]::Min(800, $rawList.Length))) -ForegroundColor DarkGray +$listResp = $rawList | ConvertFrom-Json + +# Match Notepad permissively. Win 11's UWP Notepad shows process name +# "Notepad" (capitalized) and the window class differs from classic +# notepad.exe. Match by window title containing 'Notepad' OR by +# processName matching notepad / notepad.exe case-insensitively. +$notepad = $listResp.result.windows | Where-Object { + ($_.processName -and $_.processName -imatch '^notepad') -or + ($_.windowTitle -and $_.windowTitle -imatch 'notepad') +} | Select-Object -First 1 + +if (-not $notepad) { + Write-Host "" + Write-Host "All visible windows the bridge can see:" -ForegroundColor Yellow + $listResp.result.windows | ForEach-Object { + Write-Host (" [{0}] {1} -- class={2} id={3}" -f $_.processName, $_.windowTitle, $_.windowClass, $_.windowId) + } + Write-Host "" + Write-Error "Notepad not in the visible window list. Make sure Notepad is open and visible (not minimised)." + $proc.StandardInput.Close(); $proc.WaitForExit(5000) | Out-Null + exit 1 +} +Write-Host " found: $($notepad.windowTitle) (process=$($notepad.processName), $($notepad.windowId))" + +# ── Step 3: capture Notepad ──────────────────────────────────────────── +Write-Host "" +Write-Host "Step 3 -- capture Notepad..." +$capPayload = '{"jsonrpc":"2.0","id":2,"method":"capture","params":{"windowId":"' + $notepad.windowId + '","maxDepth":8,"maxElements":400}}' +$capResp = Send-Request $proc $capPayload + +if (-not $capResp.result) { + Write-Error "capture failed: $(($capResp | ConvertTo-Json -Compress))" + $proc.StandardInput.Close(); $proc.WaitForExit(5000) | Out-Null + exit 1 +} + +$cap = $capResp.result +Write-Host " treeDepth=$($cap.treeDepth) elementCount=$($cap.elementCount)" + +# Walk children looking for a text-area-style element (Notepad's editor +# surfaces as role: text_area). On Win 11, the actual edit control may +# be nested a level or two below the top-level window. +function Find-EditElement($node) { + if ($null -eq $node) { return $null } + if ($node.role -eq 'text_area' -or $node.role -eq 'text_input') { return $node } + if ($node.children) { + foreach ($child in $node.children) { + $found = Find-EditElement $child + if ($found) { return $found } + } + } + return $null +} + +$edit = Find-EditElement $cap.root +if (-not $edit) { + Write-Error "Could not find a text_area / text_input element in Notepad's tree. Dump:" + Write-Host ($cap.root | ConvertTo-Json -Depth 10) + $proc.StandardInput.Close(); $proc.WaitForExit(5000) | Out-Null + exit 1 +} +Write-Host " editor element_id : $($edit.id) (role=$($edit.role))" +Write-Host " current value : '$($edit.value)'" + +# ── Step 4: type into the editor ─────────────────────────────────────── +$message = "Hello from AgentMark Desktop -- phase 0e3 live demo" +Write-Host "" +Write-Host "Step 4 -- execute type into editor..." +$execPayload = '{"jsonrpc":"2.0","id":3,"method":"execute","params":{"elementId":"' + $edit.id + '","actionType":"type","text":"' + $message + '","clearFirst":true}}' +$execResp = Send-Request $proc $execPayload + +if (-not $execResp.result.ok) { + Write-Error "execute failed: $($execResp.result.message)" + $proc.StandardInput.Close(); $proc.WaitForExit(5000) | Out-Null + exit 1 +} +Write-Host " ok : $($execResp.result.ok)" +Write-Host " newValue : $($execResp.result.newValue)" +if ($execResp.result.message) { Write-Host " note : $($execResp.result.message)" } + +# Give the edit a moment to reflect. +Start-Sleep -Milliseconds 250 + +# ── Step 5: re-capture and prove the change ──────────────────────────── +Write-Host "" +Write-Host "Step 5 -- re-capture and verify..." +$cap2Payload = '{"jsonrpc":"2.0","id":4,"method":"capture","params":{"windowId":"' + $notepad.windowId + '","maxDepth":8,"maxElements":400}}' +$cap2Resp = Send-Request $proc $cap2Payload +$edit2 = Find-EditElement $cap2Resp.result.root +Write-Host " editor value after type : '$($edit2.value)'" + +$ok = $edit2.value -and $edit2.value.Contains("Hello from AgentMark Desktop") +$proc.StandardInput.Close(); $proc.WaitForExit(5000) | Out-Null + +Write-Host "" +if ($ok) { + Write-Host "LIVE DEMO PASSED -- AgentMark Desktop drove real Notepad end-to-end." -ForegroundColor Green +} else { + Write-Host "LIVE DEMO FAILED -- expected text not found after execute." -ForegroundColor Red + exit 1 +}