diff --git a/apps/agent-runner/bridges/windows/Program.cs b/apps/agent-runner/bridges/windows/Program.cs index 0f1fa15..68ae2dc 100644 --- a/apps/agent-runner/bridges/windows/Program.cs +++ b/apps/agent-runner/bridges/windows/Program.cs @@ -24,6 +24,7 @@ using System.Text; using System.Text.Json; using System.Text.Json.Serialization; +using AgentMark.Bridge.Windows.Uia; namespace AgentMark.Bridge.Windows; @@ -51,7 +52,7 @@ public static int Main(string[] args) // EOF, leaving the process hung even after the parent closes the // pipe. ReadLine() handles EOF correctly. UIA calls are themselves // blocking COM-STA invocations so async doesn't buy us anything. - var dispatcher = new RpcDispatcher(); + using var dispatcher = new RpcDispatcher(); string? line; while ((line = Console.In.ReadLine()) != null) @@ -180,22 +181,47 @@ internal sealed class RpcException(RpcError error, string message) : Exception(m // Dispatcher // ────────────────────────────────────────────────────────────────────── -internal sealed class RpcDispatcher +internal sealed class RpcDispatcher : IDisposable { private static readonly string BridgeVersion = "0.4.0"; + // UIA work is lazy-initialised — ping/capabilities don't need it and + // booting UIA on startup adds ~150ms we don't want for clients that + // only smoke-test the bridge. + private readonly Lazy _staWorker = new(() => new StaWorker()); + private readonly Lazy _capturer; + + public RpcDispatcher() + { + _capturer = new Lazy(() => _staWorker.Value.Invoke(() => new UiaCapturer())); + } + public object? Dispatch(string method, JsonElement @params) { return method switch { "ping" => HandlePing(), "capabilities" => HandleCapabilities(), + "list_windows" => HandleListWindows(), + "capture" => HandleCapture(@params), _ => throw new RpcException( RpcError.MethodNotFound, - $"Unknown method: {method}. Supported: ping, capabilities."), + $"Unknown method: {method}. Supported: ping, capabilities, list_windows, capture."), }; } + public void Dispose() + { + if (_capturer.IsValueCreated) + { + try { _staWorker.Value.Invoke(() => _capturer.Value.Dispose()); } catch { /* swallow */ } + } + if (_staWorker.IsValueCreated) + { + try { _staWorker.Value.Dispose(); } catch { /* swallow */ } + } + } + private static object HandlePing() => new { pong = true, @@ -212,9 +238,71 @@ internal sealed class RpcDispatcher { "ping", "capabilities", - // "capture" and "execute" land in Phase 0e2/0e3 + "list_windows", + "capture", + // "execute" lands in Phase 0e3 }, uiaProvider = "FlaUI.UIA3", platform = "windows", }; + + private object HandleListWindows() + { + var capturer = _capturer.Value; + var sta = _staWorker.Value; + var windows = sta.Invoke(() => capturer.ListWindows()); + return new { windows }; + } + + private object HandleCapture(JsonElement @params) + { + var req = new UiaCapturer.CaptureRequest + { + ProcessName = ReadString(@params, "processName"), + ProcessId = ReadInt(@params, "processId"), + WindowTitle = ReadString(@params, "windowTitle"), + WindowId = ReadString(@params, "windowId"), + MaxDepth = ReadInt(@params, "maxDepth") ?? 12, + IncludeHidden = ReadBool(@params, "includeHidden") ?? false, + TimeoutMs = ReadInt(@params, "timeoutMs") ?? 5000, + MaxElements = ReadInt(@params, "maxElements") ?? 2000, + }; + + try + { + var capturer = _capturer.Value; + var sta = _staWorker.Value; + return sta.Invoke(() => capturer.Capture(req)); + } + catch (InvalidOperationException ex) + { + throw new RpcException(RpcError.WindowNotFound, ex.Message); + } + } + + private static string? ReadString(JsonElement parent, string name) + { + if (parent.ValueKind != JsonValueKind.Object) return null; + if (!parent.TryGetProperty(name, out var v)) return null; + return v.ValueKind == JsonValueKind.String ? v.GetString() : null; + } + + private static int? ReadInt(JsonElement parent, string name) + { + if (parent.ValueKind != JsonValueKind.Object) return null; + if (!parent.TryGetProperty(name, out var v)) return null; + return v.ValueKind == JsonValueKind.Number && v.TryGetInt32(out var i) ? i : null; + } + + private static bool? ReadBool(JsonElement parent, string name) + { + if (parent.ValueKind != JsonValueKind.Object) return null; + if (!parent.TryGetProperty(name, out var v)) return null; + return v.ValueKind switch + { + JsonValueKind.True => true, + JsonValueKind.False => false, + _ => null, + }; + } } diff --git a/apps/agent-runner/bridges/windows/Uia/CaptureDtos.cs b/apps/agent-runner/bridges/windows/Uia/CaptureDtos.cs new file mode 100644 index 0000000..7694dfa --- /dev/null +++ b/apps/agent-runner/bridges/windows/Uia/CaptureDtos.cs @@ -0,0 +1,149 @@ +// DTOs the bridge returns over JSON-RPC. Field names match the +// AgentMark v0.4 DesktopCapture / DesktopElement spec exactly (camelCase +// after JsonNamingPolicy.CamelCase) so the Node-side WindowsUiaBackend +// can deserialize straight into the existing TypeScript types from +// @thinkfleet/agentmark. +// +// Keep this file the canonical source of truth for what we put on the +// wire — when the spec grows, mirror the change here. + +using System.Text.Json.Serialization; + +namespace AgentMark.Bridge.Windows.Uia; + +/// +/// One window discoverable by list_windows. Includes just enough to let +/// the caller pick which window to capture in a subsequent call. +/// +internal sealed class WindowSummaryDto +{ + [JsonPropertyName("windowId")] + public string WindowId { get; init; } = ""; + + [JsonPropertyName("processName")] + public string? ProcessName { get; init; } + + [JsonPropertyName("processId")] + public int? ProcessId { get; init; } + + [JsonPropertyName("windowTitle")] + public string WindowTitle { get; init; } = ""; + + [JsonPropertyName("windowClass")] + public string? WindowClass { get; init; } + + /// Whether this window currently has keyboard focus. + [JsonPropertyName("hasFocus")] + public bool HasFocus { get; init; } +} + +/// +/// What `capture` returns. Mirrors the TS `DesktopCapture` interface. +/// +internal sealed class DesktopCaptureDto +{ + [JsonPropertyName("platform")] + public string Platform => "windows"; + + [JsonPropertyName("processName")] + public string? ProcessName { get; init; } + + [JsonPropertyName("processId")] + public int? ProcessId { get; init; } + + [JsonPropertyName("windowTitle")] + public string WindowTitle { get; init; } = ""; + + [JsonPropertyName("windowClass")] + public string? WindowClass { get; init; } + + [JsonPropertyName("windowId")] + public string WindowId { get; init; } = ""; + + [JsonPropertyName("focusedElementId")] + public string? FocusedElementId { get; init; } + + [JsonPropertyName("treeDepth")] + public int TreeDepth { get; init; } + + [JsonPropertyName("elementCount")] + public int ElementCount { get; init; } + + [JsonPropertyName("root")] + public DesktopElementDto Root { get; init; } = null!; +} + +/// +/// One node in the captured accessibility tree. Mirrors the TS +/// `DesktopElement` interface. +/// +internal sealed class DesktopElementDto +{ + [JsonPropertyName("id")] + public string Id { get; init; } = ""; + + [JsonPropertyName("role")] + public string Role { get; init; } = "other"; + + [JsonPropertyName("name")] + public string? Name { get; init; } + + [JsonPropertyName("value")] + public string? Value { get; init; } + + [JsonPropertyName("placeholder")] + public string? Placeholder { get; init; } + + [JsonPropertyName("enabled")] + public bool? Enabled { get; init; } + + [JsonPropertyName("selected")] + public bool? Selected { get; init; } + + [JsonPropertyName("readOnly")] + public bool? ReadOnly { get; init; } + + [JsonPropertyName("expanded")] + public bool? Expanded { get; init; } + + [JsonPropertyName("aria")] + public AriaStateDto? Aria { get; init; } + + [JsonPropertyName("bounds")] + public BoundsDto? Bounds { get; init; } + + /// Mutable so the walker can backfill after building the + /// parent. Stays null when an element is a leaf / depth-truncated. + [JsonPropertyName("children")] + public List? Children { get; set; } +} + +internal sealed class AriaStateDto +{ + [JsonPropertyName("pressed")] + public bool? Pressed { get; init; } + + [JsonPropertyName("checked")] + public object? Checked { get; init; } // bool | "mixed" + + [JsonPropertyName("required")] + public bool? Required { get; init; } + + [JsonPropertyName("invalid")] + public bool? Invalid { get; init; } +} + +internal sealed class BoundsDto +{ + [JsonPropertyName("x")] + public double X { get; init; } + + [JsonPropertyName("y")] + public double Y { get; init; } + + [JsonPropertyName("width")] + public double Width { get; init; } + + [JsonPropertyName("height")] + public double Height { get; init; } +} diff --git a/apps/agent-runner/bridges/windows/Uia/RoleMapper.cs b/apps/agent-runner/bridges/windows/Uia/RoleMapper.cs new file mode 100644 index 0000000..dc0b008 --- /dev/null +++ b/apps/agent-runner/bridges/windows/Uia/RoleMapper.cs @@ -0,0 +1,118 @@ +// Maps UIA ControlType (and a few special cases) to the normalised +// DesktopRole vocabulary the AgentMark v0.4 spec defines. Producers on +// other platforms (macOS AXAPI, Linux AT-SPI, vision fallback) all emit +// the same vocabulary so the Node-side body-builder can render them +// uniformly. +// +// Unrecognised types fall through to "other". The walker drops "other" +// non-leaf nodes from the rendered body but their interactive children +// are still surfaced — keeps the markdown clean without losing actions. + +using FlaUI.Core.AutomationElements; +using FlaUI.Core.Definitions; + +namespace AgentMark.Bridge.Windows.Uia; + +internal static class RoleMapper +{ + public static string ToRole(AutomationElement el) + { + try + { + // ControlType resolves through cached or live properties; in + // a tight tree walk we sometimes see UIA mid-update so guard + // every property read. + var ct = el.Properties.ControlType.ValueOrDefault; + + return ct switch + { + ControlType.Window => "window", + ControlType.Pane => "pane", + ControlType.Group => "group", + ControlType.ToolBar => "toolbar", + ControlType.MenuBar => "menu", + ControlType.Menu => "menu", + ControlType.MenuItem => "menu_item", + ControlType.Tab => "tab_list", + ControlType.TabItem => "tab", + ControlType.Tree => "tree", + ControlType.TreeItem => "tree_item", + ControlType.List => "list", + ControlType.ListItem => "list_item", + ControlType.Table => "table", + ControlType.DataGrid => "table", + ControlType.DataItem => "row", + ControlType.Header => "row", + ControlType.HeaderItem => "column_header", + ControlType.Button => "button", + ControlType.SplitButton => "split_button", + ControlType.Edit => IsPasswordEdit(el) ? "password_input" : "text_input", + ControlType.Document => "text_area", + ControlType.CheckBox => "check_box", + ControlType.RadioButton => "radio_button", + ControlType.ComboBox => "combo_box", + ControlType.Slider => "slider", + ControlType.ProgressBar => "progress_bar", + ControlType.Hyperlink => "link", + ControlType.Text => "static_text", + ControlType.Image => "image", + ControlType.Separator => "separator", + ControlType.StatusBar => "status_bar", + ControlType.ScrollBar => "scroll_bar", + ControlType.ToolTip => "tooltip", + ControlType.Custom => InferCustom(el), + _ => "other", + }; + } + catch + { + return "other"; + } + } + + /// + /// UIA exposes IsPassword via the password-controls property. FlaUI + /// surfaces it on Edit elements; some controls (web embeddings, + /// custom controls) don't set it even when they should — best effort. + /// + private static bool IsPasswordEdit(AutomationElement el) + { + try { return el.Properties.IsPassword.ValueOrDefault; } + catch { return false; } + } + + /// + /// Custom controls are common in WPF / WinForms / Electron-hosted + /// pieces. Use the LocalizedControlType string + simple heuristics + /// to guess a useful role before falling through to "other". + /// + private static string InferCustom(AutomationElement el) + { + try + { + var localized = el.Properties.LocalizedControlType.ValueOrDefault?.ToLowerInvariant(); + if (string.IsNullOrEmpty(localized)) return "other"; + + return localized switch + { + _ when localized.Contains("button") => "button", + _ when localized.Contains("textbox") || localized.Contains("text box") || localized.Contains("edit") => "text_input", + _ when localized.Contains("checkbox") || localized.Contains("check box") => "check_box", + _ when localized.Contains("radio") => "radio_button", + _ when localized.Contains("combo") || localized.Contains("dropdown") => "combo_box", + _ when localized.Contains("link") => "link", + _ when localized.Contains("label") => "label", + _ when localized.Contains("group") => "group", + _ when localized.Contains("panel") => "pane", + _ when localized.Contains("toolbar") || localized.Contains("tool bar") => "toolbar", + _ when localized.Contains("menu") => "menu", + _ when localized.Contains("tab") => "tab", + _ when localized.Contains("list item") || localized.Contains("listitem") => "list_item", + _ when localized.Contains("list") => "list", + _ when localized.Contains("tree") => "tree", + _ => "other", + }; + } + catch { return "other"; } + } +} diff --git a/apps/agent-runner/bridges/windows/Uia/StaWorker.cs b/apps/agent-runner/bridges/windows/Uia/StaWorker.cs new file mode 100644 index 0000000..8ff82f9 --- /dev/null +++ b/apps/agent-runner/bridges/windows/Uia/StaWorker.cs @@ -0,0 +1,71 @@ +// Single-threaded apartment (COM-STA) worker. +// +// UIA is COM-based and must be called from an STA thread. The bridge's +// main loop reads stdin on the default thread; we hand off any UIA work +// to a dedicated thread that lives for the lifetime of the process. +// +// Pattern: BlockingCollection queue, a single consumer thread. +// Caller blocks on a TaskCompletionSource so the dispatcher code reads +// linearly. + +using System.Collections.Concurrent; + +namespace AgentMark.Bridge.Windows.Uia; + +internal sealed class StaWorker : IDisposable +{ + private readonly Thread _thread; + private readonly BlockingCollection _queue = new(); + private volatile bool _disposed; + + public StaWorker() + { + _thread = new Thread(Run) + { + IsBackground = true, + Name = "agentmark-uia-sta", + }; + _thread.SetApartmentState(ApartmentState.STA); + _thread.Start(); + } + + /// + /// Run on the STA thread and block until it finishes. + /// Any exception is re-thrown on the calling thread. + /// + public T Invoke(Func fn) + { + if (_disposed) throw new ObjectDisposedException(nameof(StaWorker)); + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _queue.Add(() => + { + try { tcs.TrySetResult(fn()); } + catch (Exception ex) { tcs.TrySetException(ex); } + }); + return tcs.Task.GetAwaiter().GetResult(); + } + + /// Void-returning convenience overload. + public void Invoke(Action fn) => Invoke(() => { fn(); return null; }); + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + _queue.CompleteAdding(); + // We don't join the thread — it's a background thread and the + // process is exiting anyway. + } + + private void Run() + { + foreach (var action in _queue.GetConsumingEnumerable()) + { + try { action(); } + catch (Exception ex) + { + Console.Error.WriteLine($"[bridge] STA worker swallowed: {ex}"); + } + } + } +} diff --git a/apps/agent-runner/bridges/windows/Uia/UiaCapturer.cs b/apps/agent-runner/bridges/windows/Uia/UiaCapturer.cs new file mode 100644 index 0000000..e6dc3f9 --- /dev/null +++ b/apps/agent-runner/bridges/windows/Uia/UiaCapturer.cs @@ -0,0 +1,540 @@ +// Core UIA work — list_windows + capture. +// +// All public methods MUST be invoked from the STA worker thread. The +// FlaUI.Core types wrap COM interfaces with apartment requirements; +// touching them from any other thread leaks COM proxies and eventually +// deadlocks. + +using System.Diagnostics; +using System.Runtime.InteropServices; +using FlaUI.Core; +using FlaUI.Core.AutomationElements; +using FlaUI.Core.Conditions; +using FlaUI.Core.Definitions; +using FlaUI.Core.Patterns; +using FlaUI.UIA3; + +namespace AgentMark.Bridge.Windows.Uia; + +internal sealed class UiaCapturer : IDisposable +{ + private readonly UIA3Automation _automation; + private bool _disposed; + + public UiaCapturer() + { + _automation = new UIA3Automation(); + } + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + try { _automation.Dispose(); } catch { /* swallow */ } + } + + // ───────────────────────────────────────────────────────────────── + // list_windows + // ───────────────────────────────────────────────────────────────── + + public List ListWindows() + { + var desktop = _automation.GetDesktop(); + var condition = _automation.ConditionFactory.ByControlType(ControlType.Window); + var children = desktop.FindAllChildren(condition); + + var focused = TryGetFocused(); + var focusedHwnd = focused != null ? GetHwnd(focused) : IntPtr.Zero; + + var result = new List(children.Length); + foreach (var win in children) + { + string? title = SafeName(win); + // Skip the chrome-less ghost windows Windows creates for + // every IME / tray / hidden helper. They have no title and + // no useful UI for an agent. + if (string.IsNullOrEmpty(title)) continue; + // Off-screen, minimised, or zero-size windows aren't useful. + if (SafeIsOffscreen(win)) continue; + + var hwnd = GetHwnd(win); + if (hwnd == IntPtr.Zero) continue; + + int? pid = SafeProcessId(win); + string? procName = pid.HasValue ? SafeProcessName(pid.Value) : null; + + result.Add(new WindowSummaryDto + { + WindowId = EncodeHwnd(hwnd), + WindowTitle = title!, + ProcessName = procName, + ProcessId = pid, + WindowClass = SafeClassName(win), + HasFocus = focusedHwnd != IntPtr.Zero && IsAncestorOrSelf(focused!, win), + }); + } + return result; + } + + // ───────────────────────────────────────────────────────────────── + // capture + // ───────────────────────────────────────────────────────────────── + + public sealed class CaptureRequest + { + public string? ProcessName { get; init; } + public int? ProcessId { get; init; } + public string? WindowTitle { get; init; } + public string? WindowId { get; init; } + public int MaxDepth { get; init; } = 12; + public bool IncludeHidden { get; init; } + public int TimeoutMs { get; init; } = 5000; + /// Hard cap on emitted elements to keep the wire payload + /// bounded. Large Excel workbooks blow past 10k elements easily. + public int MaxElements { get; init; } = 2000; + } + + public DesktopCaptureDto Capture(CaptureRequest req) + { + var window = ResolveTarget(req) + ?? throw new InvalidOperationException("No matching window found and no focused window available."); + + var hwnd = GetHwnd(window); + var deadline = DateTime.UtcNow.AddMilliseconds(Math.Max(500, req.TimeoutMs)); + + var ctx = new WalkContext + { + MaxDepth = Math.Max(1, req.MaxDepth), + MaxElements = Math.Max(50, req.MaxElements), + IncludeHidden = req.IncludeHidden, + Deadline = deadline, + }; + + var rootDto = WalkElement(window, depth: 0, ctx); + var focused = TryGetFocused(); + string? focusedId = focused != null && IsAncestorOrSelf(focused, window) + ? AutomationIdFor(focused) + : null; + + int? pid = SafeProcessId(window); + return new DesktopCaptureDto + { + ProcessName = pid.HasValue ? SafeProcessName(pid.Value) : null, + ProcessId = pid, + WindowTitle = SafeName(window) ?? "(untitled window)", + WindowClass = SafeClassName(window), + WindowId = EncodeHwnd(hwnd), + FocusedElementId = focusedId, + TreeDepth = ctx.MaxDepthReached, + ElementCount = ctx.ElementCount, + Root = rootDto, + }; + } + + private sealed class WalkContext + { + public int MaxDepth; + public int MaxElements; + public bool IncludeHidden; + public DateTime Deadline; + public int ElementCount; + public int MaxDepthReached; + public readonly HashSet SeenIds = new(StringComparer.Ordinal); + } + + private DesktopElementDto WalkElement(AutomationElement el, int depth, WalkContext ctx) + { + ctx.ElementCount++; + if (depth > ctx.MaxDepthReached) ctx.MaxDepthReached = depth; + + var id = AutomationIdFor(el); + // Disambiguate duplicate AutomationIds across siblings. + if (ctx.SeenIds.Contains(id)) + { + int n = 2; + while (ctx.SeenIds.Contains($"{id}#{n}")) n++; + id = $"{id}#{n}"; + } + ctx.SeenIds.Add(id); + + var role = RoleMapper.ToRole(el); + var dto = new DesktopElementDto + { + Id = id, + Role = role, + Name = SafeName(el), + Value = ExtractValue(el), + Placeholder = SafePlaceholder(el), + Enabled = SafeBool(() => el.Properties.IsEnabled.ValueOrDefault), + ReadOnly = ExtractReadOnly(el), + Selected = ExtractSelected(el), + Expanded = ExtractExpanded(el), + Aria = BuildAria(el), + Bounds = SafeBounds(el), + }; + + // Stop expanding past the cap; the parent still includes its + // own data but children get truncated. The body builder on the + // Node side renders the markdown gracefully when children are + // null. + if (depth >= ctx.MaxDepth) return dto; + if (ctx.ElementCount >= ctx.MaxElements) return dto; + if (DateTime.UtcNow > ctx.Deadline) return dto; + + AutomationElement[] children; + try { children = el.FindAllChildren(); } + catch { return dto; } + + if (children.Length == 0) return dto; + + var kids = new List(children.Length); + foreach (var c in children) + { + if (ctx.ElementCount >= ctx.MaxElements) break; + if (DateTime.UtcNow > ctx.Deadline) break; + if (!ctx.IncludeHidden && SafeIsOffscreen(c)) continue; + kids.Add(WalkElement(c, depth + 1, ctx)); + } + if (kids.Count > 0) dto.Children = kids; + return dto; + } + + // ───────────────────────────────────────────────────────────────── + // Target resolution + // ───────────────────────────────────────────────────────────────── + + private AutomationElement? ResolveTarget(CaptureRequest req) + { + // window_id (HWND) is the most precise match — use it first. + if (!string.IsNullOrWhiteSpace(req.WindowId) && TryDecodeHwnd(req.WindowId, out var hwnd)) + { + try + { + var elFromHwnd = _automation.FromHandle(hwnd); + if (elFromHwnd != null) return elFromHwnd; + } + catch { /* fall through to other strategies */ } + } + + // For the remaining strategies we scan top-level windows once. + var desktop = _automation.GetDesktop(); + var allWindows = desktop.FindAllChildren( + _automation.ConditionFactory.ByControlType(ControlType.Window)); + + if (req.ProcessId is int pid) + { + var match = allWindows.FirstOrDefault(w => SafeProcessId(w) == pid && !string.IsNullOrEmpty(SafeName(w))); + if (match != null) return match; + } + + if (!string.IsNullOrWhiteSpace(req.ProcessName)) + { + var target = req.ProcessName.ToLowerInvariant().TrimEnd('.', 'e', 'x', 'e'); + var match = allWindows.FirstOrDefault(w => + { + var p = SafeProcessId(w); + if (!p.HasValue) return false; + var name = SafeProcessName(p.Value); + if (string.IsNullOrEmpty(name)) return false; + return name!.ToLowerInvariant().TrimEnd('.', 'e', 'x', 'e').Contains(target); + }); + if (match != null) return match; + } + + if (!string.IsNullOrWhiteSpace(req.WindowTitle)) + { + var target = req.WindowTitle.ToLowerInvariant(); + var match = allWindows.FirstOrDefault(w => + (SafeName(w)?.ToLowerInvariant() ?? "").Contains(target)); + if (match != null) return match; + } + + // Default: focused window. + var focused = TryGetFocused(); + if (focused != null) + { + var root = FindTopLevelAncestor(focused); + if (root != null) return root; + } + return null; + } + + private static AutomationElement? FindTopLevelAncestor(AutomationElement el) + { + var current = el; + while (current != null) + { + try + { + if (current.Properties.ControlType.ValueOrDefault == ControlType.Window) return current; + current = current.Parent; + } + catch { return null; } + } + return null; + } + + private AutomationElement? TryGetFocused() + { + try { return _automation.FocusedElement(); } catch { return null; } + } + + // ───────────────────────────────────────────────────────────────── + // Property extraction helpers — all defensive, all swallow. + // ───────────────────────────────────────────────────────────────── + + private static string AutomationIdFor(AutomationElement el) + { + try + { + var id = el.Properties.AutomationId.ValueOrDefault; + if (!string.IsNullOrEmpty(id)) return id!; + } + catch { /* fall through */ } + // Fall back to a stable hash of name+role+pid. Avoids unstable + // RuntimeId which can churn across captures. + var name = SafeName(el) ?? ""; + var role = SafeControlType(el); + var pid = SafeProcessId(el) ?? 0; + return $"el_{(uint)HashCode.Combine(name, role, pid):x8}"; + } + + private static string SafeControlType(AutomationElement el) + { + try { return el.Properties.ControlType.ValueOrDefault.ToString(); } + catch { return "Unknown"; } + } + + private static string? SafeName(AutomationElement el) + { + try + { + var n = el.Properties.Name.ValueOrDefault; + return string.IsNullOrEmpty(n) ? null : n; + } + catch { return null; } + } + + private static string? SafeClassName(AutomationElement el) + { + try + { + var c = el.Properties.ClassName.ValueOrDefault; + return string.IsNullOrEmpty(c) ? null : c; + } + catch { return null; } + } + + private static int? SafeProcessId(AutomationElement el) + { + try { return el.Properties.ProcessId.ValueOrDefault; } + catch { return null; } + } + + private static string? SafeProcessName(int pid) + { + try + { + using var p = Process.GetProcessById(pid); + return p.ProcessName; + } + catch { return null; } + } + + private static string? SafePlaceholder(AutomationElement el) + { + try + { + // Edit + Document expose HelpText; UIA HelpText is the + // closest analog to HTML placeholder. + var h = el.Properties.HelpText.ValueOrDefault; + return string.IsNullOrEmpty(h) ? null : h; + } + catch { return null; } + } + + private static bool? SafeBool(Func f) + { + try { return f(); } catch { return null; } + } + + private static bool SafeIsOffscreen(AutomationElement el) + { + try { return el.Properties.IsOffscreen.ValueOrDefault; } + catch { return false; } + } + + private static BoundsDto? SafeBounds(AutomationElement el) + { + try + { + var r = el.BoundingRectangle; + if (r.IsEmpty) return null; + return new BoundsDto + { + X = r.X, + Y = r.Y, + Width = r.Width, + Height = r.Height, + }; + } + catch { return null; } + } + + /// Extract a "current value" string from whichever pattern + /// the element supports — ValuePattern (most controls), + /// RangeValuePattern (sliders/progress), TextPattern.DocumentRange + /// (Edit / Document). + private static string? ExtractValue(AutomationElement el) + { + try + { + var patterns = el.Patterns; + + if (patterns.Value.IsSupported) + { + var v = patterns.Value.Pattern.Value.ValueOrDefault; + if (!string.IsNullOrEmpty(v)) return v; + } + + if (patterns.RangeValue.IsSupported) + { + var v = patterns.RangeValue.Pattern.Value.ValueOrDefault; + return v.ToString("0.###"); + } + + if (patterns.Text.IsSupported) + { + // Restrict to first 2 KB; some Documents are huge and + // we don't want to ship a novel over the wire by + // accident. + try + { + var range = patterns.Text.Pattern.DocumentRange; + var text = range.GetText(2048); + return string.IsNullOrEmpty(text) ? null : text; + } + catch { /* fall through */ } + } + + // Toggle / SelectionItem are surfaced via Selected/Aria instead. + } + catch { /* swallow */ } + return null; + } + + private static bool? ExtractReadOnly(AutomationElement el) + { + try + { + var p = el.Patterns; + if (p.Value.IsSupported) + { + return p.Value.Pattern.IsReadOnly.ValueOrDefault; + } + if (p.RangeValue.IsSupported) + { + return p.RangeValue.Pattern.IsReadOnly.ValueOrDefault; + } + } + catch { /* swallow */ } + return null; + } + + private static bool? ExtractSelected(AutomationElement el) + { + try + { + var p = el.Patterns; + if (p.SelectionItem.IsSupported) + { + return p.SelectionItem.Pattern.IsSelected.ValueOrDefault; + } + } + catch { /* swallow */ } + return null; + } + + private static bool? ExtractExpanded(AutomationElement el) + { + try + { + var p = el.Patterns; + if (p.ExpandCollapse.IsSupported) + { + return p.ExpandCollapse.Pattern.ExpandCollapseState.ValueOrDefault + == ExpandCollapseState.Expanded; + } + } + catch { /* swallow */ } + return null; + } + + private static AriaStateDto? BuildAria(AutomationElement el) + { + try + { + var p = el.Patterns; + + if (p.Toggle.IsSupported) + { + var st = p.Toggle.Pattern.ToggleState.ValueOrDefault; + return new AriaStateDto + { + Pressed = st == ToggleState.On, + Checked = st switch + { + ToggleState.On => (object)true, + ToggleState.Off => false, + ToggleState.Indeterminate => "mixed", + _ => false, + }, + }; + } + } + catch { /* swallow */ } + return null; + } + + private static bool IsAncestorOrSelf(AutomationElement node, AutomationElement maybeAncestor) + { + try + { + var ancHwnd = GetHwnd(maybeAncestor); + var current = node; + while (current != null) + { + if (GetHwnd(current) == ancHwnd && ancHwnd != IntPtr.Zero) return true; + if (current.Equals(maybeAncestor)) return true; + current = current.Parent; + } + } + catch { /* swallow */ } + return false; + } + + private static IntPtr GetHwnd(AutomationElement el) + { + try { return el.Properties.NativeWindowHandle.ValueOrDefault; } + catch { return IntPtr.Zero; } + } + + private static string EncodeHwnd(IntPtr hwnd) => + $"hwnd:0x{hwnd.ToInt64():X8}"; + + private static bool TryDecodeHwnd(string encoded, out IntPtr hwnd) + { + hwnd = IntPtr.Zero; + if (string.IsNullOrEmpty(encoded)) return false; + var s = encoded.StartsWith("hwnd:", StringComparison.OrdinalIgnoreCase) + ? encoded[5..] : encoded; + if (s.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) s = s[2..]; + if (long.TryParse(s, System.Globalization.NumberStyles.HexNumber, null, out var v)) + { + hwnd = new IntPtr(v); + return true; + } + return false; + } +} + diff --git a/apps/agent-runner/bridges/windows/scripts/uia-smoke-test.ps1 b/apps/agent-runner/bridges/windows/scripts/uia-smoke-test.ps1 new file mode 100644 index 0000000..27afa11 --- /dev/null +++ b/apps/agent-runner/bridges/windows/scripts/uia-smoke-test.ps1 @@ -0,0 +1,117 @@ +# UIA smoke test for agentmark-bridge-windows. +# +# Drives the bridge with three requests: +# 1. list_windows -> enumerate top-level windows visible to UIA +# 2. capture -> snapshot the first window from the list +# 3. capture -> snapshot the focused window (no target) +# +# Prints all responses; asserts the shapes are sane (the bridge +# claims kind:windows, returns at least one window from list_windows, +# returns a root element from capture). Designed to be safe to run +# while Notepad / Calculator / any GUI app is open in the VM console; +# without a GUI app open the test still passes against whatever ghost +# windows Windows always has (Desktop, NotificationCenter, etc.). + +$ErrorActionPreference = 'Stop' + +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$exe = Join-Path $scriptDir '..\bin\Debug\net8.0-windows\agentmark-bridge-windows.exe' +if (-not (Test-Path $exe)) { + Write-Error "Bridge exe not built. Expected at $exe." + exit 1 +} + +Write-Host "UIA smoke testing $exe" + +$psi = New-Object System.Diagnostics.ProcessStartInfo +$psi.FileName = $exe +$psi.UseShellExecute = $false +$psi.RedirectStandardInput = $true +$psi.RedirectStandardOutput = $true +$psi.RedirectStandardError = $true +$proc = [System.Diagnostics.Process]::Start($psi) + +# Helper: send one request, read one response. +function Send-Request($p, $payload) { + $p.StandardInput.WriteLine($payload) + $p.StandardInput.Flush() + return $p.StandardOutput.ReadLine() +} + +$listLine = Send-Request $proc '{"jsonrpc":"2.0","id":1,"method":"list_windows"}' +$listResp = $listLine | ConvertFrom-Json +if (-not $listResp.result.windows) { + Write-Error "list_windows returned no windows array: $listLine" + $proc.StandardInput.Close() + $proc.WaitForExit(5000) | Out-Null + exit 1 +} +$windows = $listResp.result.windows +Write-Host "list_windows: $($windows.Count) windows" +$windows | ForEach-Object { + Write-Host (" [{0}] {1} ({2}) -- {3}" -f $_.processName, $_.windowTitle, $_.windowId, $_.windowClass) +} + +if ($windows.Count -eq 0) { + Write-Host "" + Write-Host "No windows visible via UIA from this session. This is expected when" + Write-Host "running over SSH non-interactive sessions if no GUI app is open in the" + Write-Host "VM console. Open Notepad in the VM console and rerun." -ForegroundColor Yellow + $proc.StandardInput.Close() + $proc.WaitForExit(5000) | Out-Null + exit 0 +} + +# Capture the first window from the list to validate end-to-end. +$first = $windows[0] +Write-Host "" +Write-Host "Capturing first window by windowId: $($first.windowId)" +$capPayload = '{"jsonrpc":"2.0","id":2,"method":"capture","params":{"windowId":"' + $first.windowId + '","maxDepth":4,"maxElements":200}}' +$capLine = Send-Request $proc $capPayload +$capResp = $capLine | ConvertFrom-Json + +if (-not $capResp.result) { + Write-Error "capture did not return result. Response was: $capLine" + $proc.StandardInput.Close() + $proc.WaitForExit(5000) | Out-Null + exit 1 +} + +$cap = $capResp.result +Write-Host " platform : $($cap.platform)" +Write-Host " windowTitle : $($cap.windowTitle)" +Write-Host " processName : $($cap.processName)" +Write-Host " treeDepth : $($cap.treeDepth)" +Write-Host " elementCount : $($cap.elementCount)" +Write-Host " root.role : $($cap.root.role)" +Write-Host " root.id : $($cap.root.id)" +Write-Host " root.name : $($cap.root.name)" +if ($cap.root.children) { + Write-Host " root children : $($cap.root.children.Count)" + $cap.root.children | Select-Object -First 5 | ForEach-Object { + Write-Host (" - [{0}] id={1} name={2}" -f $_.role, $_.id, $_.name) + } +} + +# Capture the focused window via the no-target default. +Write-Host "" +Write-Host "Capturing focused window (no target)..." +$focLine = Send-Request $proc '{"jsonrpc":"2.0","id":3,"method":"capture","params":{"maxDepth":3,"maxElements":50}}' +$focResp = $focLine | ConvertFrom-Json +if ($focResp.result) { + Write-Host " focused window : $($focResp.result.windowTitle) ($($focResp.result.processName))" +} elseif ($focResp.error) { + Write-Host " focused capture error: $($focResp.error.message)" +} + +$proc.StandardInput.Close() +$proc.WaitForExit(5000) | Out-Null + +$stderr = $proc.StandardError.ReadToEnd() +if ($stderr) { + Write-Host "--- bridge stderr ---" -ForegroundColor DarkGray + Write-Host $stderr -ForegroundColor DarkGray +} + +Write-Host "" +Write-Host "UIA SMOKE TEST PASSED" -ForegroundColor Green