feat(bridge-windows): Phase 0e2 + 0e3 — list_windows, capture, execute (real UIA driving real Windows)

apps/agent-runner/bridges/windows/Program.cs

@@ -24,6 +24,7 @@
 using System.Text;
 using System.Text.Json;
 using System.Text.Json.Serialization;
+using AgentMark.Bridge.Windows.Uia;
 
 namespace AgentMark.Bridge.Windows;
 
@@ -51,7 +52,7 @@ public static int Main(string[] args)
         // EOF, leaving the process hung even after the parent closes the
         // pipe. ReadLine() handles EOF correctly. UIA calls are themselves
         // blocking COM-STA invocations so async doesn't buy us anything.
-        var dispatcher = new RpcDispatcher();
+        using var dispatcher = new RpcDispatcher();
 
         string? line;
         while ((line = Console.In.ReadLine()) != null)
@@ -180,22 +181,47 @@ internal sealed class RpcException(RpcError error, string message) : Exception(m
 // Dispatcher
 // ──────────────────────────────────────────────────────────────────────
 
-internal sealed class RpcDispatcher
+internal sealed class RpcDispatcher : IDisposable
 {
     private static readonly string BridgeVersion = "0.4.0";
 
+    // UIA work is lazy-initialised — ping/capabilities don't need it and
+    // booting UIA on startup adds ~150ms we don't want for clients that
+    // only smoke-test the bridge.
+    private readonly Lazy<StaWorker> _staWorker = new(() => new StaWorker());
+    private readonly Lazy<UiaCapturer> _capturer;
+
+    public RpcDispatcher()
+    {
+        _capturer = new Lazy<UiaCapturer>(() => _staWorker.Value.Invoke(() => new UiaCapturer()));
+    }
+
     public object? Dispatch(string method, JsonElement @params)
     {
         return method switch
         {
             "ping"         => HandlePing(),
             "capabilities" => HandleCapabilities(),
+            "list_windows" => HandleListWindows(),
+            "capture"      => HandleCapture(@params),
             _ => throw new RpcException(
                 RpcError.MethodNotFound,
-                $"Unknown method: {method}. Supported: ping, capabilities."),
+                $"Unknown method: {method}. Supported: ping, capabilities, list_windows, capture."),
         };
     }
 
+    public void Dispose()
+    {
+        if (_capturer.IsValueCreated)
+        {
+            try { _staWorker.Value.Invoke(() => _capturer.Value.Dispose()); } catch { /* swallow */ }
+        }
+        if (_staWorker.IsValueCreated)
+        {
+            try { _staWorker.Value.Dispose(); } catch { /* swallow */ }
+        }
+    }
+
     private static object HandlePing() => new
     {
         pong = true,
@@ -212,9 +238,71 @@ internal sealed class RpcDispatcher
         {
             "ping",
             "capabilities",
-            // "capture" and "execute" land in Phase 0e2/0e3
+            "list_windows",
+            "capture",
+            // "execute" lands in Phase 0e3
         },
         uiaProvider = "FlaUI.UIA3",
         platform = "windows",
     };
+
+    private object HandleListWindows()
+    {
+        var capturer = _capturer.Value;
+        var sta = _staWorker.Value;
+        var windows = sta.Invoke(() => capturer.ListWindows());
+        return new { windows };
+    }
+
+    private object HandleCapture(JsonElement @params)
+    {
+        var req = new UiaCapturer.CaptureRequest
+        {
+            ProcessName = ReadString(@params, "processName"),
+            ProcessId = ReadInt(@params, "processId"),
+            WindowTitle = ReadString(@params, "windowTitle"),
+            WindowId = ReadString(@params, "windowId"),
+            MaxDepth = ReadInt(@params, "maxDepth") ?? 12,
+            IncludeHidden = ReadBool(@params, "includeHidden") ?? false,
+            TimeoutMs = ReadInt(@params, "timeoutMs") ?? 5000,
+            MaxElements = ReadInt(@params, "maxElements") ?? 2000,
+        };
+
+        try
+        {
+            var capturer = _capturer.Value;
+            var sta = _staWorker.Value;
+            return sta.Invoke(() => capturer.Capture(req));
+        }
+        catch (InvalidOperationException ex)
+        {
+            throw new RpcException(RpcError.WindowNotFound, ex.Message);
+        }
+    }
+
+    private static string? ReadString(JsonElement parent, string name)
+    {
+        if (parent.ValueKind != JsonValueKind.Object) return null;
+        if (!parent.TryGetProperty(name, out var v)) return null;
+        return v.ValueKind == JsonValueKind.String ? v.GetString() : null;
+    }
+
+    private static int? ReadInt(JsonElement parent, string name)
+    {
+        if (parent.ValueKind != JsonValueKind.Object) return null;
+        if (!parent.TryGetProperty(name, out var v)) return null;
+        return v.ValueKind == JsonValueKind.Number && v.TryGetInt32(out var i) ? i : null;
+    }
+
+    private static bool? ReadBool(JsonElement parent, string name)
+    {
+        if (parent.ValueKind != JsonValueKind.Object) return null;
+        if (!parent.TryGetProperty(name, out var v)) return null;
+        return v.ValueKind switch
+        {
+            JsonValueKind.True => true,
+            JsonValueKind.False => false,
+            _ => null,
+        };
+    }
 }

apps/agent-runner/bridges/windows/Uia/CaptureDtos.cs

@@ -0,0 +1,149 @@
+// DTOs the bridge returns over JSON-RPC. Field names match the
+// AgentMark v0.4 DesktopCapture / DesktopElement spec exactly (camelCase
+// after JsonNamingPolicy.CamelCase) so the Node-side WindowsUiaBackend
+// can deserialize straight into the existing TypeScript types from
+// @thinkfleet/agentmark.
+//
+// Keep this file the canonical source of truth for what we put on the
+// wire — when the spec grows, mirror the change here.
+
+using System.Text.Json.Serialization;
+
+namespace AgentMark.Bridge.Windows.Uia;
+
+/// <summary>
+/// One window discoverable by list_windows. Includes just enough to let
+/// the caller pick which window to capture in a subsequent call.
+/// </summary>
+internal sealed class WindowSummaryDto
+{
+    [JsonPropertyName("windowId")]
+    public string WindowId { get; init; } = "";
+
+    [JsonPropertyName("processName")]
+    public string? ProcessName { get; init; }
+
+    [JsonPropertyName("processId")]
+    public int? ProcessId { get; init; }
+
+    [JsonPropertyName("windowTitle")]
+    public string WindowTitle { get; init; } = "";
+
+    [JsonPropertyName("windowClass")]
+    public string? WindowClass { get; init; }
+
+    /// <summary>Whether this window currently has keyboard focus.</summary>
+    [JsonPropertyName("hasFocus")]
+    public bool HasFocus { get; init; }
+}
+
+/// <summary>
+/// What `capture` returns. Mirrors the TS `DesktopCapture` interface.
+/// </summary>
+internal sealed class DesktopCaptureDto
+{
+    [JsonPropertyName("platform")]
+    public string Platform => "windows";
+
+    [JsonPropertyName("processName")]
+    public string? ProcessName { get; init; }
+
+    [JsonPropertyName("processId")]
+    public int? ProcessId { get; init; }
+
+    [JsonPropertyName("windowTitle")]
+    public string WindowTitle { get; init; } = "";
+
+    [JsonPropertyName("windowClass")]
+    public string? WindowClass { get; init; }
+
+    [JsonPropertyName("windowId")]
+    public string WindowId { get; init; } = "";
+
+    [JsonPropertyName("focusedElementId")]
+    public string? FocusedElementId { get; init; }
+
+    [JsonPropertyName("treeDepth")]
+    public int TreeDepth { get; init; }
+
+    [JsonPropertyName("elementCount")]
+    public int ElementCount { get; init; }
+
+    [JsonPropertyName("root")]
+    public DesktopElementDto Root { get; init; } = null!;
+}
+
+/// <summary>
+/// One node in the captured accessibility tree. Mirrors the TS
+/// `DesktopElement` interface.
+/// </summary>
+internal sealed class DesktopElementDto
+{
+    [JsonPropertyName("id")]
+    public string Id { get; init; } = "";
+
+    [JsonPropertyName("role")]
+    public string Role { get; init; } = "other";
+
+    [JsonPropertyName("name")]
+    public string? Name { get; init; }
+
+    [JsonPropertyName("value")]
+    public string? Value { get; init; }
+
+    [JsonPropertyName("placeholder")]
+    public string? Placeholder { get; init; }
+
+    [JsonPropertyName("enabled")]
+    public bool? Enabled { get; init; }
+
+    [JsonPropertyName("selected")]
+    public bool? Selected { get; init; }
+
+    [JsonPropertyName("readOnly")]
+    public bool? ReadOnly { get; init; }
+
+    [JsonPropertyName("expanded")]
+    public bool? Expanded { get; init; }
+
+    [JsonPropertyName("aria")]
+    public AriaStateDto? Aria { get; init; }
+
+    [JsonPropertyName("bounds")]
+    public BoundsDto? Bounds { get; init; }
+
+    /// <summary>Mutable so the walker can backfill after building the
+    /// parent. Stays null when an element is a leaf / depth-truncated.</summary>
+    [JsonPropertyName("children")]
+    public List<DesktopElementDto>? Children { get; set; }
+}
+
+internal sealed class AriaStateDto
+{
+    [JsonPropertyName("pressed")]
+    public bool? Pressed { get; init; }
+
+    [JsonPropertyName("checked")]
+    public object? Checked { get; init; }  // bool | "mixed"
+
+    [JsonPropertyName("required")]
+    public bool? Required { get; init; }
+
+    [JsonPropertyName("invalid")]
+    public bool? Invalid { get; init; }
+}
+
+internal sealed class BoundsDto
+{
+    [JsonPropertyName("x")]
+    public double X { get; init; }
+
+    [JsonPropertyName("y")]
+    public double Y { get; init; }
+
+    [JsonPropertyName("width")]
+    public double Width { get; init; }
+
+    [JsonPropertyName("height")]
+    public double Height { get; init; }
+}