diff --git a/.gitignore b/.gitignore
index e04537d..cbf0247 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,20 @@ dist/
# Tesseract.js downloads language data into cwd by default
*.traineddata
+
+# .NET / MSBuild output (used by apps/agent-runner/bridges/windows)
+bin/
+obj/
+*.suo
+*.user
+*.userprefs
+
+# JetBrains
+.idea/
+*.iml
+
+# Visual Studio
+.vs/
+
+# Local dev junk
+*.swp
diff --git a/apps/agent-runner/bridges/windows/AgentMark.Bridge.Windows.csproj b/apps/agent-runner/bridges/windows/AgentMark.Bridge.Windows.csproj
new file mode 100644
index 0000000..aa23fa4
--- /dev/null
+++ b/apps/agent-runner/bridges/windows/AgentMark.Bridge.Windows.csproj
@@ -0,0 +1,32 @@
+
+
+
+ Exe
+ net8.0-windows
+ AgentMark.Bridge.Windows
+ agentmark-bridge-windows
+ enable
+ enable
+ 12.0
+ true
+
+ $(MSBuildThisFileDirectory)bin\
+ $(MSBuildThisFileDirectory)obj\
+
+
+
+
+ AgentMark Windows UIA bridge — accessibility-tree capture and action execution for kind:'desktop' snapshots.
+ ThinkFleet
+ ThinkFleet
+ AgentMark Desktop
+
+
+
+
+
+
+
+
diff --git a/apps/agent-runner/bridges/windows/Program.cs b/apps/agent-runner/bridges/windows/Program.cs
new file mode 100644
index 0000000..0f1fa15
--- /dev/null
+++ b/apps/agent-runner/bridges/windows/Program.cs
@@ -0,0 +1,220 @@
+// AgentMark Windows UIA Bridge
+// ----------------------------
+// Stdio JSON-RPC 2.0 server. Parent process (typically the Node-side
+// `WindowsUiaBackend` running inside the AgentMark MCP server, or a
+// ThinkFleet SaaS agent) launches this exe and talks to it over
+// stdin/stdout. One line of JSON per message.
+//
+// Protocol:
+// Request : { "jsonrpc": "2.0", "id": , "method": "", "params": {...} }
+// Response: { "jsonrpc": "2.0", "id": , "result": {...} } on success
+// { "jsonrpc": "2.0", "id": , "error": { "code": N, "message": "..." } } on failure
+//
+// All diagnostic output goes to stderr — never stdout — so the framing stays clean.
+//
+// Methods (current set; more land as Phase 0e progresses):
+// ping — returns { pong: true, version: "...", arch: "arm64|x64" }
+// capabilities— returns { methods: [...], uia_version: "..." }
+//
+// Phase 0e2/0e3 will add:
+// capture — walk a window's UIA tree, return DesktopCapture JSON
+// execute — drive an action by element_id (click/type/select/etc.)
+// list_windows— enumerate top-level windows visible to the bridge
+
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace AgentMark.Bridge.Windows;
+
+internal static class Program
+{
+ // JSON serialization options — camelCase to match the AgentMark
+ // wire format on the Node side.
+ internal static readonly JsonSerializerOptions JsonOpts = new()
+ {
+ PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+ DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+ };
+
+ public static int Main(string[] args)
+ {
+ // Force UTF-8 on both pipes -- Windows defaults can mangle non-ASCII.
+ // Use UTF8Encoding(false) to avoid emitting a BOM on stdout, which
+ // would break JSON-RPC framing on the parent side.
+ Console.InputEncoding = new UTF8Encoding(false);
+ Console.OutputEncoding = new UTF8Encoding(false);
+ Console.Error.WriteLine($"[bridge] agentmark-bridge-windows starting (pid={Environment.ProcessId}, arch={System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture})");
+
+ // Synchronous read loop. Async stdin reads on Windows pipes have a
+ // known issue where ReadLineAsync() doesn't always return null on
+ // EOF, leaving the process hung even after the parent closes the
+ // pipe. ReadLine() handles EOF correctly. UIA calls are themselves
+ // blocking COM-STA invocations so async doesn't buy us anything.
+ var dispatcher = new RpcDispatcher();
+
+ string? line;
+ while ((line = Console.In.ReadLine()) != null)
+ {
+ line = line.Trim();
+ if (line.Length == 0) continue;
+
+ JsonElement reqId = default;
+ try
+ {
+ using var doc = JsonDocument.Parse(line);
+ var root = doc.RootElement;
+ reqId = root.TryGetProperty("id", out var idEl) ? idEl.Clone() : default;
+
+ var method = root.GetProperty("method").GetString()
+ ?? throw new RpcException(RpcError.InvalidRequest, "method is required");
+ var paramsEl = root.TryGetProperty("params", out var p) ? p : default;
+
+ var result = dispatcher.Dispatch(method, paramsEl);
+ Console.Out.WriteLine(EncodeSuccess(reqId, result));
+ }
+ catch (RpcException rex)
+ {
+ Console.Out.WriteLine(EncodeError(reqId, rex.Error.Code, rex.Message));
+ }
+ catch (JsonException jex)
+ {
+ Console.Out.WriteLine(EncodeError(reqId, RpcError.ParseError.Code, $"JSON parse error: {jex.Message}"));
+ }
+ catch (Exception ex)
+ {
+ Console.Error.WriteLine($"[bridge] unhandled: {ex}");
+ Console.Out.WriteLine(EncodeError(reqId, RpcError.InternalError.Code, ex.Message));
+ }
+
+ Console.Out.Flush();
+ }
+
+ Console.Error.WriteLine("[bridge] stdin closed; exiting");
+ return 0;
+ }
+
+ private static string EncodeSuccess(JsonElement id, object? result)
+ {
+ var env = new RpcResponseSuccess
+ {
+ Id = id.ValueKind == JsonValueKind.Undefined ? null : id,
+ Result = result,
+ };
+ return JsonSerializer.Serialize(env, JsonOpts);
+ }
+
+ private static string EncodeError(JsonElement id, int code, string message)
+ {
+ var env = new RpcResponseError
+ {
+ Id = id.ValueKind == JsonValueKind.Undefined ? null : id,
+ Error = new RpcErrorBody { Code = code, Message = message },
+ };
+ return JsonSerializer.Serialize(env, JsonOpts);
+ }
+}
+
+// ──────────────────────────────────────────────────────────────────────
+// Wire format
+// ──────────────────────────────────────────────────────────────────────
+
+internal sealed class RpcResponseSuccess
+{
+ [JsonPropertyName("jsonrpc")]
+ public string Jsonrpc => "2.0";
+
+ [JsonPropertyName("id")]
+ public JsonElement? Id { get; init; }
+
+ [JsonPropertyName("result")]
+ public object? Result { get; init; }
+}
+
+internal sealed class RpcResponseError
+{
+ [JsonPropertyName("jsonrpc")]
+ public string Jsonrpc => "2.0";
+
+ [JsonPropertyName("id")]
+ public JsonElement? Id { get; init; }
+
+ [JsonPropertyName("error")]
+ public RpcErrorBody Error { get; init; } = null!;
+}
+
+internal sealed class RpcErrorBody
+{
+ [JsonPropertyName("code")]
+ public int Code { get; init; }
+
+ [JsonPropertyName("message")]
+ public string Message { get; init; } = "";
+}
+
+// ──────────────────────────────────────────────────────────────────────
+// Error catalog
+// ──────────────────────────────────────────────────────────────────────
+
+internal readonly record struct RpcError(int Code)
+{
+ public static readonly RpcError ParseError = new(-32700);
+ public static readonly RpcError InvalidRequest = new(-32600);
+ public static readonly RpcError MethodNotFound = new(-32601);
+ public static readonly RpcError InvalidParams = new(-32602);
+ public static readonly RpcError InternalError = new(-32603);
+
+ // Bridge-specific (above -32000)
+ public static readonly RpcError WindowNotFound = new(-32010);
+ public static readonly RpcError ElementNotFound = new(-32011);
+ public static readonly RpcError UnsupportedPattern = new(-32012);
+ public static readonly RpcError ActionFailed = new(-32013);
+}
+
+internal sealed class RpcException(RpcError error, string message) : Exception(message)
+{
+ public RpcError Error { get; } = error;
+}
+
+// ──────────────────────────────────────────────────────────────────────
+// Dispatcher
+// ──────────────────────────────────────────────────────────────────────
+
+internal sealed class RpcDispatcher
+{
+ private static readonly string BridgeVersion = "0.4.0";
+
+ public object? Dispatch(string method, JsonElement @params)
+ {
+ return method switch
+ {
+ "ping" => HandlePing(),
+ "capabilities" => HandleCapabilities(),
+ _ => throw new RpcException(
+ RpcError.MethodNotFound,
+ $"Unknown method: {method}. Supported: ping, capabilities."),
+ };
+ }
+
+ private static object HandlePing() => new
+ {
+ pong = true,
+ version = BridgeVersion,
+ arch = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture.ToString().ToLowerInvariant(),
+ processId = Environment.ProcessId,
+ };
+
+ private static object HandleCapabilities() => new
+ {
+ bridge = "agentmark-bridge-windows",
+ version = BridgeVersion,
+ methods = new[]
+ {
+ "ping",
+ "capabilities",
+ // "capture" and "execute" land in Phase 0e2/0e3
+ },
+ uiaProvider = "FlaUI.UIA3",
+ platform = "windows",
+ };
+}
diff --git a/apps/agent-runner/bridges/windows/README.md b/apps/agent-runner/bridges/windows/README.md
new file mode 100644
index 0000000..7a47260
--- /dev/null
+++ b/apps/agent-runner/bridges/windows/README.md
@@ -0,0 +1,85 @@
+# AgentMark Windows UIA Bridge
+
+The Windows-side sidecar process for AgentMark Desktop. Walks the OS
+accessibility tree via UIA (using [FlaUI](https://github.com/FlaUI/FlaUI),
+MIT) and exposes capture/execute operations the Node-side
+`WindowsUiaBackend` consumes through stdio JSON-RPC.
+
+This is one implementation of the `DesktopCaptureBackend` interface
+defined in `@thinkfleet/agentmark` v0.4. macOS (AXAPI) and Linux (AT-SPI)
+bridges follow the same protocol.
+
+## Build
+
+Requires the .NET 8 SDK. From this directory:
+
+```powershell
+dotnet build
+```
+
+Output exe: `bin\Debug\net8.0-windows\agentmark-bridge-windows.exe`.
+
+## Run + smoke test
+
+The bridge speaks stdio JSON-RPC 2.0. One JSON message per line.
+
+```powershell
+.\bin\Debug\net8.0-windows\agentmark-bridge-windows.exe
+```
+
+It blocks waiting on stdin. Paste a `ping`:
+
+```
+{"jsonrpc":"2.0","id":1,"method":"ping"}
+```
+
+Expected response (single line, here pretty-printed):
+
+```json
+{
+ "jsonrpc": "2.0",
+ "id": 1,
+ "result": {
+ "pong": true,
+ "version": "0.4.0",
+ "arch": "arm64",
+ "processId": 12345
+ }
+}
+```
+
+Ctrl-Z then Enter closes stdin and the bridge exits cleanly.
+
+## Protocol
+
+All responses are JSON-RPC 2.0. Diagnostic output goes to stderr only —
+stdout is reserved for framed JSON messages.
+
+| Method | Status | Description |
+|---|---|---|
+| `ping` | Phase 0e1 ✅ | Liveness check, returns pong + bridge version |
+| `capabilities` | Phase 0e1 ✅ | Lists supported methods and UIA provider info |
+| `list_windows` | Phase 0e2 (planned) | Enumerate top-level windows visible to UIA |
+| `capture` | Phase 0e2 (planned) | Walk a window's UIA tree → `DesktopCapture` JSON |
+| `execute` | Phase 0e3 (planned) | Drive an action by element_id (click/type/select/etc.) |
+
+## Architecture notes
+
+- **Single-threaded for now.** UIA is COM-STA; once `capture` / `execute`
+ land we marshal those calls onto a dedicated STA thread.
+- **UTF-8 forced on both pipes.** Windows defaults will mangle non-ASCII
+ in window titles / cell values otherwise.
+- **All stderr goes to the parent process's stderr.** Useful for log
+ aggregation in the AgentMark MCP server when this bridge is spawned
+ as a subprocess.
+
+## Security posture
+
+This bridge runs as the logged-in user (never SYSTEM/admin). It can
+only see/drive windows that user can already see/drive — no privilege
+escalation. Stdio mode has zero network surface; only the parent
+process that spawned this exe can write to its stdin.
+
+A future `--transport=ws` mode (Phase 0f) will add a localhost-only
+WebSocket transport with auth-token gating, origin-header rejection,
+and process-identity verification for multi-consumer scenarios.
diff --git a/apps/agent-runner/bridges/windows/scripts/smoke-test.ps1 b/apps/agent-runner/bridges/windows/scripts/smoke-test.ps1
new file mode 100644
index 0000000..d8c67f4
--- /dev/null
+++ b/apps/agent-runner/bridges/windows/scripts/smoke-test.ps1
@@ -0,0 +1,101 @@
+# Smoke test for the agentmark-bridge-windows binary.
+#
+# Spawns the bridge as a child process via .NET's Process API (NOT
+# PowerShell's pipe), writes a few JSON-RPC requests, then closes
+# stdin explicitly -- that is the only reliable way to signal EOF to
+# the bridge so it exits cleanly. PowerShell's `... | & exe` pipe
+# does NOT close stdin on the native exe; that's why the naive
+# version hangs.
+
+$ErrorActionPreference = 'Stop'
+
+$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
+$exe = Join-Path $scriptDir '..\bin\Debug\net8.0-windows\agentmark-bridge-windows.exe'
+
+if (-not (Test-Path $exe)) {
+ Write-Error "Bridge exe not built -- expected at $exe. Run 'dotnet build' first."
+ exit 1
+}
+
+Write-Host "Smoke testing $exe"
+
+$psi = New-Object System.Diagnostics.ProcessStartInfo
+$psi.FileName = $exe
+$psi.UseShellExecute = $false
+$psi.RedirectStandardInput = $true
+$psi.RedirectStandardOutput = $true
+$psi.RedirectStandardError = $true
+# StandardInputEncoding/OutputEncoding only exist on .NET Core/5+;
+# Windows PowerShell 5.1 runs on .NET Framework which lacks them.
+# Our JSON payloads are pure ASCII so the system codepage default works.
+# The bridge process itself forces UTF-8 on its own streams (Program.cs).
+
+$proc = [System.Diagnostics.Process]::Start($psi)
+
+$requests = @(
+ '{"jsonrpc":"2.0","id":1,"method":"ping"}',
+ '{"jsonrpc":"2.0","id":2,"method":"capabilities"}'
+)
+
+foreach ($req in $requests) {
+ $proc.StandardInput.WriteLine($req)
+}
+$proc.StandardInput.Close()
+
+# 10-second cap to keep the test sane.
+if (-not $proc.WaitForExit(10000)) {
+ $proc.Kill()
+ Write-Error "Bridge did not exit within 10 seconds."
+ exit 1
+}
+
+$stdout = $proc.StandardOutput.ReadToEnd()
+$stderr = $proc.StandardError.ReadToEnd()
+
+if ($stderr) {
+ Write-Host "--- bridge stderr ---" -ForegroundColor DarkGray
+ Write-Host $stderr -ForegroundColor DarkGray
+}
+
+$responseLines = $stdout -split "`r?`n" | Where-Object { $_.Trim().Length -gt 0 }
+Write-Host "Received $($responseLines.Count) response line(s)."
+
+if ($responseLines.Count -lt 2) {
+ Write-Error "Expected 2 responses; got $($responseLines.Count)."
+ Write-Host "stdout was:"
+ Write-Host $stdout
+ exit 1
+}
+
+foreach ($line in $responseLines) {
+ Write-Host " -> $line"
+}
+
+$pingResp = $responseLines[0] | ConvertFrom-Json
+$capResp = $responseLines[1] | ConvertFrom-Json
+
+if ($pingResp.result.pong -ne $true) {
+ Write-Error "ping: expected result.pong=true, got: $($pingResp | ConvertTo-Json -Compress)"
+ exit 1
+}
+if ($pingResp.id -ne 1) {
+ Write-Error "ping: id mismatch (expected 1, got $($pingResp.id))"
+ exit 1
+}
+
+if ($capResp.result.bridge -ne 'agentmark-bridge-windows') {
+ Write-Error "capabilities: expected bridge=agentmark-bridge-windows, got: $($capResp | ConvertTo-Json -Compress)"
+ exit 1
+}
+if ($capResp.id -ne 2) {
+ Write-Error "capabilities: id mismatch (expected 2, got $($capResp.id))"
+ exit 1
+}
+
+Write-Host ""
+Write-Host "SMOKE TEST PASSED" -ForegroundColor Green
+Write-Host " bridge version : $($pingResp.result.version)"
+Write-Host " architecture : $($pingResp.result.arch)"
+Write-Host " process id : $($pingResp.result.processId)"
+Write-Host " methods : $($capResp.result.methods -join ', ')"
+Write-Host " exit code : $($proc.ExitCode)"
diff --git a/src/desktop/body-builder.ts b/src/desktop/body-builder.ts
new file mode 100644
index 0000000..65c1c21
--- /dev/null
+++ b/src/desktop/body-builder.ts
@@ -0,0 +1,311 @@
+/**
+ * Walk a captured desktop accessibility tree and emit:
+ *
+ * 1. A markdown body that renders the UI as structured text agents
+ * can read. Headings for windows/panes/dialogs; lists for menus
+ * and tab lists; tables for grid controls; inline tags for
+ * interactive elements.
+ *
+ * 2. An `actions` map keyed by `act_` that the runtime later
+ * passes to the backend's `execute()` call.
+ *
+ * The tag conventions mirror the rest of AgentMark:
+ *
+ * - `[ACTION:act_save]` — clickable / triggers (button, link, menu_item)
+ * - `[INPUT:act_first_name]` — editable controls (text_input, combo_box,
+ * check_box, radio_button, slider, etc.)
+ * - `[ELEMENT:e_status]` — non-interactive references the agent might
+ * cite (status text, table cell IDs)
+ * - `[WINDOW:w_excel_1]` — window boundary marker for multi-window
+ * captures
+ *
+ * Interactive elements always land in the `actions` map. Static text
+ * usually just appears in body text — but when the producer wants to
+ * give the agent a stable reference, it emits `[ELEMENT:e_x]` and
+ * doesn't add to actions.
+ */
+
+import type { ActionDefinition, ActionType } from '../types'
+import type { DesktopCapture, DesktopElement, DesktopRole } from './types'
+
+export interface BuildDesktopBodyResult {
+ /** The markdown body. */
+ body: string
+ /** Actions discovered while walking the tree, keyed by act_. */
+ actions: Record
+ /** Mapping action ID → original desktop element_id. The runtime
+ * pushes this into the ActionBinding so that `execute(act_X)`
+ * translates to `backend.execute({ element_id: ... })`. */
+ element_ids: Record
+ /** Whether the tree contained any interactive elements. */
+ has_interactive: boolean
+}
+
+const ESCAPE_REGEX = /\[(?=[A-Z])/g
+
+function escapeBody(text: string): string {
+ return text.replace(/\\/g, '\\\\').replace(ESCAPE_REGEX, '\\[')
+}
+
+/** Roles whose elements become entries in the `actions` map. */
+const INTERACTIVE_ROLES: ReadonlySet = new Set([
+ 'button',
+ 'split_button',
+ 'menu_item',
+ 'tab',
+ 'link',
+ 'text_input',
+ 'password_input',
+ 'text_area',
+ 'check_box',
+ 'radio_button',
+ 'combo_box',
+ 'list_box',
+ 'slider',
+ 'tree_item',
+ 'list_item',
+])
+
+/** Map normalised desktop role → AgentMark ActionType. */
+function actionTypeFor(role: DesktopRole): ActionType {
+ switch (role) {
+ case 'text_input':
+ case 'password_input':
+ case 'text_area':
+ return 'type'
+ case 'check_box':
+ return 'check'
+ case 'radio_button':
+ return 'check'
+ case 'combo_box':
+ case 'list_box':
+ return 'select'
+ case 'slider':
+ return 'range'
+ default:
+ return 'click'
+ }
+}
+
+/** Tag used for an element — INPUT for editable controls, ACTION otherwise. */
+function tagKindFor(role: DesktopRole): 'INPUT' | 'ACTION' {
+ switch (role) {
+ case 'text_input':
+ case 'password_input':
+ case 'text_area':
+ case 'check_box':
+ case 'radio_button':
+ case 'combo_box':
+ case 'list_box':
+ case 'slider':
+ return 'INPUT'
+ default:
+ return 'ACTION'
+ }
+}
+
+interface BuilderState {
+ actions: Record
+ element_ids: Record
+ seen: Set
+ has_interactive: boolean
+}
+
+function uniqueActionId(state: BuilderState, raw: string): string {
+ // Sanitise to AgentMark action ID rules: lowercase, alnum + underscore,
+ // starts with a letter.
+ const base = `act_${raw.toLowerCase().replace(/[^a-z0-9_]/g, '_').replace(/^_+/, '').slice(0, 56) || 'el'}`
+ if (!state.seen.has(base)) {
+ state.seen.add(base)
+ return base
+ }
+ let n = 2
+ while (state.seen.has(`${base}_${n}`)) n++
+ const id = `${base}_${n}`
+ state.seen.add(id)
+ return id
+}
+
+export function buildDesktopBody(capture: DesktopCapture): BuildDesktopBodyResult {
+ const state: BuilderState = {
+ actions: {},
+ element_ids: {},
+ seen: new Set(),
+ has_interactive: false,
+ }
+
+ const lines: string[] = []
+ // Top-of-body window marker so multi-window producers (or future
+ // multi-tab captures) can interleave consistently.
+ const winId = `w_${(capture.process_name ?? 'window').toLowerCase().replace(/[^a-z0-9_]/g, '_').slice(0, 24) || 'app'}`
+ lines.push(`[WINDOW:${winId}]`)
+ lines.push('')
+ lines.push(`# ${capture.window_title || '(untitled window)'}`)
+ lines.push('')
+
+ renderElement(capture.root, lines, state, 0)
+
+ // Trim trailing blank lines
+ while (lines.length && lines[lines.length - 1] === '') lines.pop()
+
+ return {
+ body: lines.join('\n') + '\n',
+ actions: state.actions,
+ element_ids: state.element_ids,
+ has_interactive: state.has_interactive,
+ }
+}
+
+function renderElement(
+ el: DesktopElement,
+ lines: string[],
+ state: BuilderState,
+ depth: number,
+): void {
+ const role = el.role
+
+ // Containers — render a heading or list marker, then recurse.
+ if (role === 'window' || role === 'pane' || role === 'dialog' || role === 'group') {
+ if (el.name) {
+ const level = Math.min(2 + depth, 6)
+ lines.push(`${'#'.repeat(level)} ${escapeBody(el.name)}`)
+ lines.push('')
+ }
+ renderChildren(el, lines, state, depth + 1)
+ return
+ }
+
+ if (role === 'toolbar' || role === 'menu' || role === 'tab_list') {
+ if (el.name) {
+ lines.push(`**${escapeBody(el.name)}**`)
+ lines.push('')
+ }
+ renderChildren(el, lines, state, depth + 1)
+ return
+ }
+
+ if (role === 'tree' || role === 'list') {
+ if (el.name) {
+ lines.push(`**${escapeBody(el.name)}**`)
+ lines.push('')
+ }
+ for (const child of el.children ?? []) {
+ const item = renderInteractiveAsTag(child, state)
+ if (item) {
+ lines.push(`- ${item}`)
+ } else if (child.name) {
+ lines.push(`- ${escapeBody(child.name)}`)
+ } else {
+ renderElement(child, lines, state, depth + 1)
+ }
+ }
+ lines.push('')
+ return
+ }
+
+ if (role === 'table') {
+ renderTable(el, lines, state)
+ return
+ }
+
+ // Interactive leaves — emit a tag inline, push action to map.
+ if (INTERACTIVE_ROLES.has(role)) {
+ const tag = renderInteractiveAsTag(el, state)
+ if (tag) {
+ lines.push(tag)
+ lines.push('')
+ }
+ return
+ }
+
+ // Static text / labels / status — render plain.
+ if (role === 'label' || role === 'static_text' || role === 'status_bar') {
+ if (el.name || el.value) {
+ lines.push(escapeBody(el.value ?? el.name ?? ''))
+ lines.push('')
+ }
+ return
+ }
+
+ // Anything else — recurse into children but otherwise ignore.
+ renderChildren(el, lines, state, depth + 1)
+}
+
+function renderChildren(
+ el: DesktopElement,
+ lines: string[],
+ state: BuilderState,
+ depth: number,
+): void {
+ for (const child of el.children ?? []) {
+ renderElement(child, lines, state, depth)
+ }
+}
+
+function renderInteractiveAsTag(el: DesktopElement, state: BuilderState): string | null {
+ const actionId = uniqueActionId(state, el.id || el.name || el.role)
+ const def: ActionDefinition = {
+ type: actionTypeFor(el.role),
+ label: el.name || el.value || el.id || el.role,
+ }
+ if (el.value !== undefined) def.value = el.value
+ if (el.placeholder !== undefined) def.placeholder = el.placeholder
+ if (el.enabled === false) {
+ def.disabled = true
+ }
+ if (el.read_only === true) def.read_only = true
+ if (el.aria) {
+ def.aria = {}
+ if (el.aria.pressed !== undefined) def.aria.pressed = el.aria.pressed
+ if (el.aria.checked !== undefined) def.aria.checked = el.aria.checked
+ }
+ if (el.selected !== undefined) {
+ def.aria = def.aria ?? {}
+ def.aria.selected = el.selected
+ }
+ if (el.expanded !== undefined) {
+ def.aria = def.aria ?? {}
+ def.aria.expanded = el.expanded
+ }
+
+ state.actions[actionId] = def
+ if (el.id) state.element_ids[actionId] = el.id
+ state.has_interactive = true
+
+ const tagKind = tagKindFor(el.role)
+ return `[${tagKind}:${actionId}]`
+}
+
+function renderTable(el: DesktopElement, lines: string[], state: BuilderState): void {
+ if (el.name) {
+ lines.push(`**${escapeBody(el.name)}**`)
+ lines.push('')
+ }
+ const rows = (el.children ?? []).filter(c => c.role === 'row')
+ const headerRow = (el.children ?? []).find(c => c.role === 'row' && (c.children ?? []).some(cc => cc.role === 'column_header'))
+ const headers = headerRow
+ ? (headerRow.children ?? []).filter(c => c.role === 'column_header' || c.role === 'cell').map(c => c.name ?? '')
+ : (el.children ?? []).filter(c => c.role === 'column_header').map(c => c.name ?? '')
+
+ if (headers.length === 0) {
+ // No headers — fall back to plain list of rows.
+ for (const row of rows) {
+ const cells = (row.children ?? []).filter(c => c.role === 'cell')
+ const line = cells.map(c => escapeBody(c.value ?? c.name ?? '')).join(' | ')
+ if (line) lines.push(`- ${line}`)
+ }
+ lines.push('')
+ return
+ }
+
+ lines.push(`| ${headers.map(escapeBody).join(' | ')} |`)
+ lines.push(`| ${headers.map(() => '---').join(' | ')} |`)
+
+ for (const row of rows) {
+ if (row === headerRow) continue
+ const cells = (row.children ?? []).filter(c => c.role === 'cell')
+ const padded = headers.map((_, i) => escapeBody(cells[i]?.value ?? cells[i]?.name ?? ''))
+ lines.push(`| ${padded.join(' | ')} |`)
+ }
+ lines.push('')
+}
diff --git a/src/desktop/desktop-converter.ts b/src/desktop/desktop-converter.ts
new file mode 100644
index 0000000..7793bae
--- /dev/null
+++ b/src/desktop/desktop-converter.ts
@@ -0,0 +1,183 @@
+/**
+ * `convertDesktop()` — capture a native window via an accessibility-tree
+ * backend and emit an AgentMark snapshot with `kind: 'desktop'`.
+ *
+ * Mirrors `convertAudio()` and `convertVideo()`: the converter is
+ * pure-Node and OS-agnostic; OS-specific work is hidden behind the
+ * `DesktopCaptureBackend` interface.
+ *
+ * Output body grammar (see `body-builder.ts` for full detail):
+ *
+ * [WINDOW:w_excel]
+ *
+ * # Microsoft Excel — Book1
+ *
+ * ## Worksheet
+ *
+ * [INPUT:act_cell_a1]
+ *
+ * [ACTION:act_save]
+ *
+ * Interactive controls produce entries in the snapshot's `actions` map;
+ * the runtime invokes the backend's `execute()` with the matching
+ * `element_id` (stored as `target_id` on each action so the renderer
+ * can resolve it).
+ */
+
+import {
+ AGENTMARK_VERSION,
+ type ConversionResult,
+ type DesktopMeta,
+ type Snapshot,
+} from '../types'
+import { serializeSnapshot } from '../serializers/yaml-frontmatter'
+import { InMemoryActionBinding } from '../binding/action-binding'
+import { noopLogger, type Logger } from '../observability/logger'
+import { SnapshotError } from '../errors'
+import { buildDesktopBody } from './body-builder'
+import type {
+ DesktopCaptureBackend,
+ DesktopCapture,
+ DesktopTarget,
+} from './types'
+
+export interface ConvertDesktopOptions {
+ /** Accessibility-tree backend (Windows UIA, macOS AXAPI, vision fallback, fixture, etc.). */
+ backend: DesktopCaptureBackend
+
+ /** Which window to capture. Defaults to focused window if omitted. */
+ target?: DesktopTarget
+
+ /** Override the snapshot title. Default: the captured window title. */
+ title?: string
+
+ /** Override the snapshot URL — use this when the producer wants to
+ * identify the surface canonically (e.g. `desktop://hostname/app/process_id`).
+ * Default: a synthesised `desktop:///` URI. */
+ url?: string
+
+ /** Max tree depth to traverse. Default: 12. */
+ maxDepth?: number
+
+ /** Include hidden / off-screen elements. Default: false. */
+ includeHidden?: boolean
+
+ /** Per-capture timeout (ms). Default: 5000. */
+ timeoutMs?: number
+
+ /** TTL for `expires_at` (ms). Desktop UI is highly dynamic, so the
+ * default is short — 15 seconds. Producers should re-capture before
+ * acting on a stale snapshot. */
+ ttlMs?: number
+
+ /** Structured logger. */
+ logger?: Logger
+
+ /** Vendor extensions (`x-` prefixed fields) attached to the snapshot. */
+ vendorExtensions?: Record
+}
+
+export async function convertDesktop(options: ConvertDesktopOptions): Promise {
+ const logger = options.logger ?? noopLogger
+ const ttlMs = options.ttlMs ?? 15_000
+
+ logger.debug('snapshot.capture.start', {
+ source: options.target?.window_title ?? options.target?.process_name ?? '',
+ kind: 'desktop',
+ backend: options.backend.name,
+ })
+
+ let capture: DesktopCapture
+ try {
+ capture = await options.backend.capture({
+ target: options.target,
+ maxDepth: options.maxDepth ?? 12,
+ includeHidden: options.includeHidden ?? false,
+ timeoutMs: options.timeoutMs ?? 5000,
+ })
+ } catch (err) {
+ logger.error('snapshot.failed', { error: (err as Error).message })
+ if (err instanceof SnapshotError) throw err
+ throw new SnapshotError(
+ `Desktop capture failed (${options.backend.name}): ${(err as Error).message}`,
+ err as Error,
+ )
+ }
+
+ const captured_at = new Date().toISOString()
+ const expires_at = new Date(Date.now() + ttlMs).toISOString()
+
+ const { body, actions, element_ids, has_interactive } = buildDesktopBody(capture)
+
+ // Stash each action's underlying desktop element_id in the
+ // ActionBinding. The runtime later resolves binding.get(actionId)
+ // to the opaque element_id and passes it to backend.execute().
+ const binding = new InMemoryActionBinding()
+ for (const [actionId, elementId] of Object.entries(element_ids)) {
+ binding.set(actionId, elementId)
+ }
+
+ const desktopMeta: DesktopMeta = stripUndefined({
+ platform: capture.platform,
+ process_name: capture.process_name,
+ process_id: capture.process_id,
+ window_class: capture.window_class,
+ focused_element_id: capture.focused_element_id,
+ a11y_backend: options.backend.name,
+ tree_depth: capture.tree_depth,
+ element_count: capture.element_count,
+ })
+
+ const snapshot: Snapshot = {
+ agentmark: AGENTMARK_VERSION,
+ kind: 'desktop',
+ url: options.url ?? synthesiseDesktopUrl(capture),
+ title: options.title ?? capture.window_title,
+ captured_at,
+ expires_at,
+ source: 'rendered',
+ desktop_meta: desktopMeta,
+ actions: has_interactive ? actions : undefined,
+ capabilities: {
+ preview_media: false,
+ expand_disclosures: true,
+ paginate: false,
+ scroll: true,
+ keyboard: true,
+ drag: false,
+ ocr: false,
+ vision: false,
+ },
+ body,
+ }
+
+ if (options.vendorExtensions) {
+ for (const [k, v] of Object.entries(options.vendorExtensions)) {
+ if (k.startsWith('x-')) (snapshot as unknown as Record)[k] = v
+ }
+ }
+
+ const text = serializeSnapshot(snapshot)
+ logger.info('snapshot.captured', {
+ source: snapshot.url,
+ kind: 'desktop',
+ backend: options.backend.name,
+ elements: capture.element_count,
+ actions: Object.keys(actions).length,
+ bytes: text.length,
+ })
+
+ return { agentmark: text, binding }
+}
+
+function synthesiseDesktopUrl(capture: DesktopCapture): string {
+ const host = capture.process_name?.toLowerCase().replace(/[^a-z0-9._-]/g, '_') ?? 'unknown'
+ const path = capture.window_id ?? String(capture.process_id ?? 'window')
+ return `desktop://${capture.platform}/${host}/${path}`
+}
+
+function stripUndefined(obj: T): T {
+ const out: Record = {}
+ for (const [k, v] of Object.entries(obj)) if (v !== undefined) out[k] = v
+ return out as T
+}
diff --git a/src/desktop/fixture-backend.ts b/src/desktop/fixture-backend.ts
new file mode 100644
index 0000000..b27d794
--- /dev/null
+++ b/src/desktop/fixture-backend.ts
@@ -0,0 +1,269 @@
+/**
+ * `FixtureBackend` — an in-memory `DesktopCaptureBackend` that returns
+ * pre-baked accessibility trees. Useful for:
+ *
+ * - Local development on machines without the OS bridge installed
+ * (e.g. running the MCP server on macOS before the AXAPI bridge
+ * exists, or on a CI runner with no GUI).
+ * - Tests for the converter and MCP server.
+ * - Demo flows that don't need real applications.
+ *
+ * Ships one preset by default — an Excel-like spreadsheet window —
+ * mirroring the shape a real Windows UIA backend would produce. Callers
+ * can supply their own `presets` map for custom scenarios.
+ */
+
+import type {
+ DesktopCapture,
+ DesktopCaptureBackend,
+ CaptureDesktopOptions,
+ DesktopElement,
+ ExecuteDesktopOptions,
+ ExecuteDesktopResult,
+} from './types'
+
+export interface FixtureBackendOptions {
+ /** Preset name → DesktopCapture. Defaults to a built-in Excel preset. */
+ presets?: Record
+ /** Default preset to return when `target` is omitted or doesn't match. */
+ defaultPreset?: string
+ /** Optional latency to simulate the bridge round-trip (ms). */
+ latencyMs?: number
+}
+
+export class FixtureBackend implements DesktopCaptureBackend {
+ readonly name = 'fixture'
+ private readonly presets: Record
+ private readonly defaultPreset: string
+ private readonly latencyMs: number
+
+ /** Captured executions — handy for tests to assert what was driven. */
+ public readonly executed: Array = []
+
+ /** Mutable map of element_id → new value, applied to subsequent
+ * capture() results so a test can roundtrip type → re-capture →
+ * observe the typed text. */
+ public readonly elementValues = new Map()
+
+ constructor(opts: FixtureBackendOptions = {}) {
+ this.presets = { ...DEFAULT_PRESETS, ...(opts.presets ?? {}) }
+ this.defaultPreset = opts.defaultPreset ?? 'excel_blank'
+ this.latencyMs = opts.latencyMs ?? 0
+ }
+
+ async capture(opts: CaptureDesktopOptions = {}): Promise {
+ if (this.latencyMs) await delay(this.latencyMs)
+
+ const key = pickPresetKey(opts, this.presets, this.defaultPreset)
+ const base = this.presets[key]
+ if (!base) {
+ throw new Error(`FixtureBackend: no preset named "${key}"`)
+ }
+
+ // Clone so mutations don't leak across calls, then layer in any
+ // values the test has typed.
+ const cloned = structuredClone(base) as DesktopCapture
+ if (this.elementValues.size > 0) {
+ applyValues(cloned.root, this.elementValues)
+ }
+ return cloned
+ }
+
+ async execute(opts: ExecuteDesktopOptions): Promise {
+ if (this.latencyMs) await delay(this.latencyMs)
+ this.executed.push(opts)
+
+ switch (opts.action.type) {
+ case 'type':
+ this.elementValues.set(opts.action.element_id, opts.action.text)
+ return { ok: true, new_value: opts.action.text }
+ case 'check':
+ this.elementValues.set(opts.action.element_id, opts.action.checked ? 'true' : 'false')
+ return { ok: true, new_value: String(opts.action.checked) }
+ case 'select':
+ this.elementValues.set(opts.action.element_id, opts.action.value)
+ return { ok: true, new_value: opts.action.value }
+ default:
+ return { ok: true }
+ }
+ }
+}
+
+function delay(ms: number): Promise {
+ return new Promise(resolve => setTimeout(resolve, ms))
+}
+
+function pickPresetKey(
+ opts: CaptureDesktopOptions,
+ presets: Record,
+ fallback: string,
+): string {
+ const t = opts.target
+ if (!t) return fallback
+ if (t.window_id && presets[t.window_id]) return t.window_id
+ if (t.process_name) {
+ const k = t.process_name.toLowerCase().replace(/\..*$/, '')
+ if (presets[k]) return k
+ }
+ if (t.window_title) {
+ const k = t.window_title.toLowerCase().replace(/[^a-z0-9_]/g, '_')
+ if (presets[k]) return k
+ }
+ return fallback
+}
+
+function applyValues(el: DesktopElement, values: Map): void {
+ const v = values.get(el.id)
+ if (v !== undefined) el.value = v
+ for (const child of el.children ?? []) applyValues(child, values)
+}
+
+// ──────────────────────────────────────────────────────────────────────────
+// Built-in presets
+// ──────────────────────────────────────────────────────────────────────────
+
+const EXCEL_BLANK: DesktopCapture = {
+ platform: 'windows',
+ process_name: 'EXCEL.EXE',
+ process_id: 12345,
+ window_title: 'Microsoft Excel - Book1',
+ window_class: 'XLMAIN',
+ window_id: 'excel_blank',
+ focused_element_id: 'cell_A1',
+ tree_depth: 4,
+ element_count: 9,
+ root: {
+ id: 'root',
+ role: 'window',
+ name: 'Microsoft Excel - Book1',
+ enabled: true,
+ children: [
+ {
+ id: 'ribbon',
+ role: 'toolbar',
+ name: 'Ribbon',
+ enabled: true,
+ children: [
+ { id: 'btn_save', role: 'button', name: 'Save', enabled: true },
+ { id: 'btn_undo', role: 'button', name: 'Undo', enabled: false },
+ { id: 'btn_redo', role: 'button', name: 'Redo', enabled: false },
+ ],
+ },
+ {
+ id: 'sheet_sheet1',
+ role: 'pane',
+ name: 'Sheet1',
+ enabled: true,
+ children: [
+ {
+ id: 'grid_main',
+ role: 'table',
+ name: 'Sheet1 grid',
+ enabled: true,
+ children: [
+ {
+ id: 'row_header',
+ role: 'row',
+ children: [
+ { id: 'col_A', role: 'column_header', name: 'A' },
+ { id: 'col_B', role: 'column_header', name: 'B' },
+ { id: 'col_C', role: 'column_header', name: 'C' },
+ ],
+ },
+ {
+ id: 'row_1',
+ role: 'row',
+ children: [
+ { id: 'cell_A1', role: 'cell', name: 'A1', value: '', enabled: true },
+ { id: 'cell_B1', role: 'cell', name: 'B1', value: '', enabled: true },
+ { id: 'cell_C1', role: 'cell', name: 'C1', value: '', enabled: true },
+ ],
+ },
+ ],
+ },
+ ],
+ },
+ {
+ id: 'status_bar',
+ role: 'status_bar',
+ name: 'Ready',
+ value: 'Ready',
+ },
+ ],
+ },
+}
+
+const NOWCERTS_CUSTOMER: DesktopCapture = {
+ platform: 'windows',
+ process_name: 'NowCerts.exe',
+ process_id: 22120,
+ window_title: 'NowCerts - Customer Detail - Acme Corp',
+ window_class: 'WindowsForms10.Window.8.app.0.378734a',
+ window_id: 'nowcerts_customer',
+ focused_element_id: 'in_company',
+ tree_depth: 5,
+ element_count: 12,
+ root: {
+ id: 'root',
+ role: 'window',
+ name: 'NowCerts - Customer Detail',
+ enabled: true,
+ children: [
+ {
+ id: 'panel_company',
+ role: 'group',
+ name: 'Company Information',
+ enabled: true,
+ children: [
+ {
+ id: 'in_company',
+ role: 'text_input',
+ name: 'Company Name',
+ value: 'Acme Corp',
+ enabled: true,
+ },
+ {
+ id: 'in_email',
+ role: 'text_input',
+ name: 'Primary Email',
+ value: 'contact@acme.com',
+ enabled: true,
+ },
+ {
+ id: 'in_phone',
+ role: 'text_input',
+ name: 'Phone',
+ value: '(555) 123-4567',
+ enabled: true,
+ },
+ {
+ id: 'cb_active',
+ role: 'check_box',
+ name: 'Active Customer',
+ selected: true,
+ enabled: true,
+ aria: { checked: true },
+ },
+ ],
+ },
+ {
+ id: 'panel_actions',
+ role: 'toolbar',
+ name: 'Actions',
+ enabled: true,
+ children: [
+ { id: 'btn_save', role: 'button', name: 'Save', enabled: true },
+ { id: 'btn_cancel', role: 'button', name: 'Cancel', enabled: true },
+ { id: 'btn_new_policy', role: 'button', name: 'New Policy', enabled: true },
+ ],
+ },
+ ],
+ },
+}
+
+const DEFAULT_PRESETS: Record = {
+ excel_blank: EXCEL_BLANK,
+ excel: EXCEL_BLANK,
+ nowcerts_customer: NOWCERTS_CUSTOMER,
+ nowcerts: NOWCERTS_CUSTOMER,
+}
diff --git a/src/desktop/index.ts b/src/desktop/index.ts
new file mode 100644
index 0000000..a734af4
--- /dev/null
+++ b/src/desktop/index.ts
@@ -0,0 +1,25 @@
+/**
+ * Desktop support — convertDesktop() + accessibility-tree capture backends.
+ */
+
+export { convertDesktop } from './desktop-converter'
+export type { ConvertDesktopOptions } from './desktop-converter'
+
+export { FixtureBackend } from './fixture-backend'
+export type { FixtureBackendOptions } from './fixture-backend'
+
+export type {
+ DesktopCaptureBackend,
+ CaptureDesktopOptions,
+ DesktopCapture,
+ DesktopElement,
+ DesktopRole,
+ DesktopTarget,
+ ExecuteDesktopOptions,
+ ExecuteDesktopAction,
+ ExecuteDesktopResult,
+ KeyModifier,
+} from './types'
+
+export { buildDesktopBody } from './body-builder'
+export type { BuildDesktopBodyResult } from './body-builder'
diff --git a/src/desktop/types.ts b/src/desktop/types.ts
new file mode 100644
index 0000000..54e897c
--- /dev/null
+++ b/src/desktop/types.ts
@@ -0,0 +1,211 @@
+/**
+ * Desktop support — types for OS-accessibility capture backends and the
+ * structured tree `convertDesktop()` consumes.
+ *
+ * Backends implement `DesktopCaptureBackend` and are typically OS-bound:
+ *
+ * - Windows: a .NET sidecar process exposing UIA via FlaUI
+ * - macOS: a Swift/ObjC sidecar exposing AXAPI
+ * - Linux: a sidecar over AT-SPI / D-Bus
+ * - Vision: a fallback that captures a screenshot and parses it
+ * (e.g. OmniParser, GUI-Actor) when no accessibility tree
+ * is available (Electron canvas, Citrix sessions, etc.)
+ *
+ * The converter is OS-agnostic: it consumes a `DesktopCapture` tree and
+ * produces an AgentMark Snapshot with `kind: 'desktop'`. Bridges live in
+ * separate processes/repos; the Node-side just talks to whatever bridge
+ * implements this interface.
+ */
+
+export interface DesktopCaptureBackend {
+ /** Stable backend identifier — populated into `desktop_meta.a11y_backend`.
+ * Convention: `windows_uia`, `macos_axapi`, `linux_atspi`, `vision_fallback`,
+ * or `fixture` for tests. */
+ readonly name: string
+
+ /** Capture the current state of a target window/application. */
+ capture(opts: CaptureDesktopOptions): Promise
+
+ /** Execute an action against a previously-captured element. The element
+ * is identified by `element_id` which comes from the capture tree. */
+ execute(opts: ExecuteDesktopOptions): Promise
+
+ /** Optional teardown — release native handles, close sidecar process. */
+ close?(): Promise
+}
+
+/** Identifies a target window. Backends accept any combination they can
+ * resolve; if nothing is provided the currently focused window is used. */
+export interface DesktopTarget {
+ process_name?: string
+ process_id?: number
+ window_title?: string
+ /** Opaque OS handle — e.g. Windows HWND as a stringified pointer, macOS
+ * AXUIElementRef pointer encoded as a base64 string. Returned in
+ * `DesktopCapture.window_id` so callers can re-target precisely. */
+ window_id?: string
+}
+
+export interface CaptureDesktopOptions {
+ /** Which window/app to capture. Defaults to focused window if omitted. */
+ target?: DesktopTarget
+ /** Maximum tree depth to traverse. Default: backend-defined (typically 12). */
+ maxDepth?: number
+ /** Whether to include elements that are off-screen or invisible. Default: false. */
+ includeHidden?: boolean
+ /** Per-request timeout (ms). Default: 5000. */
+ timeoutMs?: number
+}
+
+export interface DesktopCapture {
+ platform: 'windows' | 'macos' | 'linux'
+ process_name?: string
+ process_id?: number
+ window_title: string
+ window_class?: string
+ /** Backend-defined opaque handle for re-targeting this exact window. */
+ window_id?: string
+ /** Stable accessibility ID of the currently focused element, when one
+ * is focused inside the captured window. */
+ focused_element_id?: string
+ /** Maximum depth the backend traversed (may be < requested if a leaf
+ * was reached first). */
+ tree_depth: number
+ /** Total elements captured (interactive + static). */
+ element_count: number
+ /** Root of the accessibility tree. */
+ root: DesktopElement
+}
+
+/**
+ * One node in the accessibility tree. Roles map to the underlying OS
+ * concept (UIA ControlType / AXRole). The converter inspects `role` to
+ * decide whether the element becomes an `ActionDefinition` (interactive)
+ * or a static body block (label/text/group).
+ */
+export interface DesktopElement {
+ /** Stable accessibility identifier. Windows: UIA AutomationId; macOS:
+ * AXIdentifier; Linux: AT-SPI accessible-id. Falls back to a
+ * backend-generated stable hash when the platform omits the ID. */
+ id: string
+
+ /** Control role — backends normalise to a small vocabulary the
+ * converter understands. */
+ role: DesktopRole
+
+ /** Accessible name / display label. */
+ name?: string
+
+ /** Current value (for inputs, sliders, combo selections, etc.). */
+ value?: string
+
+ /** Placeholder / help text when the element shows one. */
+ placeholder?: string
+
+ /** Whether the element is enabled. Disabled elements are still
+ * captured but won't generate executable actions. */
+ enabled?: boolean
+
+ /** Selection state for checkboxes, radios, list items, tabs. */
+ selected?: boolean
+
+ /** Read-only text fields, etc. */
+ read_only?: boolean
+
+ /** ExpandCollapsePattern state for combo boxes, tree items, menus. */
+ expanded?: boolean
+
+ /** ARIA-equivalent properties when supplied by the backend. */
+ aria?: {
+ pressed?: boolean
+ checked?: boolean | 'mixed'
+ required?: boolean
+ invalid?: boolean
+ }
+
+ /** Bounds in screen coordinates. Optional — used by vision-fallback
+ * backends or when the agent wants pixel-targeted clicks. */
+ bounds?: { x: number; y: number; width: number; height: number }
+
+ /** Nested children. Backends pre-order traversal; converter renders
+ * in document order. */
+ children?: DesktopElement[]
+}
+
+/**
+ * Normalised role vocabulary. Backends translate UIA/AXAPI roles to
+ * these. Unknowns become `'other'` and render as static body text.
+ */
+export type DesktopRole =
+ | 'window'
+ | 'pane'
+ | 'group'
+ | 'toolbar'
+ | 'menu'
+ | 'menu_item'
+ | 'tab_list'
+ | 'tab'
+ | 'tree'
+ | 'tree_item'
+ | 'list'
+ | 'list_item'
+ | 'table'
+ | 'row'
+ | 'cell'
+ | 'column_header'
+ | 'button'
+ | 'split_button'
+ | 'text_input'
+ | 'password_input'
+ | 'text_area'
+ | 'check_box'
+ | 'radio_button'
+ | 'combo_box'
+ | 'list_box'
+ | 'slider'
+ | 'progress_bar'
+ | 'link'
+ | 'label'
+ | 'static_text'
+ | 'image'
+ | 'separator'
+ | 'status_bar'
+ | 'scroll_bar'
+ | 'dialog'
+ | 'tooltip'
+ | 'other'
+
+export interface ExecuteDesktopOptions {
+ /** Which window the element lives in. Optional if the backend can
+ * resolve the element_id globally. */
+ target?: DesktopTarget
+ /** What to do. */
+ action: ExecuteDesktopAction
+ /** Per-request timeout (ms). Default: 5000. */
+ timeoutMs?: number
+}
+
+/** All supported actions. The backend translates these to UIA patterns
+ * (InvokePattern, ValuePattern, TogglePattern, etc.) or AXAPI equivalents. */
+export type ExecuteDesktopAction =
+ | { type: 'click'; element_id: string }
+ | { type: 'type'; element_id: string; text: string; clear_first?: boolean }
+ | { type: 'select'; element_id: string; value: string }
+ | { type: 'check'; element_id: string; checked: boolean }
+ | { type: 'expand'; element_id: string; expanded: boolean }
+ | { type: 'focus'; element_id: string }
+ | { type: 'scroll_to'; element_id: string }
+ | { type: 'key'; element_id?: string; key: string; modifiers?: ReadonlyArray }
+
+export type KeyModifier = 'ctrl' | 'alt' | 'shift' | 'meta' | 'win'
+
+export interface ExecuteDesktopResult {
+ ok: boolean
+ /** Backend-supplied detail when ok is false (e.g. 'element disabled',
+ * 'element no longer in tree'). */
+ message?: string
+ /** Snapshot of the element AFTER the action when the backend can
+ * cheaply re-query it. Lets the agent confirm state without a full
+ * re-capture. */
+ new_value?: string
+}
diff --git a/src/index.ts b/src/index.ts
index 91ca6ad..68e8b72 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -211,3 +211,22 @@ export type {
ExtractFramesOptions,
ExtractedFrame,
} from './video'
+
+// ── v0.4 spec / v0.12 lib: Desktop support (kind: 'desktop') ─────────────
+
+export { convertDesktop, FixtureBackend, buildDesktopBody } from './desktop'
+export type {
+ ConvertDesktopOptions,
+ FixtureBackendOptions,
+ BuildDesktopBodyResult,
+ DesktopCaptureBackend,
+ CaptureDesktopOptions,
+ DesktopCapture,
+ DesktopElement,
+ DesktopRole,
+ DesktopTarget,
+ ExecuteDesktopOptions,
+ ExecuteDesktopAction,
+ ExecuteDesktopResult,
+ KeyModifier,
+} from './desktop'
diff --git a/src/mcp/dispatcher.ts b/src/mcp/dispatcher.ts
index b1e8e7c..eee402f 100644
--- a/src/mcp/dispatcher.ts
+++ b/src/mcp/dispatcher.ts
@@ -12,21 +12,29 @@ import * as path from 'node:path'
import { pathToFileURL } from 'node:url'
import {
createBrowser,
+ convertDesktop,
+ FixtureBackend,
openPdfDocument,
isAgentMarkError,
+ parseSnapshot,
PopplerRenderBackend,
TesseractOcrBackend,
type Browser,
+ type DesktopCaptureBackend,
+ type DesktopTarget,
+ type ExecuteDesktopAction,
+ type KeyModifier,
type Page,
type PdfDocument,
type OcrPipelineOptions,
} from '../index'
-import { generateSessionId, type BrowserSession, type PdfSession } from './types'
+import { generateSessionId, type BrowserSession, type DesktopSession, type PdfSession } from './types'
export interface DispatcherState {
browsers: Map
pages: Map
pdfs: Map
+ desktops: Map
}
export function createDispatcherState(): DispatcherState {
@@ -34,6 +42,7 @@ export function createDispatcherState(): DispatcherState {
browsers: new Map(),
pages: new Map(),
pdfs: new Map(),
+ desktops: new Map(),
}
}
@@ -83,6 +92,16 @@ export async function dispatch(
case 'agentmark_pdf_reset':
return await pdfReset(state, args)
+ // ── Desktop ──────────────────────────────────────────────────
+ case 'agentmark_desktop_open':
+ return await openDesktop(state, args)
+ case 'agentmark_desktop_close':
+ return await closeDesktop(state, args)
+ case 'agentmark_desktop_snapshot':
+ return await desktopSnapshot(state, args)
+ case 'agentmark_desktop_execute':
+ return await desktopExecute(state, args)
+
// ── Meta ─────────────────────────────────────────────────────
case 'agentmark_list_sessions':
return listSessions(state)
@@ -312,6 +331,194 @@ async function pdfReset(state: DispatcherState, args: Record):
return { text: `PDF ${id} pending values cleared.` }
}
+// ──────────────────────────────────────────────────────────────────────────
+// Desktop tool handlers
+// ──────────────────────────────────────────────────────────────────────────
+
+async function openDesktop(state: DispatcherState, args: Record): Promise {
+ const requested = typeof args.backend === 'string' ? args.backend : 'fixture'
+ let backend: DesktopCaptureBackend
+ switch (requested) {
+ case 'fixture':
+ backend = new FixtureBackend()
+ break
+ case 'windows_uia':
+ case 'macos_axapi':
+ return {
+ text:
+ `Backend "${requested}" is not yet bundled with this build of agentmark. `
+ + 'Run agentmark_desktop_open with backend="fixture" to use the in-memory '
+ + 'preset trees. Real OS bridges land in subsequent releases.',
+ isError: true,
+ }
+ default:
+ return { text: `Unknown desktop backend: ${requested}`, isError: true }
+ }
+
+ const id = generateSessionId('dt')
+ state.desktops.set(id, {
+ id,
+ backend,
+ createdAt: new Date(),
+ })
+ return {
+ text: JSON.stringify({ desktop_id: id, backend: requested }, null, 2),
+ }
+}
+
+async function closeDesktop(state: DispatcherState, args: Record): Promise {
+ const id = requireString(args, 'desktop_id')
+ const session = state.desktops.get(id)
+ if (!session) return { text: `Unknown desktop_id: ${id}`, isError: true }
+ await session.backend.close?.()
+ state.desktops.delete(id)
+ return { text: `Desktop session ${id} closed.` }
+}
+
+async function desktopSnapshot(state: DispatcherState, args: Record): Promise {
+ const id = requireString(args, 'desktop_id')
+ const session = requireDesktop(state, id)
+
+ const target = parseTarget(args.target)
+ const maxDepth = typeof args.max_depth === 'number' ? args.max_depth : undefined
+ const includeHidden = args.include_hidden === true
+ const timeoutMs = typeof args.timeout_ms === 'number' ? args.timeout_ms : undefined
+
+ const { agentmark, binding } = await convertDesktop({
+ backend: session.backend,
+ target,
+ maxDepth,
+ includeHidden,
+ timeoutMs,
+ })
+
+ // Cache binding + action types so subsequent _execute can resolve.
+ session.lastTarget = target
+ session.lastBinding = binding
+ const snap = parseSnapshot(agentmark)
+ session.lastActionTypes = new Map(
+ Object.entries(snap.actions ?? {}).map(([k, def]) => [k, def.type]),
+ )
+
+ return { text: agentmark }
+}
+
+async function desktopExecute(state: DispatcherState, args: Record): Promise {
+ const id = requireString(args, 'desktop_id')
+ const actionId = requireString(args, 'action_id')
+ const session = requireDesktop(state, id)
+
+ if (!session.lastBinding || !session.lastActionTypes) {
+ return {
+ text:
+ `No cached snapshot for desktop_id ${id}. Call `
+ + `agentmark_desktop_snapshot first so the action_id can be resolved.`,
+ isError: true,
+ }
+ }
+
+ const elementId = session.lastBinding.get(actionId)
+ if (!elementId) {
+ return { text: `Unknown action_id: ${actionId}`, isError: true }
+ }
+
+ const actionType = session.lastActionTypes.get(actionId) ?? 'click'
+ const value = args.value
+ const modifiers = parseModifiers(args.modifiers)
+ const clearFirst = args.clear_first === true
+
+ const action = buildExecuteAction(actionType, elementId, value, modifiers, clearFirst)
+ const result = await session.backend.execute({
+ target: session.lastTarget,
+ action,
+ })
+
+ return {
+ text: JSON.stringify(
+ {
+ action_id: actionId,
+ action_type: actionType,
+ element_id: elementId,
+ ok: result.ok,
+ ...(result.message !== undefined ? { message: result.message } : {}),
+ ...(result.new_value !== undefined ? { new_value: result.new_value } : {}),
+ },
+ null,
+ 2,
+ ),
+ isError: !result.ok,
+ }
+}
+
+function parseTarget(input: unknown): DesktopTarget | undefined {
+ if (!input || typeof input !== 'object') return undefined
+ const t = input as Record
+ const out: DesktopTarget = {}
+ if (typeof t.process_name === 'string') out.process_name = t.process_name
+ if (typeof t.process_id === 'number') out.process_id = t.process_id
+ if (typeof t.window_title === 'string') out.window_title = t.window_title
+ if (typeof t.window_id === 'string') out.window_id = t.window_id
+ return Object.keys(out).length > 0 ? out : undefined
+}
+
+function parseModifiers(input: unknown): KeyModifier[] | undefined {
+ if (!Array.isArray(input)) return undefined
+ const allowed: ReadonlySet = new Set(['ctrl', 'alt', 'shift', 'meta', 'win'])
+ const out: KeyModifier[] = []
+ for (const m of input) {
+ if (typeof m === 'string' && allowed.has(m as KeyModifier)) out.push(m as KeyModifier)
+ }
+ return out.length > 0 ? out : undefined
+}
+
+function buildExecuteAction(
+ actionType: string,
+ elementId: string,
+ value: unknown,
+ modifiers: KeyModifier[] | undefined,
+ clearFirst: boolean,
+): ExecuteDesktopAction {
+ switch (actionType) {
+ case 'type':
+ return {
+ type: 'type',
+ element_id: elementId,
+ text: typeof value === 'string' ? value : String(value ?? ''),
+ clear_first: clearFirst,
+ }
+ case 'check':
+ return {
+ type: 'check',
+ element_id: elementId,
+ checked: value === true || value === 'true',
+ }
+ case 'select':
+ case 'multi_select':
+ return {
+ type: 'select',
+ element_id: elementId,
+ value: typeof value === 'string' ? value : String(value ?? ''),
+ }
+ case 'range':
+ return {
+ type: 'type',
+ element_id: elementId,
+ text: typeof value === 'number' ? String(value) : String(value ?? ''),
+ }
+ case 'key':
+ return {
+ type: 'key',
+ element_id: elementId,
+ key: typeof value === 'string' ? value : String(value ?? ''),
+ modifiers,
+ }
+ case 'scroll_to':
+ return { type: 'scroll_to', element_id: elementId }
+ default:
+ return { type: 'click', element_id: elementId }
+ }
+}
+
// ──────────────────────────────────────────────────────────────────────────
// Meta
// ──────────────────────────────────────────────────────────────────────────
@@ -331,6 +538,12 @@ function listSessions(state: DispatcherState): DispatchResult {
pending: s.document.pending.size,
created_at: s.createdAt.toISOString(),
})),
+ desktops: Array.from(state.desktops.values()).map((s) => ({
+ desktop_id: s.id,
+ backend: s.backend.name,
+ has_snapshot: s.lastBinding !== undefined,
+ created_at: s.createdAt.toISOString(),
+ })),
},
null,
2,
@@ -349,10 +562,14 @@ export async function disposeAll(state: DispatcherState): Promise {
for (const session of state.pdfs.values()) {
closers.push(session.document.close().catch(() => {}))
}
+ for (const session of state.desktops.values()) {
+ if (session.backend.close) closers.push(session.backend.close().catch(() => {}))
+ }
await Promise.allSettled(closers)
state.browsers.clear()
state.pages.clear()
state.pdfs.clear()
+ state.desktops.clear()
}
// ──────────────────────────────────────────────────────────────────────────
@@ -379,6 +596,12 @@ function requirePdf(state: DispatcherState, docId: string): PdfDocument {
return session.document
}
+function requireDesktop(state: DispatcherState, desktopId: string): DesktopSession {
+ const session = state.desktops.get(desktopId)
+ if (!session) throw new Error(`Unknown desktop_id: ${desktopId}`)
+ return session
+}
+
/**
* Load PDF bytes from either a file path OR a data URL. Data URLs are
* useful for clients that have the PDF in memory and don't want to write
diff --git a/src/mcp/tool-defs.ts b/src/mcp/tool-defs.ts
index 3770ab2..463b6f2 100644
--- a/src/mcp/tool-defs.ts
+++ b/src/mcp/tool-defs.ts
@@ -275,6 +275,149 @@ const PDF_TOOLS: McpToolDef[] = [
},
]
+// ──────────────────────────────────────────────────────────────────────────
+// Desktop application tools (v0.4)
+// ──────────────────────────────────────────────────────────────────────────
+
+const DESKTOP_TOOLS: McpToolDef[] = [
+ {
+ name: 'agentmark_desktop_open',
+ description:
+ 'Attach to a desktop accessibility-tree backend and return a '
+ + 'desktop_id. Backends:\n'
+ + ' - "fixture" (default): pre-baked Excel + NowCerts trees, '
+ + 'works on any OS — useful for testing without a real bridge '
+ + 'installed.\n'
+ + ' - "windows_uia": connects to the Windows FlaUI sidecar '
+ + 'process (requires the bridge running on the same machine).\n'
+ + ' - "macos_axapi": connects to the macOS AXAPI sidecar '
+ + '(requires the bridge + Accessibility permission granted).\n'
+ + '\nThe session lives for the duration of the MCP connection '
+ + 'unless explicitly closed.',
+ inputSchema: {
+ type: 'object',
+ properties: {
+ backend: {
+ type: 'string',
+ enum: ['fixture', 'windows_uia', 'macos_axapi'],
+ description: 'Which backend to use. Default: "fixture".',
+ },
+ bridge_url: {
+ type: 'string',
+ description:
+ 'Override the bridge WebSocket URL (for non-fixture '
+ + 'backends). Default: ws://127.0.0.1:9325/agentmark-bridge.',
+ },
+ },
+ },
+ },
+ {
+ name: 'agentmark_desktop_close',
+ description: 'Close a desktop session and release the bridge connection.',
+ inputSchema: {
+ type: 'object',
+ properties: {
+ desktop_id: { type: 'string' },
+ },
+ required: ['desktop_id'],
+ },
+ },
+ {
+ name: 'agentmark_desktop_snapshot',
+ description:
+ 'Capture an AgentMark snapshot of a desktop window. If `target` '
+ + 'is omitted, the currently focused window is captured. The '
+ + 'result is cached on the session so subsequent '
+ + 'agentmark_desktop_execute calls can resolve action IDs back '
+ + 'to native accessibility element IDs.',
+ inputSchema: {
+ type: 'object',
+ properties: {
+ desktop_id: { type: 'string' },
+ target: {
+ type: 'object',
+ description:
+ 'Which window to capture. Provide any combination; '
+ + 'the backend resolves whichever it can. Omit to '
+ + 'capture the focused window.',
+ properties: {
+ process_name: { type: 'string' },
+ process_id: { type: 'number' },
+ window_title: { type: 'string' },
+ window_id: {
+ type: 'string',
+ description:
+ 'Backend-defined opaque handle returned by a '
+ + 'previous snapshot. Most precise targeting.',
+ },
+ },
+ },
+ max_depth: {
+ type: 'number',
+ description:
+ 'Maximum accessibility-tree depth to traverse. '
+ + 'Default: 12.',
+ },
+ include_hidden: {
+ type: 'boolean',
+ description:
+ 'Include off-screen / invisible elements. Default: false.',
+ },
+ timeout_ms: {
+ type: 'number',
+ description: 'Per-capture timeout in milliseconds. Default: 5000.',
+ },
+ },
+ required: ['desktop_id'],
+ },
+ },
+ {
+ name: 'agentmark_desktop_execute',
+ description:
+ 'Execute an action against the most recent snapshot of a '
+ + 'desktop session. `action_id` is one of the keys from the '
+ + 'snapshot\'s `actions` map (e.g. `act_btn_save`). The MCP '
+ + 'server resolves it to the underlying element via the '
+ + 'ActionBinding captured at snapshot time.\n'
+ + '\nValue semantics by action type:\n'
+ + ' - click / focus / scroll_to: omit `value`\n'
+ + ' - type: string (text to enter)\n'
+ + ' - check: boolean (target state)\n'
+ + ' - select: string (option value or label)\n'
+ + ' - key: string (key name, e.g. "Enter", "F5") + optional '
+ + '`modifiers` array',
+ inputSchema: {
+ type: 'object',
+ properties: {
+ desktop_id: { type: 'string' },
+ action_id: { type: 'string' },
+ value: {
+ description:
+ 'Value for input-style actions. Type depends on the '
+ + 'action: string, boolean, etc.',
+ },
+ modifiers: {
+ type: 'array',
+ items: {
+ type: 'string',
+ enum: ['ctrl', 'alt', 'shift', 'meta', 'win'],
+ },
+ description:
+ 'Key modifiers for `key` actions (or holding modifiers '
+ + 'during a click).',
+ },
+ clear_first: {
+ type: 'boolean',
+ description:
+ 'For `type` actions, clear the existing value before '
+ + 'typing. Default: false.',
+ },
+ },
+ required: ['desktop_id', 'action_id'],
+ },
+ },
+]
+
// ──────────────────────────────────────────────────────────────────────────
// Session inspection
// ──────────────────────────────────────────────────────────────────────────
@@ -283,8 +426,9 @@ const META_TOOLS: McpToolDef[] = [
{
name: 'agentmark_list_sessions',
description:
- 'List all currently open browsers, pages, and PDF documents '
- + 'with their IDs. Useful for debugging or recovering a stuck session.',
+ 'List all currently open browsers, pages, PDF documents, and '
+ + 'desktop sessions with their IDs. Useful for debugging or '
+ + 'recovering a stuck session.',
inputSchema: {
type: 'object',
properties: {},
@@ -292,4 +436,4 @@ const META_TOOLS: McpToolDef[] = [
},
]
-export const ALL_TOOLS: McpToolDef[] = [...WEB_TOOLS, ...PDF_TOOLS, ...META_TOOLS]
+export const ALL_TOOLS: McpToolDef[] = [...WEB_TOOLS, ...PDF_TOOLS, ...DESKTOP_TOOLS, ...META_TOOLS]
diff --git a/src/mcp/types.ts b/src/mcp/types.ts
index a39b330..af18dc9 100644
--- a/src/mcp/types.ts
+++ b/src/mcp/types.ts
@@ -4,7 +4,15 @@
* MCP client can drive multiple parallel agents from one connection.
*/
-import type { Browser, Page, PdfDocument } from '../index'
+import type {
+ ActionBinding,
+ Browser,
+ DesktopCapture,
+ DesktopCaptureBackend,
+ DesktopTarget,
+ Page,
+ PdfDocument,
+} from '../index'
export interface BrowserSession {
id: string
@@ -19,12 +27,31 @@ export interface PdfSession {
createdAt: Date
}
+export interface DesktopSession {
+ id: string
+ backend: DesktopCaptureBackend
+ /** Last `target` passed to capture(); reused by execute() when the
+ * client doesn't re-specify it. */
+ lastTarget?: DesktopTarget
+ /** Result of the most recent capture — used so execute() knows which
+ * process to drive and which element_count to report. */
+ lastCapture?: DesktopCapture
+ /** Binding map from the most recent convertDesktop() call —
+ * resolves actionId → element_id for execute(). */
+ lastBinding?: ActionBinding
+ /** Action types keyed by actionId from the most recent snapshot.
+ * Used by execute() to translate the client's `value` argument
+ * into the right ExecuteDesktopAction variant. */
+ lastActionTypes?: Map
+ createdAt: Date
+}
+
/**
* Generates a short unique ID. Uses crypto.randomUUID() if available,
* otherwise a Math.random fallback. The IDs are opaque to clients —
* they're returned by `_open` tools and passed back on subsequent calls.
*/
-export function generateSessionId(prefix: 'br' | 'pdf' | 'pg'): string {
+export function generateSessionId(prefix: 'br' | 'pdf' | 'pg' | 'dt'): string {
const r =
typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function'
? crypto.randomUUID().replace(/-/g, '').slice(0, 12)
diff --git a/test/desktop/desktop-converter.test.ts b/test/desktop/desktop-converter.test.ts
new file mode 100644
index 0000000..a06e141
--- /dev/null
+++ b/test/desktop/desktop-converter.test.ts
@@ -0,0 +1,148 @@
+import { describe, it, expect } from 'vitest'
+import { convertDesktop } from '../../src/desktop/desktop-converter'
+import { FixtureBackend } from '../../src/desktop/fixture-backend'
+import { parseSnapshot } from '../../src/serializers/yaml-frontmatter'
+import { validateSnapshot } from '../../src/validators/schema-validator'
+
+describe('convertDesktop', () => {
+ it('produces a valid v0.4 desktop snapshot from the Excel fixture', async () => {
+ const backend = new FixtureBackend()
+ const { agentmark, binding } = await convertDesktop({ backend })
+
+ const snap = parseSnapshot(agentmark)
+ expect(snap.agentmark).toBe('0.4')
+ expect(snap.kind).toBe('desktop')
+ expect(snap.title).toBe('Microsoft Excel - Book1')
+ expect(snap.desktop_meta?.platform).toBe('windows')
+ expect(snap.desktop_meta?.process_name).toBe('EXCEL.EXE')
+ expect(snap.desktop_meta?.a11y_backend).toBe('fixture')
+ expect(snap.desktop_meta?.element_count).toBe(9)
+
+ // Body includes the window marker, the title, and at least one
+ // INPUT/ACTION reference resolvable in actions.
+ expect(snap.body).toContain('[WINDOW:w_excel_exe]')
+ expect(snap.body).toContain('# Microsoft Excel - Book1')
+ expect(snap.body).toContain('[ACTION:')
+
+ // Validates clean against the schema + cross-field invariants.
+ const result = validateSnapshot(snap)
+ expect(result.errors).toEqual([])
+ expect(result.valid).toBe(true)
+
+ // Binding has entries for every action.
+ for (const actionId of Object.keys(snap.actions ?? {})) {
+ expect(binding.get(actionId)).toBeDefined()
+ }
+ })
+
+ it('renders the NowCerts fixture with editable inputs as INPUT tags', async () => {
+ const backend = new FixtureBackend()
+ const { agentmark } = await convertDesktop({
+ backend,
+ target: { window_id: 'nowcerts_customer' },
+ })
+ const snap = parseSnapshot(agentmark)
+ expect(snap.title).toBe('NowCerts - Customer Detail - Acme Corp')
+ expect(snap.body).toContain('[INPUT:act_in_company]')
+ expect(snap.actions?.act_in_company?.type).toBe('type')
+ expect(snap.actions?.act_in_company?.value).toBe('Acme Corp')
+ expect(snap.actions?.act_in_company?.label).toBe('Company Name')
+ })
+
+ it('round-trips type → re-capture → observe the typed text', async () => {
+ const backend = new FixtureBackend()
+
+ // First capture — Company Name has its initial fixture value.
+ const initial = await convertDesktop({
+ backend,
+ target: { window_id: 'nowcerts_customer' },
+ })
+ const initialSnap = parseSnapshot(initial.agentmark)
+ expect(initialSnap.actions?.act_in_company?.value).toBe('Acme Corp')
+
+ // Resolve the binding to get the underlying element_id, then
+ // drive the backend's execute() the way the runtime will.
+ const elementId = initial.binding.get('act_in_company')
+ expect(elementId).toBe('in_company')
+
+ await backend.execute({
+ action: { type: 'type', element_id: elementId!, text: 'Beta Industries' },
+ })
+ expect(backend.executed).toHaveLength(1)
+
+ // Second capture — the typed value should be reflected.
+ const updated = await convertDesktop({
+ backend,
+ target: { window_id: 'nowcerts_customer' },
+ })
+ const updatedSnap = parseSnapshot(updated.agentmark)
+ expect(updatedSnap.actions?.act_in_company?.value).toBe('Beta Industries')
+ })
+
+ it('returns a synthesised desktop:// URL when none is provided', async () => {
+ const backend = new FixtureBackend()
+ const { agentmark } = await convertDesktop({ backend })
+ const snap = parseSnapshot(agentmark)
+ expect(snap.url).toMatch(/^desktop:\/\/windows\/excel\.exe\//)
+ })
+
+ it('respects an explicit URL override', async () => {
+ const backend = new FixtureBackend()
+ const { agentmark } = await convertDesktop({
+ backend,
+ url: 'desktop://my-host/excel/12345',
+ })
+ const snap = parseSnapshot(agentmark)
+ expect(snap.url).toBe('desktop://my-host/excel/12345')
+ })
+
+ it('reports the backend name in desktop_meta.a11y_backend', async () => {
+ const backend = new FixtureBackend()
+ const { agentmark } = await convertDesktop({ backend })
+ const snap = parseSnapshot(agentmark)
+ expect(snap.desktop_meta?.a11y_backend).toBe('fixture')
+ })
+
+ it('handles a check_box element with aria.checked state', async () => {
+ const backend = new FixtureBackend()
+ const { agentmark } = await convertDesktop({
+ backend,
+ target: { window_id: 'nowcerts_customer' },
+ })
+ const snap = parseSnapshot(agentmark)
+ const activeCheckbox = snap.actions?.act_cb_active
+ expect(activeCheckbox?.type).toBe('check')
+ expect(activeCheckbox?.aria?.checked).toBe(true)
+ })
+
+ it('drops the `actions` field when no interactive elements are captured', async () => {
+ const staticBackend: import('../../src/desktop/types').DesktopCaptureBackend = {
+ name: 'static-fixture',
+ async capture() {
+ return {
+ platform: 'macos',
+ process_name: 'Pages',
+ window_title: 'Untitled - Pages',
+ tree_depth: 2,
+ element_count: 2,
+ root: {
+ id: 'root',
+ role: 'window',
+ name: 'Pages',
+ children: [
+ { id: 'label_1', role: 'static_text', name: 'Empty document.' },
+ ],
+ },
+ }
+ },
+ async execute() {
+ return { ok: true }
+ },
+ }
+ const { agentmark } = await convertDesktop({ backend: staticBackend })
+ const snap = parseSnapshot(agentmark)
+ expect(snap.kind).toBe('desktop')
+ expect(snap.actions).toBeUndefined()
+ expect(snap.body).toContain('Empty document.')
+ })
+})
diff --git a/test/mcp/desktop-dispatcher.test.ts b/test/mcp/desktop-dispatcher.test.ts
new file mode 100644
index 0000000..e1cefc3
--- /dev/null
+++ b/test/mcp/desktop-dispatcher.test.ts
@@ -0,0 +1,170 @@
+/**
+ * MCP dispatcher tests for the desktop tools (v0.4).
+ *
+ * Drives `dispatch()` directly with the agentmark_desktop_* tool names.
+ * Uses the FixtureBackend, so these tests are deterministic and run on
+ * any OS (no real bridge required).
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import {
+ dispatch,
+ createDispatcherState,
+ disposeAll,
+ type DispatcherState,
+} from '../../src/mcp/dispatcher'
+import { ALL_TOOLS } from '../../src/mcp/tool-defs'
+import { parseSnapshot } from '../../src/serializers/yaml-frontmatter'
+
+let state: DispatcherState
+
+beforeEach(() => {
+ state = createDispatcherState()
+})
+
+afterEach(async () => {
+ await disposeAll(state)
+})
+
+describe('MCP — desktop tools', () => {
+ it('registers all four desktop tools in ALL_TOOLS', () => {
+ const names = ALL_TOOLS.map(t => t.name)
+ expect(names).toContain('agentmark_desktop_open')
+ expect(names).toContain('agentmark_desktop_close')
+ expect(names).toContain('agentmark_desktop_snapshot')
+ expect(names).toContain('agentmark_desktop_execute')
+ })
+
+ it('agentmark_desktop_open with no args defaults to the fixture backend', async () => {
+ const result = await dispatch(state, 'agentmark_desktop_open', {})
+ expect(result.isError).toBeFalsy()
+ const body = JSON.parse(result.text)
+ expect(body.desktop_id).toMatch(/^dt_/)
+ expect(body.backend).toBe('fixture')
+ })
+
+ it('agentmark_desktop_open with backend=fixture returns a desktop_id', async () => {
+ const result = await dispatch(state, 'agentmark_desktop_open', { backend: 'fixture' })
+ expect(result.isError).toBeFalsy()
+ const body = JSON.parse(result.text)
+ expect(state.desktops.has(body.desktop_id)).toBe(true)
+ })
+
+ it('agentmark_desktop_open rejects unknown backend names', async () => {
+ const result = await dispatch(state, 'agentmark_desktop_open', { backend: 'mystery' })
+ expect(result.isError).toBe(true)
+ expect(result.text).toContain('Unknown desktop backend')
+ })
+
+ it('agentmark_desktop_open with windows_uia returns "not yet bundled" (until the bridge ships)', async () => {
+ const result = await dispatch(state, 'agentmark_desktop_open', { backend: 'windows_uia' })
+ expect(result.isError).toBe(true)
+ expect(result.text).toContain('not yet bundled')
+ })
+
+ it('agentmark_desktop_snapshot returns a valid v0.4 desktop snapshot', async () => {
+ const open = await dispatch(state, 'agentmark_desktop_open', {})
+ const { desktop_id } = JSON.parse(open.text)
+
+ const result = await dispatch(state, 'agentmark_desktop_snapshot', { desktop_id })
+ expect(result.isError).toBeFalsy()
+
+ const snap = parseSnapshot(result.text)
+ expect(snap.agentmark).toBe('0.4')
+ expect(snap.kind).toBe('desktop')
+ expect(snap.desktop_meta?.platform).toBe('windows')
+ expect(snap.desktop_meta?.a11y_backend).toBe('fixture')
+ })
+
+ it('snapshot can target a specific preset via target.window_id', async () => {
+ const open = await dispatch(state, 'agentmark_desktop_open', {})
+ const { desktop_id } = JSON.parse(open.text)
+
+ const result = await dispatch(state, 'agentmark_desktop_snapshot', {
+ desktop_id,
+ target: { window_id: 'nowcerts_customer' },
+ })
+ const snap = parseSnapshot(result.text)
+ expect(snap.title).toContain('NowCerts')
+ expect(snap.actions?.act_in_company?.label).toBe('Company Name')
+ })
+
+ it('agentmark_desktop_execute drives a type action and the next snapshot reflects it', async () => {
+ const open = await dispatch(state, 'agentmark_desktop_open', {})
+ const { desktop_id } = JSON.parse(open.text)
+
+ // First snapshot — caches the binding.
+ await dispatch(state, 'agentmark_desktop_snapshot', {
+ desktop_id,
+ target: { window_id: 'nowcerts_customer' },
+ })
+
+ const exec = await dispatch(state, 'agentmark_desktop_execute', {
+ desktop_id,
+ action_id: 'act_in_company',
+ value: 'Beta Industries',
+ })
+ expect(exec.isError).toBeFalsy()
+ const body = JSON.parse(exec.text)
+ expect(body.action_type).toBe('type')
+ expect(body.element_id).toBe('in_company')
+ expect(body.ok).toBe(true)
+ expect(body.new_value).toBe('Beta Industries')
+
+ // Re-snapshot the same target → value should reflect the typed text.
+ const after = await dispatch(state, 'agentmark_desktop_snapshot', {
+ desktop_id,
+ target: { window_id: 'nowcerts_customer' },
+ })
+ const snap = parseSnapshot(after.text)
+ expect(snap.actions?.act_in_company?.value).toBe('Beta Industries')
+ })
+
+ it('agentmark_desktop_execute rejects unknown action IDs', async () => {
+ const open = await dispatch(state, 'agentmark_desktop_open', {})
+ const { desktop_id } = JSON.parse(open.text)
+
+ await dispatch(state, 'agentmark_desktop_snapshot', { desktop_id })
+
+ const exec = await dispatch(state, 'agentmark_desktop_execute', {
+ desktop_id,
+ action_id: 'act_bogus_id',
+ })
+ expect(exec.isError).toBe(true)
+ expect(exec.text).toContain('Unknown action_id')
+ })
+
+ it('agentmark_desktop_execute errors when no snapshot has been captured yet', async () => {
+ const open = await dispatch(state, 'agentmark_desktop_open', {})
+ const { desktop_id } = JSON.parse(open.text)
+
+ const exec = await dispatch(state, 'agentmark_desktop_execute', {
+ desktop_id,
+ action_id: 'act_anything',
+ })
+ expect(exec.isError).toBe(true)
+ expect(exec.text).toContain('No cached snapshot')
+ })
+
+ it('agentmark_desktop_close removes the session', async () => {
+ const open = await dispatch(state, 'agentmark_desktop_open', {})
+ const { desktop_id } = JSON.parse(open.text)
+ expect(state.desktops.has(desktop_id)).toBe(true)
+
+ const close = await dispatch(state, 'agentmark_desktop_close', { desktop_id })
+ expect(close.isError).toBeFalsy()
+ expect(state.desktops.has(desktop_id)).toBe(false)
+ })
+
+ it('agentmark_list_sessions includes desktop sessions', async () => {
+ const open = await dispatch(state, 'agentmark_desktop_open', {})
+ const { desktop_id } = JSON.parse(open.text)
+
+ const list = await dispatch(state, 'agentmark_list_sessions', {})
+ const body = JSON.parse(list.text)
+ expect(body.desktops).toHaveLength(1)
+ expect(body.desktops[0].desktop_id).toBe(desktop_id)
+ expect(body.desktops[0].backend).toBe('fixture')
+ expect(body.desktops[0].has_snapshot).toBe(false)
+ })
+})