From 9773e491da5a5ea0aa9af1e7dc05dab0cb314878 Mon Sep 17 00:00:00 2001 From: rrader26 Date: Mon, 11 May 2026 09:58:39 -0400 Subject: [PATCH 1/3] feat(desktop): Node producer + FixtureBackend for kind: 'desktop' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the spec-v0.4 producer side. Pure-Node and OS-agnostic — OS work is hidden behind the new DesktopCaptureBackend interface. Real OS bridges (Windows UIA via FlaUI, macOS AXAPI via Swift) ship in subsequent PRs as separate sidecar processes that implement this interface. New module: src/desktop/ - types.ts — DesktopCaptureBackend interface contract, DesktopCapture (the tree shape backends return), DesktopElement (one node), DesktopRole (normalised UIA/AXAPI role vocabulary), ExecuteDesktopAction (click | type | select | check | expand | focus | scroll_to | key) - body-builder.ts — walks the captured tree → emits markdown + Records. Headings for windows/panes/dialogs, lists for menus and tree controls, native tables for grids, [ACTION:]/[INPUT:] tags for interactive elements, [WINDOW:]/[ELEMENT:] for structural markers - desktop-converter.ts — convertDesktop() entry point. Backend agnostic. Populates ActionBinding with the original element_id so the runtime can resolve execute(actionId) → backend.execute({ element_id }) - fixture-backend.ts — in-memory DesktopCaptureBackend returning pre-baked Excel + NowCerts trees. Lets the MCP server work end-to-end on machines without the real OS bridge (e.g. testing on a Mac before the AXAPI bridge exists; CI runners with no GUI). Round-trippable — execute() type mutates a value map that the next capture() reflects. - index.ts — module exports Wired into the package root src/index.ts under the existing v0.* section headers, matching how PDF/audio/video are exported. Tests: test/desktop/desktop-converter.test.ts — 8 tests covering: - Valid v0.4 desktop snapshot from Excel fixture (validates clean against schema + cross-field invariants) - NowCerts fixture with editable inputs rendered as [INPUT:] tags - Round-trip: type → re-capture → observe the typed text - Synthesised desktop:// URL when none provided - Explicit URL override respected - Backend name reported in desktop_meta.a11y_backend - Check_box aria.checked state preserved - Static-only snapshot drops the `actions` field Full suite: 283 passed, 10 skipped, 0 failed. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/desktop/body-builder.ts | 311 +++++++++++++++++++++++++ src/desktop/desktop-converter.ts | 183 +++++++++++++++ src/desktop/fixture-backend.ts | 269 +++++++++++++++++++++ src/desktop/index.ts | 25 ++ src/desktop/types.ts | 211 +++++++++++++++++ src/index.ts | 19 ++ test/desktop/desktop-converter.test.ts | 148 ++++++++++++ 7 files changed, 1166 insertions(+) create mode 100644 src/desktop/body-builder.ts create mode 100644 src/desktop/desktop-converter.ts create mode 100644 src/desktop/fixture-backend.ts create mode 100644 src/desktop/index.ts create mode 100644 src/desktop/types.ts create mode 100644 test/desktop/desktop-converter.test.ts diff --git a/src/desktop/body-builder.ts b/src/desktop/body-builder.ts new file mode 100644 index 0000000..65c1c21 --- /dev/null +++ b/src/desktop/body-builder.ts @@ -0,0 +1,311 @@ +/** + * Walk a captured desktop accessibility tree and emit: + * + * 1. A markdown body that renders the UI as structured text agents + * can read. Headings for windows/panes/dialogs; lists for menus + * and tab lists; tables for grid controls; inline tags for + * interactive elements. + * + * 2. An `actions` map keyed by `act_` that the runtime later + * passes to the backend's `execute()` call. + * + * The tag conventions mirror the rest of AgentMark: + * + * - `[ACTION:act_save]` — clickable / triggers (button, link, menu_item) + * - `[INPUT:act_first_name]` — editable controls (text_input, combo_box, + * check_box, radio_button, slider, etc.) + * - `[ELEMENT:e_status]` — non-interactive references the agent might + * cite (status text, table cell IDs) + * - `[WINDOW:w_excel_1]` — window boundary marker for multi-window + * captures + * + * Interactive elements always land in the `actions` map. Static text + * usually just appears in body text — but when the producer wants to + * give the agent a stable reference, it emits `[ELEMENT:e_x]` and + * doesn't add to actions. + */ + +import type { ActionDefinition, ActionType } from '../types' +import type { DesktopCapture, DesktopElement, DesktopRole } from './types' + +export interface BuildDesktopBodyResult { + /** The markdown body. */ + body: string + /** Actions discovered while walking the tree, keyed by act_. */ + actions: Record + /** Mapping action ID → original desktop element_id. The runtime + * pushes this into the ActionBinding so that `execute(act_X)` + * translates to `backend.execute({ element_id: ... })`. */ + element_ids: Record + /** Whether the tree contained any interactive elements. */ + has_interactive: boolean +} + +const ESCAPE_REGEX = /\[(?=[A-Z])/g + +function escapeBody(text: string): string { + return text.replace(/\\/g, '\\\\').replace(ESCAPE_REGEX, '\\[') +} + +/** Roles whose elements become entries in the `actions` map. */ +const INTERACTIVE_ROLES: ReadonlySet = new Set([ + 'button', + 'split_button', + 'menu_item', + 'tab', + 'link', + 'text_input', + 'password_input', + 'text_area', + 'check_box', + 'radio_button', + 'combo_box', + 'list_box', + 'slider', + 'tree_item', + 'list_item', +]) + +/** Map normalised desktop role → AgentMark ActionType. */ +function actionTypeFor(role: DesktopRole): ActionType { + switch (role) { + case 'text_input': + case 'password_input': + case 'text_area': + return 'type' + case 'check_box': + return 'check' + case 'radio_button': + return 'check' + case 'combo_box': + case 'list_box': + return 'select' + case 'slider': + return 'range' + default: + return 'click' + } +} + +/** Tag used for an element — INPUT for editable controls, ACTION otherwise. */ +function tagKindFor(role: DesktopRole): 'INPUT' | 'ACTION' { + switch (role) { + case 'text_input': + case 'password_input': + case 'text_area': + case 'check_box': + case 'radio_button': + case 'combo_box': + case 'list_box': + case 'slider': + return 'INPUT' + default: + return 'ACTION' + } +} + +interface BuilderState { + actions: Record + element_ids: Record + seen: Set + has_interactive: boolean +} + +function uniqueActionId(state: BuilderState, raw: string): string { + // Sanitise to AgentMark action ID rules: lowercase, alnum + underscore, + // starts with a letter. + const base = `act_${raw.toLowerCase().replace(/[^a-z0-9_]/g, '_').replace(/^_+/, '').slice(0, 56) || 'el'}` + if (!state.seen.has(base)) { + state.seen.add(base) + return base + } + let n = 2 + while (state.seen.has(`${base}_${n}`)) n++ + const id = `${base}_${n}` + state.seen.add(id) + return id +} + +export function buildDesktopBody(capture: DesktopCapture): BuildDesktopBodyResult { + const state: BuilderState = { + actions: {}, + element_ids: {}, + seen: new Set(), + has_interactive: false, + } + + const lines: string[] = [] + // Top-of-body window marker so multi-window producers (or future + // multi-tab captures) can interleave consistently. + const winId = `w_${(capture.process_name ?? 'window').toLowerCase().replace(/[^a-z0-9_]/g, '_').slice(0, 24) || 'app'}` + lines.push(`[WINDOW:${winId}]`) + lines.push('') + lines.push(`# ${capture.window_title || '(untitled window)'}`) + lines.push('') + + renderElement(capture.root, lines, state, 0) + + // Trim trailing blank lines + while (lines.length && lines[lines.length - 1] === '') lines.pop() + + return { + body: lines.join('\n') + '\n', + actions: state.actions, + element_ids: state.element_ids, + has_interactive: state.has_interactive, + } +} + +function renderElement( + el: DesktopElement, + lines: string[], + state: BuilderState, + depth: number, +): void { + const role = el.role + + // Containers — render a heading or list marker, then recurse. + if (role === 'window' || role === 'pane' || role === 'dialog' || role === 'group') { + if (el.name) { + const level = Math.min(2 + depth, 6) + lines.push(`${'#'.repeat(level)} ${escapeBody(el.name)}`) + lines.push('') + } + renderChildren(el, lines, state, depth + 1) + return + } + + if (role === 'toolbar' || role === 'menu' || role === 'tab_list') { + if (el.name) { + lines.push(`**${escapeBody(el.name)}**`) + lines.push('') + } + renderChildren(el, lines, state, depth + 1) + return + } + + if (role === 'tree' || role === 'list') { + if (el.name) { + lines.push(`**${escapeBody(el.name)}**`) + lines.push('') + } + for (const child of el.children ?? []) { + const item = renderInteractiveAsTag(child, state) + if (item) { + lines.push(`- ${item}`) + } else if (child.name) { + lines.push(`- ${escapeBody(child.name)}`) + } else { + renderElement(child, lines, state, depth + 1) + } + } + lines.push('') + return + } + + if (role === 'table') { + renderTable(el, lines, state) + return + } + + // Interactive leaves — emit a tag inline, push action to map. + if (INTERACTIVE_ROLES.has(role)) { + const tag = renderInteractiveAsTag(el, state) + if (tag) { + lines.push(tag) + lines.push('') + } + return + } + + // Static text / labels / status — render plain. + if (role === 'label' || role === 'static_text' || role === 'status_bar') { + if (el.name || el.value) { + lines.push(escapeBody(el.value ?? el.name ?? '')) + lines.push('') + } + return + } + + // Anything else — recurse into children but otherwise ignore. + renderChildren(el, lines, state, depth + 1) +} + +function renderChildren( + el: DesktopElement, + lines: string[], + state: BuilderState, + depth: number, +): void { + for (const child of el.children ?? []) { + renderElement(child, lines, state, depth) + } +} + +function renderInteractiveAsTag(el: DesktopElement, state: BuilderState): string | null { + const actionId = uniqueActionId(state, el.id || el.name || el.role) + const def: ActionDefinition = { + type: actionTypeFor(el.role), + label: el.name || el.value || el.id || el.role, + } + if (el.value !== undefined) def.value = el.value + if (el.placeholder !== undefined) def.placeholder = el.placeholder + if (el.enabled === false) { + def.disabled = true + } + if (el.read_only === true) def.read_only = true + if (el.aria) { + def.aria = {} + if (el.aria.pressed !== undefined) def.aria.pressed = el.aria.pressed + if (el.aria.checked !== undefined) def.aria.checked = el.aria.checked + } + if (el.selected !== undefined) { + def.aria = def.aria ?? {} + def.aria.selected = el.selected + } + if (el.expanded !== undefined) { + def.aria = def.aria ?? {} + def.aria.expanded = el.expanded + } + + state.actions[actionId] = def + if (el.id) state.element_ids[actionId] = el.id + state.has_interactive = true + + const tagKind = tagKindFor(el.role) + return `[${tagKind}:${actionId}]` +} + +function renderTable(el: DesktopElement, lines: string[], state: BuilderState): void { + if (el.name) { + lines.push(`**${escapeBody(el.name)}**`) + lines.push('') + } + const rows = (el.children ?? []).filter(c => c.role === 'row') + const headerRow = (el.children ?? []).find(c => c.role === 'row' && (c.children ?? []).some(cc => cc.role === 'column_header')) + const headers = headerRow + ? (headerRow.children ?? []).filter(c => c.role === 'column_header' || c.role === 'cell').map(c => c.name ?? '') + : (el.children ?? []).filter(c => c.role === 'column_header').map(c => c.name ?? '') + + if (headers.length === 0) { + // No headers — fall back to plain list of rows. + for (const row of rows) { + const cells = (row.children ?? []).filter(c => c.role === 'cell') + const line = cells.map(c => escapeBody(c.value ?? c.name ?? '')).join(' | ') + if (line) lines.push(`- ${line}`) + } + lines.push('') + return + } + + lines.push(`| ${headers.map(escapeBody).join(' | ')} |`) + lines.push(`| ${headers.map(() => '---').join(' | ')} |`) + + for (const row of rows) { + if (row === headerRow) continue + const cells = (row.children ?? []).filter(c => c.role === 'cell') + const padded = headers.map((_, i) => escapeBody(cells[i]?.value ?? cells[i]?.name ?? '')) + lines.push(`| ${padded.join(' | ')} |`) + } + lines.push('') +} diff --git a/src/desktop/desktop-converter.ts b/src/desktop/desktop-converter.ts new file mode 100644 index 0000000..7793bae --- /dev/null +++ b/src/desktop/desktop-converter.ts @@ -0,0 +1,183 @@ +/** + * `convertDesktop()` — capture a native window via an accessibility-tree + * backend and emit an AgentMark snapshot with `kind: 'desktop'`. + * + * Mirrors `convertAudio()` and `convertVideo()`: the converter is + * pure-Node and OS-agnostic; OS-specific work is hidden behind the + * `DesktopCaptureBackend` interface. + * + * Output body grammar (see `body-builder.ts` for full detail): + * + * [WINDOW:w_excel] + * + * # Microsoft Excel — Book1 + * + * ## Worksheet + * + * [INPUT:act_cell_a1] + * + * [ACTION:act_save] + * + * Interactive controls produce entries in the snapshot's `actions` map; + * the runtime invokes the backend's `execute()` with the matching + * `element_id` (stored as `target_id` on each action so the renderer + * can resolve it). + */ + +import { + AGENTMARK_VERSION, + type ConversionResult, + type DesktopMeta, + type Snapshot, +} from '../types' +import { serializeSnapshot } from '../serializers/yaml-frontmatter' +import { InMemoryActionBinding } from '../binding/action-binding' +import { noopLogger, type Logger } from '../observability/logger' +import { SnapshotError } from '../errors' +import { buildDesktopBody } from './body-builder' +import type { + DesktopCaptureBackend, + DesktopCapture, + DesktopTarget, +} from './types' + +export interface ConvertDesktopOptions { + /** Accessibility-tree backend (Windows UIA, macOS AXAPI, vision fallback, fixture, etc.). */ + backend: DesktopCaptureBackend + + /** Which window to capture. Defaults to focused window if omitted. */ + target?: DesktopTarget + + /** Override the snapshot title. Default: the captured window title. */ + title?: string + + /** Override the snapshot URL — use this when the producer wants to + * identify the surface canonically (e.g. `desktop://hostname/app/process_id`). + * Default: a synthesised `desktop:///` URI. */ + url?: string + + /** Max tree depth to traverse. Default: 12. */ + maxDepth?: number + + /** Include hidden / off-screen elements. Default: false. */ + includeHidden?: boolean + + /** Per-capture timeout (ms). Default: 5000. */ + timeoutMs?: number + + /** TTL for `expires_at` (ms). Desktop UI is highly dynamic, so the + * default is short — 15 seconds. Producers should re-capture before + * acting on a stale snapshot. */ + ttlMs?: number + + /** Structured logger. */ + logger?: Logger + + /** Vendor extensions (`x-` prefixed fields) attached to the snapshot. */ + vendorExtensions?: Record +} + +export async function convertDesktop(options: ConvertDesktopOptions): Promise { + const logger = options.logger ?? noopLogger + const ttlMs = options.ttlMs ?? 15_000 + + logger.debug('snapshot.capture.start', { + source: options.target?.window_title ?? options.target?.process_name ?? '', + kind: 'desktop', + backend: options.backend.name, + }) + + let capture: DesktopCapture + try { + capture = await options.backend.capture({ + target: options.target, + maxDepth: options.maxDepth ?? 12, + includeHidden: options.includeHidden ?? false, + timeoutMs: options.timeoutMs ?? 5000, + }) + } catch (err) { + logger.error('snapshot.failed', { error: (err as Error).message }) + if (err instanceof SnapshotError) throw err + throw new SnapshotError( + `Desktop capture failed (${options.backend.name}): ${(err as Error).message}`, + err as Error, + ) + } + + const captured_at = new Date().toISOString() + const expires_at = new Date(Date.now() + ttlMs).toISOString() + + const { body, actions, element_ids, has_interactive } = buildDesktopBody(capture) + + // Stash each action's underlying desktop element_id in the + // ActionBinding. The runtime later resolves binding.get(actionId) + // to the opaque element_id and passes it to backend.execute(). + const binding = new InMemoryActionBinding() + for (const [actionId, elementId] of Object.entries(element_ids)) { + binding.set(actionId, elementId) + } + + const desktopMeta: DesktopMeta = stripUndefined({ + platform: capture.platform, + process_name: capture.process_name, + process_id: capture.process_id, + window_class: capture.window_class, + focused_element_id: capture.focused_element_id, + a11y_backend: options.backend.name, + tree_depth: capture.tree_depth, + element_count: capture.element_count, + }) + + const snapshot: Snapshot = { + agentmark: AGENTMARK_VERSION, + kind: 'desktop', + url: options.url ?? synthesiseDesktopUrl(capture), + title: options.title ?? capture.window_title, + captured_at, + expires_at, + source: 'rendered', + desktop_meta: desktopMeta, + actions: has_interactive ? actions : undefined, + capabilities: { + preview_media: false, + expand_disclosures: true, + paginate: false, + scroll: true, + keyboard: true, + drag: false, + ocr: false, + vision: false, + }, + body, + } + + if (options.vendorExtensions) { + for (const [k, v] of Object.entries(options.vendorExtensions)) { + if (k.startsWith('x-')) (snapshot as unknown as Record)[k] = v + } + } + + const text = serializeSnapshot(snapshot) + logger.info('snapshot.captured', { + source: snapshot.url, + kind: 'desktop', + backend: options.backend.name, + elements: capture.element_count, + actions: Object.keys(actions).length, + bytes: text.length, + }) + + return { agentmark: text, binding } +} + +function synthesiseDesktopUrl(capture: DesktopCapture): string { + const host = capture.process_name?.toLowerCase().replace(/[^a-z0-9._-]/g, '_') ?? 'unknown' + const path = capture.window_id ?? String(capture.process_id ?? 'window') + return `desktop://${capture.platform}/${host}/${path}` +} + +function stripUndefined(obj: T): T { + const out: Record = {} + for (const [k, v] of Object.entries(obj)) if (v !== undefined) out[k] = v + return out as T +} diff --git a/src/desktop/fixture-backend.ts b/src/desktop/fixture-backend.ts new file mode 100644 index 0000000..b27d794 --- /dev/null +++ b/src/desktop/fixture-backend.ts @@ -0,0 +1,269 @@ +/** + * `FixtureBackend` — an in-memory `DesktopCaptureBackend` that returns + * pre-baked accessibility trees. Useful for: + * + * - Local development on machines without the OS bridge installed + * (e.g. running the MCP server on macOS before the AXAPI bridge + * exists, or on a CI runner with no GUI). + * - Tests for the converter and MCP server. + * - Demo flows that don't need real applications. + * + * Ships one preset by default — an Excel-like spreadsheet window — + * mirroring the shape a real Windows UIA backend would produce. Callers + * can supply their own `presets` map for custom scenarios. + */ + +import type { + DesktopCapture, + DesktopCaptureBackend, + CaptureDesktopOptions, + DesktopElement, + ExecuteDesktopOptions, + ExecuteDesktopResult, +} from './types' + +export interface FixtureBackendOptions { + /** Preset name → DesktopCapture. Defaults to a built-in Excel preset. */ + presets?: Record + /** Default preset to return when `target` is omitted or doesn't match. */ + defaultPreset?: string + /** Optional latency to simulate the bridge round-trip (ms). */ + latencyMs?: number +} + +export class FixtureBackend implements DesktopCaptureBackend { + readonly name = 'fixture' + private readonly presets: Record + private readonly defaultPreset: string + private readonly latencyMs: number + + /** Captured executions — handy for tests to assert what was driven. */ + public readonly executed: Array = [] + + /** Mutable map of element_id → new value, applied to subsequent + * capture() results so a test can roundtrip type → re-capture → + * observe the typed text. */ + public readonly elementValues = new Map() + + constructor(opts: FixtureBackendOptions = {}) { + this.presets = { ...DEFAULT_PRESETS, ...(opts.presets ?? {}) } + this.defaultPreset = opts.defaultPreset ?? 'excel_blank' + this.latencyMs = opts.latencyMs ?? 0 + } + + async capture(opts: CaptureDesktopOptions = {}): Promise { + if (this.latencyMs) await delay(this.latencyMs) + + const key = pickPresetKey(opts, this.presets, this.defaultPreset) + const base = this.presets[key] + if (!base) { + throw new Error(`FixtureBackend: no preset named "${key}"`) + } + + // Clone so mutations don't leak across calls, then layer in any + // values the test has typed. + const cloned = structuredClone(base) as DesktopCapture + if (this.elementValues.size > 0) { + applyValues(cloned.root, this.elementValues) + } + return cloned + } + + async execute(opts: ExecuteDesktopOptions): Promise { + if (this.latencyMs) await delay(this.latencyMs) + this.executed.push(opts) + + switch (opts.action.type) { + case 'type': + this.elementValues.set(opts.action.element_id, opts.action.text) + return { ok: true, new_value: opts.action.text } + case 'check': + this.elementValues.set(opts.action.element_id, opts.action.checked ? 'true' : 'false') + return { ok: true, new_value: String(opts.action.checked) } + case 'select': + this.elementValues.set(opts.action.element_id, opts.action.value) + return { ok: true, new_value: opts.action.value } + default: + return { ok: true } + } + } +} + +function delay(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)) +} + +function pickPresetKey( + opts: CaptureDesktopOptions, + presets: Record, + fallback: string, +): string { + const t = opts.target + if (!t) return fallback + if (t.window_id && presets[t.window_id]) return t.window_id + if (t.process_name) { + const k = t.process_name.toLowerCase().replace(/\..*$/, '') + if (presets[k]) return k + } + if (t.window_title) { + const k = t.window_title.toLowerCase().replace(/[^a-z0-9_]/g, '_') + if (presets[k]) return k + } + return fallback +} + +function applyValues(el: DesktopElement, values: Map): void { + const v = values.get(el.id) + if (v !== undefined) el.value = v + for (const child of el.children ?? []) applyValues(child, values) +} + +// ────────────────────────────────────────────────────────────────────────── +// Built-in presets +// ────────────────────────────────────────────────────────────────────────── + +const EXCEL_BLANK: DesktopCapture = { + platform: 'windows', + process_name: 'EXCEL.EXE', + process_id: 12345, + window_title: 'Microsoft Excel - Book1', + window_class: 'XLMAIN', + window_id: 'excel_blank', + focused_element_id: 'cell_A1', + tree_depth: 4, + element_count: 9, + root: { + id: 'root', + role: 'window', + name: 'Microsoft Excel - Book1', + enabled: true, + children: [ + { + id: 'ribbon', + role: 'toolbar', + name: 'Ribbon', + enabled: true, + children: [ + { id: 'btn_save', role: 'button', name: 'Save', enabled: true }, + { id: 'btn_undo', role: 'button', name: 'Undo', enabled: false }, + { id: 'btn_redo', role: 'button', name: 'Redo', enabled: false }, + ], + }, + { + id: 'sheet_sheet1', + role: 'pane', + name: 'Sheet1', + enabled: true, + children: [ + { + id: 'grid_main', + role: 'table', + name: 'Sheet1 grid', + enabled: true, + children: [ + { + id: 'row_header', + role: 'row', + children: [ + { id: 'col_A', role: 'column_header', name: 'A' }, + { id: 'col_B', role: 'column_header', name: 'B' }, + { id: 'col_C', role: 'column_header', name: 'C' }, + ], + }, + { + id: 'row_1', + role: 'row', + children: [ + { id: 'cell_A1', role: 'cell', name: 'A1', value: '', enabled: true }, + { id: 'cell_B1', role: 'cell', name: 'B1', value: '', enabled: true }, + { id: 'cell_C1', role: 'cell', name: 'C1', value: '', enabled: true }, + ], + }, + ], + }, + ], + }, + { + id: 'status_bar', + role: 'status_bar', + name: 'Ready', + value: 'Ready', + }, + ], + }, +} + +const NOWCERTS_CUSTOMER: DesktopCapture = { + platform: 'windows', + process_name: 'NowCerts.exe', + process_id: 22120, + window_title: 'NowCerts - Customer Detail - Acme Corp', + window_class: 'WindowsForms10.Window.8.app.0.378734a', + window_id: 'nowcerts_customer', + focused_element_id: 'in_company', + tree_depth: 5, + element_count: 12, + root: { + id: 'root', + role: 'window', + name: 'NowCerts - Customer Detail', + enabled: true, + children: [ + { + id: 'panel_company', + role: 'group', + name: 'Company Information', + enabled: true, + children: [ + { + id: 'in_company', + role: 'text_input', + name: 'Company Name', + value: 'Acme Corp', + enabled: true, + }, + { + id: 'in_email', + role: 'text_input', + name: 'Primary Email', + value: 'contact@acme.com', + enabled: true, + }, + { + id: 'in_phone', + role: 'text_input', + name: 'Phone', + value: '(555) 123-4567', + enabled: true, + }, + { + id: 'cb_active', + role: 'check_box', + name: 'Active Customer', + selected: true, + enabled: true, + aria: { checked: true }, + }, + ], + }, + { + id: 'panel_actions', + role: 'toolbar', + name: 'Actions', + enabled: true, + children: [ + { id: 'btn_save', role: 'button', name: 'Save', enabled: true }, + { id: 'btn_cancel', role: 'button', name: 'Cancel', enabled: true }, + { id: 'btn_new_policy', role: 'button', name: 'New Policy', enabled: true }, + ], + }, + ], + }, +} + +const DEFAULT_PRESETS: Record = { + excel_blank: EXCEL_BLANK, + excel: EXCEL_BLANK, + nowcerts_customer: NOWCERTS_CUSTOMER, + nowcerts: NOWCERTS_CUSTOMER, +} diff --git a/src/desktop/index.ts b/src/desktop/index.ts new file mode 100644 index 0000000..a734af4 --- /dev/null +++ b/src/desktop/index.ts @@ -0,0 +1,25 @@ +/** + * Desktop support — convertDesktop() + accessibility-tree capture backends. + */ + +export { convertDesktop } from './desktop-converter' +export type { ConvertDesktopOptions } from './desktop-converter' + +export { FixtureBackend } from './fixture-backend' +export type { FixtureBackendOptions } from './fixture-backend' + +export type { + DesktopCaptureBackend, + CaptureDesktopOptions, + DesktopCapture, + DesktopElement, + DesktopRole, + DesktopTarget, + ExecuteDesktopOptions, + ExecuteDesktopAction, + ExecuteDesktopResult, + KeyModifier, +} from './types' + +export { buildDesktopBody } from './body-builder' +export type { BuildDesktopBodyResult } from './body-builder' diff --git a/src/desktop/types.ts b/src/desktop/types.ts new file mode 100644 index 0000000..54e897c --- /dev/null +++ b/src/desktop/types.ts @@ -0,0 +1,211 @@ +/** + * Desktop support — types for OS-accessibility capture backends and the + * structured tree `convertDesktop()` consumes. + * + * Backends implement `DesktopCaptureBackend` and are typically OS-bound: + * + * - Windows: a .NET sidecar process exposing UIA via FlaUI + * - macOS: a Swift/ObjC sidecar exposing AXAPI + * - Linux: a sidecar over AT-SPI / D-Bus + * - Vision: a fallback that captures a screenshot and parses it + * (e.g. OmniParser, GUI-Actor) when no accessibility tree + * is available (Electron canvas, Citrix sessions, etc.) + * + * The converter is OS-agnostic: it consumes a `DesktopCapture` tree and + * produces an AgentMark Snapshot with `kind: 'desktop'`. Bridges live in + * separate processes/repos; the Node-side just talks to whatever bridge + * implements this interface. + */ + +export interface DesktopCaptureBackend { + /** Stable backend identifier — populated into `desktop_meta.a11y_backend`. + * Convention: `windows_uia`, `macos_axapi`, `linux_atspi`, `vision_fallback`, + * or `fixture` for tests. */ + readonly name: string + + /** Capture the current state of a target window/application. */ + capture(opts: CaptureDesktopOptions): Promise + + /** Execute an action against a previously-captured element. The element + * is identified by `element_id` which comes from the capture tree. */ + execute(opts: ExecuteDesktopOptions): Promise + + /** Optional teardown — release native handles, close sidecar process. */ + close?(): Promise +} + +/** Identifies a target window. Backends accept any combination they can + * resolve; if nothing is provided the currently focused window is used. */ +export interface DesktopTarget { + process_name?: string + process_id?: number + window_title?: string + /** Opaque OS handle — e.g. Windows HWND as a stringified pointer, macOS + * AXUIElementRef pointer encoded as a base64 string. Returned in + * `DesktopCapture.window_id` so callers can re-target precisely. */ + window_id?: string +} + +export interface CaptureDesktopOptions { + /** Which window/app to capture. Defaults to focused window if omitted. */ + target?: DesktopTarget + /** Maximum tree depth to traverse. Default: backend-defined (typically 12). */ + maxDepth?: number + /** Whether to include elements that are off-screen or invisible. Default: false. */ + includeHidden?: boolean + /** Per-request timeout (ms). Default: 5000. */ + timeoutMs?: number +} + +export interface DesktopCapture { + platform: 'windows' | 'macos' | 'linux' + process_name?: string + process_id?: number + window_title: string + window_class?: string + /** Backend-defined opaque handle for re-targeting this exact window. */ + window_id?: string + /** Stable accessibility ID of the currently focused element, when one + * is focused inside the captured window. */ + focused_element_id?: string + /** Maximum depth the backend traversed (may be < requested if a leaf + * was reached first). */ + tree_depth: number + /** Total elements captured (interactive + static). */ + element_count: number + /** Root of the accessibility tree. */ + root: DesktopElement +} + +/** + * One node in the accessibility tree. Roles map to the underlying OS + * concept (UIA ControlType / AXRole). The converter inspects `role` to + * decide whether the element becomes an `ActionDefinition` (interactive) + * or a static body block (label/text/group). + */ +export interface DesktopElement { + /** Stable accessibility identifier. Windows: UIA AutomationId; macOS: + * AXIdentifier; Linux: AT-SPI accessible-id. Falls back to a + * backend-generated stable hash when the platform omits the ID. */ + id: string + + /** Control role — backends normalise to a small vocabulary the + * converter understands. */ + role: DesktopRole + + /** Accessible name / display label. */ + name?: string + + /** Current value (for inputs, sliders, combo selections, etc.). */ + value?: string + + /** Placeholder / help text when the element shows one. */ + placeholder?: string + + /** Whether the element is enabled. Disabled elements are still + * captured but won't generate executable actions. */ + enabled?: boolean + + /** Selection state for checkboxes, radios, list items, tabs. */ + selected?: boolean + + /** Read-only text fields, etc. */ + read_only?: boolean + + /** ExpandCollapsePattern state for combo boxes, tree items, menus. */ + expanded?: boolean + + /** ARIA-equivalent properties when supplied by the backend. */ + aria?: { + pressed?: boolean + checked?: boolean | 'mixed' + required?: boolean + invalid?: boolean + } + + /** Bounds in screen coordinates. Optional — used by vision-fallback + * backends or when the agent wants pixel-targeted clicks. */ + bounds?: { x: number; y: number; width: number; height: number } + + /** Nested children. Backends pre-order traversal; converter renders + * in document order. */ + children?: DesktopElement[] +} + +/** + * Normalised role vocabulary. Backends translate UIA/AXAPI roles to + * these. Unknowns become `'other'` and render as static body text. + */ +export type DesktopRole = + | 'window' + | 'pane' + | 'group' + | 'toolbar' + | 'menu' + | 'menu_item' + | 'tab_list' + | 'tab' + | 'tree' + | 'tree_item' + | 'list' + | 'list_item' + | 'table' + | 'row' + | 'cell' + | 'column_header' + | 'button' + | 'split_button' + | 'text_input' + | 'password_input' + | 'text_area' + | 'check_box' + | 'radio_button' + | 'combo_box' + | 'list_box' + | 'slider' + | 'progress_bar' + | 'link' + | 'label' + | 'static_text' + | 'image' + | 'separator' + | 'status_bar' + | 'scroll_bar' + | 'dialog' + | 'tooltip' + | 'other' + +export interface ExecuteDesktopOptions { + /** Which window the element lives in. Optional if the backend can + * resolve the element_id globally. */ + target?: DesktopTarget + /** What to do. */ + action: ExecuteDesktopAction + /** Per-request timeout (ms). Default: 5000. */ + timeoutMs?: number +} + +/** All supported actions. The backend translates these to UIA patterns + * (InvokePattern, ValuePattern, TogglePattern, etc.) or AXAPI equivalents. */ +export type ExecuteDesktopAction = + | { type: 'click'; element_id: string } + | { type: 'type'; element_id: string; text: string; clear_first?: boolean } + | { type: 'select'; element_id: string; value: string } + | { type: 'check'; element_id: string; checked: boolean } + | { type: 'expand'; element_id: string; expanded: boolean } + | { type: 'focus'; element_id: string } + | { type: 'scroll_to'; element_id: string } + | { type: 'key'; element_id?: string; key: string; modifiers?: ReadonlyArray } + +export type KeyModifier = 'ctrl' | 'alt' | 'shift' | 'meta' | 'win' + +export interface ExecuteDesktopResult { + ok: boolean + /** Backend-supplied detail when ok is false (e.g. 'element disabled', + * 'element no longer in tree'). */ + message?: string + /** Snapshot of the element AFTER the action when the backend can + * cheaply re-query it. Lets the agent confirm state without a full + * re-capture. */ + new_value?: string +} diff --git a/src/index.ts b/src/index.ts index 91ca6ad..68e8b72 100644 --- a/src/index.ts +++ b/src/index.ts @@ -211,3 +211,22 @@ export type { ExtractFramesOptions, ExtractedFrame, } from './video' + +// ── v0.4 spec / v0.12 lib: Desktop support (kind: 'desktop') ───────────── + +export { convertDesktop, FixtureBackend, buildDesktopBody } from './desktop' +export type { + ConvertDesktopOptions, + FixtureBackendOptions, + BuildDesktopBodyResult, + DesktopCaptureBackend, + CaptureDesktopOptions, + DesktopCapture, + DesktopElement, + DesktopRole, + DesktopTarget, + ExecuteDesktopOptions, + ExecuteDesktopAction, + ExecuteDesktopResult, + KeyModifier, +} from './desktop' diff --git a/test/desktop/desktop-converter.test.ts b/test/desktop/desktop-converter.test.ts new file mode 100644 index 0000000..a06e141 --- /dev/null +++ b/test/desktop/desktop-converter.test.ts @@ -0,0 +1,148 @@ +import { describe, it, expect } from 'vitest' +import { convertDesktop } from '../../src/desktop/desktop-converter' +import { FixtureBackend } from '../../src/desktop/fixture-backend' +import { parseSnapshot } from '../../src/serializers/yaml-frontmatter' +import { validateSnapshot } from '../../src/validators/schema-validator' + +describe('convertDesktop', () => { + it('produces a valid v0.4 desktop snapshot from the Excel fixture', async () => { + const backend = new FixtureBackend() + const { agentmark, binding } = await convertDesktop({ backend }) + + const snap = parseSnapshot(agentmark) + expect(snap.agentmark).toBe('0.4') + expect(snap.kind).toBe('desktop') + expect(snap.title).toBe('Microsoft Excel - Book1') + expect(snap.desktop_meta?.platform).toBe('windows') + expect(snap.desktop_meta?.process_name).toBe('EXCEL.EXE') + expect(snap.desktop_meta?.a11y_backend).toBe('fixture') + expect(snap.desktop_meta?.element_count).toBe(9) + + // Body includes the window marker, the title, and at least one + // INPUT/ACTION reference resolvable in actions. + expect(snap.body).toContain('[WINDOW:w_excel_exe]') + expect(snap.body).toContain('# Microsoft Excel - Book1') + expect(snap.body).toContain('[ACTION:') + + // Validates clean against the schema + cross-field invariants. + const result = validateSnapshot(snap) + expect(result.errors).toEqual([]) + expect(result.valid).toBe(true) + + // Binding has entries for every action. + for (const actionId of Object.keys(snap.actions ?? {})) { + expect(binding.get(actionId)).toBeDefined() + } + }) + + it('renders the NowCerts fixture with editable inputs as INPUT tags', async () => { + const backend = new FixtureBackend() + const { agentmark } = await convertDesktop({ + backend, + target: { window_id: 'nowcerts_customer' }, + }) + const snap = parseSnapshot(agentmark) + expect(snap.title).toBe('NowCerts - Customer Detail - Acme Corp') + expect(snap.body).toContain('[INPUT:act_in_company]') + expect(snap.actions?.act_in_company?.type).toBe('type') + expect(snap.actions?.act_in_company?.value).toBe('Acme Corp') + expect(snap.actions?.act_in_company?.label).toBe('Company Name') + }) + + it('round-trips type → re-capture → observe the typed text', async () => { + const backend = new FixtureBackend() + + // First capture — Company Name has its initial fixture value. + const initial = await convertDesktop({ + backend, + target: { window_id: 'nowcerts_customer' }, + }) + const initialSnap = parseSnapshot(initial.agentmark) + expect(initialSnap.actions?.act_in_company?.value).toBe('Acme Corp') + + // Resolve the binding to get the underlying element_id, then + // drive the backend's execute() the way the runtime will. + const elementId = initial.binding.get('act_in_company') + expect(elementId).toBe('in_company') + + await backend.execute({ + action: { type: 'type', element_id: elementId!, text: 'Beta Industries' }, + }) + expect(backend.executed).toHaveLength(1) + + // Second capture — the typed value should be reflected. + const updated = await convertDesktop({ + backend, + target: { window_id: 'nowcerts_customer' }, + }) + const updatedSnap = parseSnapshot(updated.agentmark) + expect(updatedSnap.actions?.act_in_company?.value).toBe('Beta Industries') + }) + + it('returns a synthesised desktop:// URL when none is provided', async () => { + const backend = new FixtureBackend() + const { agentmark } = await convertDesktop({ backend }) + const snap = parseSnapshot(agentmark) + expect(snap.url).toMatch(/^desktop:\/\/windows\/excel\.exe\//) + }) + + it('respects an explicit URL override', async () => { + const backend = new FixtureBackend() + const { agentmark } = await convertDesktop({ + backend, + url: 'desktop://my-host/excel/12345', + }) + const snap = parseSnapshot(agentmark) + expect(snap.url).toBe('desktop://my-host/excel/12345') + }) + + it('reports the backend name in desktop_meta.a11y_backend', async () => { + const backend = new FixtureBackend() + const { agentmark } = await convertDesktop({ backend }) + const snap = parseSnapshot(agentmark) + expect(snap.desktop_meta?.a11y_backend).toBe('fixture') + }) + + it('handles a check_box element with aria.checked state', async () => { + const backend = new FixtureBackend() + const { agentmark } = await convertDesktop({ + backend, + target: { window_id: 'nowcerts_customer' }, + }) + const snap = parseSnapshot(agentmark) + const activeCheckbox = snap.actions?.act_cb_active + expect(activeCheckbox?.type).toBe('check') + expect(activeCheckbox?.aria?.checked).toBe(true) + }) + + it('drops the `actions` field when no interactive elements are captured', async () => { + const staticBackend: import('../../src/desktop/types').DesktopCaptureBackend = { + name: 'static-fixture', + async capture() { + return { + platform: 'macos', + process_name: 'Pages', + window_title: 'Untitled - Pages', + tree_depth: 2, + element_count: 2, + root: { + id: 'root', + role: 'window', + name: 'Pages', + children: [ + { id: 'label_1', role: 'static_text', name: 'Empty document.' }, + ], + }, + } + }, + async execute() { + return { ok: true } + }, + } + const { agentmark } = await convertDesktop({ backend: staticBackend }) + const snap = parseSnapshot(agentmark) + expect(snap.kind).toBe('desktop') + expect(snap.actions).toBeUndefined() + expect(snap.body).toContain('Empty document.') + }) +}) From 500d3cad9189ec888b1b48d132dc1077e01d543e Mon Sep 17 00:00:00 2001 From: rrader26 Date: Mon, 11 May 2026 10:03:04 -0400 Subject: [PATCH 2/3] feat(mcp): expose desktop tools (open / snapshot / execute / close) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the v0.4 desktop kind through the existing MCP server so any MCP-compatible AI tool (Claude Code, Cursor, Codex, Windsurf, Continue, Cody, Agent Zero, etc.) can drive native applications via AgentMark by adding one stanza to its mcp config: { "mcpServers": { "agentmark": { "command": "npx", "args": ["-y", "@thinkfleet/agentmark", "mcp"] } } } Once connected the AI gets: - agentmark_desktop_open — attach to a capture backend (default: fixture) - agentmark_desktop_snapshot — capture the focused window (or specified target) as v0.4 markdown - agentmark_desktop_execute — drive an action by ID against the most recent snapshot. Server resolves action_id → backend element_id via the ActionBinding cached at snapshot time - agentmark_desktop_close — release the backend Backend selection at open-time: `fixture` ships today (in-memory Excel + NowCerts trees, works on any OS). `windows_uia` and `macos_axapi` return a friendly "not yet bundled" error until the real bridges ship in follow-up PRs. Action-type → ExecuteDesktopAction mapping happens server-side so the client only needs to know the action_id from the snapshot, not the underlying UIA/AXAPI pattern. Files touched: - src/mcp/types.ts — DesktopSession type + 'dt' prefix - src/mcp/tool-defs.ts — 4 new tools added to ALL_TOOLS - src/mcp/dispatcher.ts — handlers, switch cases, listSessions, disposeAll all extended - test/mcp/desktop-dispatcher.test.ts — 12 new tests covering open, snapshot, execute round-trip, unknown action handling, close, list_sessions integration Full suite: 295 passed, 10 skipped, 0 failed. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/mcp/dispatcher.ts | 225 +++++++++++++++++++++++++++- src/mcp/tool-defs.ts | 150 ++++++++++++++++++- src/mcp/types.ts | 31 +++- test/mcp/desktop-dispatcher.test.ts | 170 +++++++++++++++++++++ 4 files changed, 570 insertions(+), 6 deletions(-) create mode 100644 test/mcp/desktop-dispatcher.test.ts diff --git a/src/mcp/dispatcher.ts b/src/mcp/dispatcher.ts index b1e8e7c..eee402f 100644 --- a/src/mcp/dispatcher.ts +++ b/src/mcp/dispatcher.ts @@ -12,21 +12,29 @@ import * as path from 'node:path' import { pathToFileURL } from 'node:url' import { createBrowser, + convertDesktop, + FixtureBackend, openPdfDocument, isAgentMarkError, + parseSnapshot, PopplerRenderBackend, TesseractOcrBackend, type Browser, + type DesktopCaptureBackend, + type DesktopTarget, + type ExecuteDesktopAction, + type KeyModifier, type Page, type PdfDocument, type OcrPipelineOptions, } from '../index' -import { generateSessionId, type BrowserSession, type PdfSession } from './types' +import { generateSessionId, type BrowserSession, type DesktopSession, type PdfSession } from './types' export interface DispatcherState { browsers: Map pages: Map pdfs: Map + desktops: Map } export function createDispatcherState(): DispatcherState { @@ -34,6 +42,7 @@ export function createDispatcherState(): DispatcherState { browsers: new Map(), pages: new Map(), pdfs: new Map(), + desktops: new Map(), } } @@ -83,6 +92,16 @@ export async function dispatch( case 'agentmark_pdf_reset': return await pdfReset(state, args) + // ── Desktop ────────────────────────────────────────────────── + case 'agentmark_desktop_open': + return await openDesktop(state, args) + case 'agentmark_desktop_close': + return await closeDesktop(state, args) + case 'agentmark_desktop_snapshot': + return await desktopSnapshot(state, args) + case 'agentmark_desktop_execute': + return await desktopExecute(state, args) + // ── Meta ───────────────────────────────────────────────────── case 'agentmark_list_sessions': return listSessions(state) @@ -312,6 +331,194 @@ async function pdfReset(state: DispatcherState, args: Record): return { text: `PDF ${id} pending values cleared.` } } +// ────────────────────────────────────────────────────────────────────────── +// Desktop tool handlers +// ────────────────────────────────────────────────────────────────────────── + +async function openDesktop(state: DispatcherState, args: Record): Promise { + const requested = typeof args.backend === 'string' ? args.backend : 'fixture' + let backend: DesktopCaptureBackend + switch (requested) { + case 'fixture': + backend = new FixtureBackend() + break + case 'windows_uia': + case 'macos_axapi': + return { + text: + `Backend "${requested}" is not yet bundled with this build of agentmark. ` + + 'Run agentmark_desktop_open with backend="fixture" to use the in-memory ' + + 'preset trees. Real OS bridges land in subsequent releases.', + isError: true, + } + default: + return { text: `Unknown desktop backend: ${requested}`, isError: true } + } + + const id = generateSessionId('dt') + state.desktops.set(id, { + id, + backend, + createdAt: new Date(), + }) + return { + text: JSON.stringify({ desktop_id: id, backend: requested }, null, 2), + } +} + +async function closeDesktop(state: DispatcherState, args: Record): Promise { + const id = requireString(args, 'desktop_id') + const session = state.desktops.get(id) + if (!session) return { text: `Unknown desktop_id: ${id}`, isError: true } + await session.backend.close?.() + state.desktops.delete(id) + return { text: `Desktop session ${id} closed.` } +} + +async function desktopSnapshot(state: DispatcherState, args: Record): Promise { + const id = requireString(args, 'desktop_id') + const session = requireDesktop(state, id) + + const target = parseTarget(args.target) + const maxDepth = typeof args.max_depth === 'number' ? args.max_depth : undefined + const includeHidden = args.include_hidden === true + const timeoutMs = typeof args.timeout_ms === 'number' ? args.timeout_ms : undefined + + const { agentmark, binding } = await convertDesktop({ + backend: session.backend, + target, + maxDepth, + includeHidden, + timeoutMs, + }) + + // Cache binding + action types so subsequent _execute can resolve. + session.lastTarget = target + session.lastBinding = binding + const snap = parseSnapshot(agentmark) + session.lastActionTypes = new Map( + Object.entries(snap.actions ?? {}).map(([k, def]) => [k, def.type]), + ) + + return { text: agentmark } +} + +async function desktopExecute(state: DispatcherState, args: Record): Promise { + const id = requireString(args, 'desktop_id') + const actionId = requireString(args, 'action_id') + const session = requireDesktop(state, id) + + if (!session.lastBinding || !session.lastActionTypes) { + return { + text: + `No cached snapshot for desktop_id ${id}. Call ` + + `agentmark_desktop_snapshot first so the action_id can be resolved.`, + isError: true, + } + } + + const elementId = session.lastBinding.get(actionId) + if (!elementId) { + return { text: `Unknown action_id: ${actionId}`, isError: true } + } + + const actionType = session.lastActionTypes.get(actionId) ?? 'click' + const value = args.value + const modifiers = parseModifiers(args.modifiers) + const clearFirst = args.clear_first === true + + const action = buildExecuteAction(actionType, elementId, value, modifiers, clearFirst) + const result = await session.backend.execute({ + target: session.lastTarget, + action, + }) + + return { + text: JSON.stringify( + { + action_id: actionId, + action_type: actionType, + element_id: elementId, + ok: result.ok, + ...(result.message !== undefined ? { message: result.message } : {}), + ...(result.new_value !== undefined ? { new_value: result.new_value } : {}), + }, + null, + 2, + ), + isError: !result.ok, + } +} + +function parseTarget(input: unknown): DesktopTarget | undefined { + if (!input || typeof input !== 'object') return undefined + const t = input as Record + const out: DesktopTarget = {} + if (typeof t.process_name === 'string') out.process_name = t.process_name + if (typeof t.process_id === 'number') out.process_id = t.process_id + if (typeof t.window_title === 'string') out.window_title = t.window_title + if (typeof t.window_id === 'string') out.window_id = t.window_id + return Object.keys(out).length > 0 ? out : undefined +} + +function parseModifiers(input: unknown): KeyModifier[] | undefined { + if (!Array.isArray(input)) return undefined + const allowed: ReadonlySet = new Set(['ctrl', 'alt', 'shift', 'meta', 'win']) + const out: KeyModifier[] = [] + for (const m of input) { + if (typeof m === 'string' && allowed.has(m as KeyModifier)) out.push(m as KeyModifier) + } + return out.length > 0 ? out : undefined +} + +function buildExecuteAction( + actionType: string, + elementId: string, + value: unknown, + modifiers: KeyModifier[] | undefined, + clearFirst: boolean, +): ExecuteDesktopAction { + switch (actionType) { + case 'type': + return { + type: 'type', + element_id: elementId, + text: typeof value === 'string' ? value : String(value ?? ''), + clear_first: clearFirst, + } + case 'check': + return { + type: 'check', + element_id: elementId, + checked: value === true || value === 'true', + } + case 'select': + case 'multi_select': + return { + type: 'select', + element_id: elementId, + value: typeof value === 'string' ? value : String(value ?? ''), + } + case 'range': + return { + type: 'type', + element_id: elementId, + text: typeof value === 'number' ? String(value) : String(value ?? ''), + } + case 'key': + return { + type: 'key', + element_id: elementId, + key: typeof value === 'string' ? value : String(value ?? ''), + modifiers, + } + case 'scroll_to': + return { type: 'scroll_to', element_id: elementId } + default: + return { type: 'click', element_id: elementId } + } +} + // ────────────────────────────────────────────────────────────────────────── // Meta // ────────────────────────────────────────────────────────────────────────── @@ -331,6 +538,12 @@ function listSessions(state: DispatcherState): DispatchResult { pending: s.document.pending.size, created_at: s.createdAt.toISOString(), })), + desktops: Array.from(state.desktops.values()).map((s) => ({ + desktop_id: s.id, + backend: s.backend.name, + has_snapshot: s.lastBinding !== undefined, + created_at: s.createdAt.toISOString(), + })), }, null, 2, @@ -349,10 +562,14 @@ export async function disposeAll(state: DispatcherState): Promise { for (const session of state.pdfs.values()) { closers.push(session.document.close().catch(() => {})) } + for (const session of state.desktops.values()) { + if (session.backend.close) closers.push(session.backend.close().catch(() => {})) + } await Promise.allSettled(closers) state.browsers.clear() state.pages.clear() state.pdfs.clear() + state.desktops.clear() } // ────────────────────────────────────────────────────────────────────────── @@ -379,6 +596,12 @@ function requirePdf(state: DispatcherState, docId: string): PdfDocument { return session.document } +function requireDesktop(state: DispatcherState, desktopId: string): DesktopSession { + const session = state.desktops.get(desktopId) + if (!session) throw new Error(`Unknown desktop_id: ${desktopId}`) + return session +} + /** * Load PDF bytes from either a file path OR a data URL. Data URLs are * useful for clients that have the PDF in memory and don't want to write diff --git a/src/mcp/tool-defs.ts b/src/mcp/tool-defs.ts index 3770ab2..463b6f2 100644 --- a/src/mcp/tool-defs.ts +++ b/src/mcp/tool-defs.ts @@ -275,6 +275,149 @@ const PDF_TOOLS: McpToolDef[] = [ }, ] +// ────────────────────────────────────────────────────────────────────────── +// Desktop application tools (v0.4) +// ────────────────────────────────────────────────────────────────────────── + +const DESKTOP_TOOLS: McpToolDef[] = [ + { + name: 'agentmark_desktop_open', + description: + 'Attach to a desktop accessibility-tree backend and return a ' + + 'desktop_id. Backends:\n' + + ' - "fixture" (default): pre-baked Excel + NowCerts trees, ' + + 'works on any OS — useful for testing without a real bridge ' + + 'installed.\n' + + ' - "windows_uia": connects to the Windows FlaUI sidecar ' + + 'process (requires the bridge running on the same machine).\n' + + ' - "macos_axapi": connects to the macOS AXAPI sidecar ' + + '(requires the bridge + Accessibility permission granted).\n' + + '\nThe session lives for the duration of the MCP connection ' + + 'unless explicitly closed.', + inputSchema: { + type: 'object', + properties: { + backend: { + type: 'string', + enum: ['fixture', 'windows_uia', 'macos_axapi'], + description: 'Which backend to use. Default: "fixture".', + }, + bridge_url: { + type: 'string', + description: + 'Override the bridge WebSocket URL (for non-fixture ' + + 'backends). Default: ws://127.0.0.1:9325/agentmark-bridge.', + }, + }, + }, + }, + { + name: 'agentmark_desktop_close', + description: 'Close a desktop session and release the bridge connection.', + inputSchema: { + type: 'object', + properties: { + desktop_id: { type: 'string' }, + }, + required: ['desktop_id'], + }, + }, + { + name: 'agentmark_desktop_snapshot', + description: + 'Capture an AgentMark snapshot of a desktop window. If `target` ' + + 'is omitted, the currently focused window is captured. The ' + + 'result is cached on the session so subsequent ' + + 'agentmark_desktop_execute calls can resolve action IDs back ' + + 'to native accessibility element IDs.', + inputSchema: { + type: 'object', + properties: { + desktop_id: { type: 'string' }, + target: { + type: 'object', + description: + 'Which window to capture. Provide any combination; ' + + 'the backend resolves whichever it can. Omit to ' + + 'capture the focused window.', + properties: { + process_name: { type: 'string' }, + process_id: { type: 'number' }, + window_title: { type: 'string' }, + window_id: { + type: 'string', + description: + 'Backend-defined opaque handle returned by a ' + + 'previous snapshot. Most precise targeting.', + }, + }, + }, + max_depth: { + type: 'number', + description: + 'Maximum accessibility-tree depth to traverse. ' + + 'Default: 12.', + }, + include_hidden: { + type: 'boolean', + description: + 'Include off-screen / invisible elements. Default: false.', + }, + timeout_ms: { + type: 'number', + description: 'Per-capture timeout in milliseconds. Default: 5000.', + }, + }, + required: ['desktop_id'], + }, + }, + { + name: 'agentmark_desktop_execute', + description: + 'Execute an action against the most recent snapshot of a ' + + 'desktop session. `action_id` is one of the keys from the ' + + 'snapshot\'s `actions` map (e.g. `act_btn_save`). The MCP ' + + 'server resolves it to the underlying element via the ' + + 'ActionBinding captured at snapshot time.\n' + + '\nValue semantics by action type:\n' + + ' - click / focus / scroll_to: omit `value`\n' + + ' - type: string (text to enter)\n' + + ' - check: boolean (target state)\n' + + ' - select: string (option value or label)\n' + + ' - key: string (key name, e.g. "Enter", "F5") + optional ' + + '`modifiers` array', + inputSchema: { + type: 'object', + properties: { + desktop_id: { type: 'string' }, + action_id: { type: 'string' }, + value: { + description: + 'Value for input-style actions. Type depends on the ' + + 'action: string, boolean, etc.', + }, + modifiers: { + type: 'array', + items: { + type: 'string', + enum: ['ctrl', 'alt', 'shift', 'meta', 'win'], + }, + description: + 'Key modifiers for `key` actions (or holding modifiers ' + + 'during a click).', + }, + clear_first: { + type: 'boolean', + description: + 'For `type` actions, clear the existing value before ' + + 'typing. Default: false.', + }, + }, + required: ['desktop_id', 'action_id'], + }, + }, +] + // ────────────────────────────────────────────────────────────────────────── // Session inspection // ────────────────────────────────────────────────────────────────────────── @@ -283,8 +426,9 @@ const META_TOOLS: McpToolDef[] = [ { name: 'agentmark_list_sessions', description: - 'List all currently open browsers, pages, and PDF documents ' - + 'with their IDs. Useful for debugging or recovering a stuck session.', + 'List all currently open browsers, pages, PDF documents, and ' + + 'desktop sessions with their IDs. Useful for debugging or ' + + 'recovering a stuck session.', inputSchema: { type: 'object', properties: {}, @@ -292,4 +436,4 @@ const META_TOOLS: McpToolDef[] = [ }, ] -export const ALL_TOOLS: McpToolDef[] = [...WEB_TOOLS, ...PDF_TOOLS, ...META_TOOLS] +export const ALL_TOOLS: McpToolDef[] = [...WEB_TOOLS, ...PDF_TOOLS, ...DESKTOP_TOOLS, ...META_TOOLS] diff --git a/src/mcp/types.ts b/src/mcp/types.ts index a39b330..af18dc9 100644 --- a/src/mcp/types.ts +++ b/src/mcp/types.ts @@ -4,7 +4,15 @@ * MCP client can drive multiple parallel agents from one connection. */ -import type { Browser, Page, PdfDocument } from '../index' +import type { + ActionBinding, + Browser, + DesktopCapture, + DesktopCaptureBackend, + DesktopTarget, + Page, + PdfDocument, +} from '../index' export interface BrowserSession { id: string @@ -19,12 +27,31 @@ export interface PdfSession { createdAt: Date } +export interface DesktopSession { + id: string + backend: DesktopCaptureBackend + /** Last `target` passed to capture(); reused by execute() when the + * client doesn't re-specify it. */ + lastTarget?: DesktopTarget + /** Result of the most recent capture — used so execute() knows which + * process to drive and which element_count to report. */ + lastCapture?: DesktopCapture + /** Binding map from the most recent convertDesktop() call — + * resolves actionId → element_id for execute(). */ + lastBinding?: ActionBinding + /** Action types keyed by actionId from the most recent snapshot. + * Used by execute() to translate the client's `value` argument + * into the right ExecuteDesktopAction variant. */ + lastActionTypes?: Map + createdAt: Date +} + /** * Generates a short unique ID. Uses crypto.randomUUID() if available, * otherwise a Math.random fallback. The IDs are opaque to clients — * they're returned by `_open` tools and passed back on subsequent calls. */ -export function generateSessionId(prefix: 'br' | 'pdf' | 'pg'): string { +export function generateSessionId(prefix: 'br' | 'pdf' | 'pg' | 'dt'): string { const r = typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function' ? crypto.randomUUID().replace(/-/g, '').slice(0, 12) diff --git a/test/mcp/desktop-dispatcher.test.ts b/test/mcp/desktop-dispatcher.test.ts new file mode 100644 index 0000000..e1cefc3 --- /dev/null +++ b/test/mcp/desktop-dispatcher.test.ts @@ -0,0 +1,170 @@ +/** + * MCP dispatcher tests for the desktop tools (v0.4). + * + * Drives `dispatch()` directly with the agentmark_desktop_* tool names. + * Uses the FixtureBackend, so these tests are deterministic and run on + * any OS (no real bridge required). + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { + dispatch, + createDispatcherState, + disposeAll, + type DispatcherState, +} from '../../src/mcp/dispatcher' +import { ALL_TOOLS } from '../../src/mcp/tool-defs' +import { parseSnapshot } from '../../src/serializers/yaml-frontmatter' + +let state: DispatcherState + +beforeEach(() => { + state = createDispatcherState() +}) + +afterEach(async () => { + await disposeAll(state) +}) + +describe('MCP — desktop tools', () => { + it('registers all four desktop tools in ALL_TOOLS', () => { + const names = ALL_TOOLS.map(t => t.name) + expect(names).toContain('agentmark_desktop_open') + expect(names).toContain('agentmark_desktop_close') + expect(names).toContain('agentmark_desktop_snapshot') + expect(names).toContain('agentmark_desktop_execute') + }) + + it('agentmark_desktop_open with no args defaults to the fixture backend', async () => { + const result = await dispatch(state, 'agentmark_desktop_open', {}) + expect(result.isError).toBeFalsy() + const body = JSON.parse(result.text) + expect(body.desktop_id).toMatch(/^dt_/) + expect(body.backend).toBe('fixture') + }) + + it('agentmark_desktop_open with backend=fixture returns a desktop_id', async () => { + const result = await dispatch(state, 'agentmark_desktop_open', { backend: 'fixture' }) + expect(result.isError).toBeFalsy() + const body = JSON.parse(result.text) + expect(state.desktops.has(body.desktop_id)).toBe(true) + }) + + it('agentmark_desktop_open rejects unknown backend names', async () => { + const result = await dispatch(state, 'agentmark_desktop_open', { backend: 'mystery' }) + expect(result.isError).toBe(true) + expect(result.text).toContain('Unknown desktop backend') + }) + + it('agentmark_desktop_open with windows_uia returns "not yet bundled" (until the bridge ships)', async () => { + const result = await dispatch(state, 'agentmark_desktop_open', { backend: 'windows_uia' }) + expect(result.isError).toBe(true) + expect(result.text).toContain('not yet bundled') + }) + + it('agentmark_desktop_snapshot returns a valid v0.4 desktop snapshot', async () => { + const open = await dispatch(state, 'agentmark_desktop_open', {}) + const { desktop_id } = JSON.parse(open.text) + + const result = await dispatch(state, 'agentmark_desktop_snapshot', { desktop_id }) + expect(result.isError).toBeFalsy() + + const snap = parseSnapshot(result.text) + expect(snap.agentmark).toBe('0.4') + expect(snap.kind).toBe('desktop') + expect(snap.desktop_meta?.platform).toBe('windows') + expect(snap.desktop_meta?.a11y_backend).toBe('fixture') + }) + + it('snapshot can target a specific preset via target.window_id', async () => { + const open = await dispatch(state, 'agentmark_desktop_open', {}) + const { desktop_id } = JSON.parse(open.text) + + const result = await dispatch(state, 'agentmark_desktop_snapshot', { + desktop_id, + target: { window_id: 'nowcerts_customer' }, + }) + const snap = parseSnapshot(result.text) + expect(snap.title).toContain('NowCerts') + expect(snap.actions?.act_in_company?.label).toBe('Company Name') + }) + + it('agentmark_desktop_execute drives a type action and the next snapshot reflects it', async () => { + const open = await dispatch(state, 'agentmark_desktop_open', {}) + const { desktop_id } = JSON.parse(open.text) + + // First snapshot — caches the binding. + await dispatch(state, 'agentmark_desktop_snapshot', { + desktop_id, + target: { window_id: 'nowcerts_customer' }, + }) + + const exec = await dispatch(state, 'agentmark_desktop_execute', { + desktop_id, + action_id: 'act_in_company', + value: 'Beta Industries', + }) + expect(exec.isError).toBeFalsy() + const body = JSON.parse(exec.text) + expect(body.action_type).toBe('type') + expect(body.element_id).toBe('in_company') + expect(body.ok).toBe(true) + expect(body.new_value).toBe('Beta Industries') + + // Re-snapshot the same target → value should reflect the typed text. + const after = await dispatch(state, 'agentmark_desktop_snapshot', { + desktop_id, + target: { window_id: 'nowcerts_customer' }, + }) + const snap = parseSnapshot(after.text) + expect(snap.actions?.act_in_company?.value).toBe('Beta Industries') + }) + + it('agentmark_desktop_execute rejects unknown action IDs', async () => { + const open = await dispatch(state, 'agentmark_desktop_open', {}) + const { desktop_id } = JSON.parse(open.text) + + await dispatch(state, 'agentmark_desktop_snapshot', { desktop_id }) + + const exec = await dispatch(state, 'agentmark_desktop_execute', { + desktop_id, + action_id: 'act_bogus_id', + }) + expect(exec.isError).toBe(true) + expect(exec.text).toContain('Unknown action_id') + }) + + it('agentmark_desktop_execute errors when no snapshot has been captured yet', async () => { + const open = await dispatch(state, 'agentmark_desktop_open', {}) + const { desktop_id } = JSON.parse(open.text) + + const exec = await dispatch(state, 'agentmark_desktop_execute', { + desktop_id, + action_id: 'act_anything', + }) + expect(exec.isError).toBe(true) + expect(exec.text).toContain('No cached snapshot') + }) + + it('agentmark_desktop_close removes the session', async () => { + const open = await dispatch(state, 'agentmark_desktop_open', {}) + const { desktop_id } = JSON.parse(open.text) + expect(state.desktops.has(desktop_id)).toBe(true) + + const close = await dispatch(state, 'agentmark_desktop_close', { desktop_id }) + expect(close.isError).toBeFalsy() + expect(state.desktops.has(desktop_id)).toBe(false) + }) + + it('agentmark_list_sessions includes desktop sessions', async () => { + const open = await dispatch(state, 'agentmark_desktop_open', {}) + const { desktop_id } = JSON.parse(open.text) + + const list = await dispatch(state, 'agentmark_list_sessions', {}) + const body = JSON.parse(list.text) + expect(body.desktops).toHaveLength(1) + expect(body.desktops[0].desktop_id).toBe(desktop_id) + expect(body.desktops[0].backend).toBe('fixture') + expect(body.desktops[0].has_snapshot).toBe(false) + }) +}) From 83f7116826675f8dff30db024f7efb38818fdd26 Mon Sep 17 00:00:00 2001 From: rrader26 Date: Mon, 11 May 2026 12:23:07 -0400 Subject: [PATCH 3/3] feat(bridge-windows): Phase 0e1 scaffold + stdio JSON-RPC smoke test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the Windows UIA bridge as a .NET 8 console project at apps/agent-runner/bridges/windows/. Stdio JSON-RPC 2.0 server; one message per line; stdout strictly framed JSON, stderr for diagnostics. This first commit ships the protocol substrate and two no-UIA methods so we can validate the cross-language pipeline before adding the heavy COM-STA capture/execute logic in Phase 0e2/0e3: - ping -> { pong, version, arch, processId } - capabilities -> supported methods + UIA provider info Built + smoke-tested over SSH from the Mac against a Parallels-shared checkout. ARM64 native build (no x64 emulation overhead). Round-trip verified: pipe two requests -> close stdin -> get two responses -> bridge exits 0. Implementation notes: - Sync stdin reads (Console.In.ReadLine). Async ReadLineAsync has a known issue on Windows pipes where it does not return null on EOF, hanging the process even after the parent closes the pipe. Sync reads handle EOF correctly. UIA is COM-STA anyway, so async would not buy us anything for capture/execute. - UTF8Encoding(false) on both pipes to avoid BOM corruption of the JSON-RPC framing. - All diagnostic output to stderr, never stdout. - Smoke test uses .NET Process API rather than PowerShell pipes — PowerShell's pipe-to-native-exe does not propagate EOF, which would hang the bridge on smoke-test runs. .gitignore extended for bin/, obj/, *.user, .vs/, .idea/. Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 17 ++ .../windows/AgentMark.Bridge.Windows.csproj | 32 +++ apps/agent-runner/bridges/windows/Program.cs | 220 ++++++++++++++++++ apps/agent-runner/bridges/windows/README.md | 85 +++++++ .../bridges/windows/scripts/smoke-test.ps1 | 101 ++++++++ 5 files changed, 455 insertions(+) create mode 100644 apps/agent-runner/bridges/windows/AgentMark.Bridge.Windows.csproj create mode 100644 apps/agent-runner/bridges/windows/Program.cs create mode 100644 apps/agent-runner/bridges/windows/README.md create mode 100644 apps/agent-runner/bridges/windows/scripts/smoke-test.ps1 diff --git a/.gitignore b/.gitignore index e04537d..cbf0247 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,20 @@ dist/ # Tesseract.js downloads language data into cwd by default *.traineddata + +# .NET / MSBuild output (used by apps/agent-runner/bridges/windows) +bin/ +obj/ +*.suo +*.user +*.userprefs + +# JetBrains +.idea/ +*.iml + +# Visual Studio +.vs/ + +# Local dev junk +*.swp diff --git a/apps/agent-runner/bridges/windows/AgentMark.Bridge.Windows.csproj b/apps/agent-runner/bridges/windows/AgentMark.Bridge.Windows.csproj new file mode 100644 index 0000000..aa23fa4 --- /dev/null +++ b/apps/agent-runner/bridges/windows/AgentMark.Bridge.Windows.csproj @@ -0,0 +1,32 @@ + + + + Exe + net8.0-windows + AgentMark.Bridge.Windows + agentmark-bridge-windows + enable + enable + 12.0 + true + + $(MSBuildThisFileDirectory)bin\ + $(MSBuildThisFileDirectory)obj\ + + + + + AgentMark Windows UIA bridge — accessibility-tree capture and action execution for kind:'desktop' snapshots. + ThinkFleet + ThinkFleet + AgentMark Desktop + + + + + + + + diff --git a/apps/agent-runner/bridges/windows/Program.cs b/apps/agent-runner/bridges/windows/Program.cs new file mode 100644 index 0000000..0f1fa15 --- /dev/null +++ b/apps/agent-runner/bridges/windows/Program.cs @@ -0,0 +1,220 @@ +// AgentMark Windows UIA Bridge +// ---------------------------- +// Stdio JSON-RPC 2.0 server. Parent process (typically the Node-side +// `WindowsUiaBackend` running inside the AgentMark MCP server, or a +// ThinkFleet SaaS agent) launches this exe and talks to it over +// stdin/stdout. One line of JSON per message. +// +// Protocol: +// Request : { "jsonrpc": "2.0", "id": , "method": "", "params": {...} } +// Response: { "jsonrpc": "2.0", "id": , "result": {...} } on success +// { "jsonrpc": "2.0", "id": , "error": { "code": N, "message": "..." } } on failure +// +// All diagnostic output goes to stderr — never stdout — so the framing stays clean. +// +// Methods (current set; more land as Phase 0e progresses): +// ping — returns { pong: true, version: "...", arch: "arm64|x64" } +// capabilities— returns { methods: [...], uia_version: "..." } +// +// Phase 0e2/0e3 will add: +// capture — walk a window's UIA tree, return DesktopCapture JSON +// execute — drive an action by element_id (click/type/select/etc.) +// list_windows— enumerate top-level windows visible to the bridge + +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AgentMark.Bridge.Windows; + +internal static class Program +{ + // JSON serialization options — camelCase to match the AgentMark + // wire format on the Node side. + internal static readonly JsonSerializerOptions JsonOpts = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + public static int Main(string[] args) + { + // Force UTF-8 on both pipes -- Windows defaults can mangle non-ASCII. + // Use UTF8Encoding(false) to avoid emitting a BOM on stdout, which + // would break JSON-RPC framing on the parent side. + Console.InputEncoding = new UTF8Encoding(false); + Console.OutputEncoding = new UTF8Encoding(false); + Console.Error.WriteLine($"[bridge] agentmark-bridge-windows starting (pid={Environment.ProcessId}, arch={System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture})"); + + // Synchronous read loop. Async stdin reads on Windows pipes have a + // known issue where ReadLineAsync() doesn't always return null on + // EOF, leaving the process hung even after the parent closes the + // pipe. ReadLine() handles EOF correctly. UIA calls are themselves + // blocking COM-STA invocations so async doesn't buy us anything. + var dispatcher = new RpcDispatcher(); + + string? line; + while ((line = Console.In.ReadLine()) != null) + { + line = line.Trim(); + if (line.Length == 0) continue; + + JsonElement reqId = default; + try + { + using var doc = JsonDocument.Parse(line); + var root = doc.RootElement; + reqId = root.TryGetProperty("id", out var idEl) ? idEl.Clone() : default; + + var method = root.GetProperty("method").GetString() + ?? throw new RpcException(RpcError.InvalidRequest, "method is required"); + var paramsEl = root.TryGetProperty("params", out var p) ? p : default; + + var result = dispatcher.Dispatch(method, paramsEl); + Console.Out.WriteLine(EncodeSuccess(reqId, result)); + } + catch (RpcException rex) + { + Console.Out.WriteLine(EncodeError(reqId, rex.Error.Code, rex.Message)); + } + catch (JsonException jex) + { + Console.Out.WriteLine(EncodeError(reqId, RpcError.ParseError.Code, $"JSON parse error: {jex.Message}")); + } + catch (Exception ex) + { + Console.Error.WriteLine($"[bridge] unhandled: {ex}"); + Console.Out.WriteLine(EncodeError(reqId, RpcError.InternalError.Code, ex.Message)); + } + + Console.Out.Flush(); + } + + Console.Error.WriteLine("[bridge] stdin closed; exiting"); + return 0; + } + + private static string EncodeSuccess(JsonElement id, object? result) + { + var env = new RpcResponseSuccess + { + Id = id.ValueKind == JsonValueKind.Undefined ? null : id, + Result = result, + }; + return JsonSerializer.Serialize(env, JsonOpts); + } + + private static string EncodeError(JsonElement id, int code, string message) + { + var env = new RpcResponseError + { + Id = id.ValueKind == JsonValueKind.Undefined ? null : id, + Error = new RpcErrorBody { Code = code, Message = message }, + }; + return JsonSerializer.Serialize(env, JsonOpts); + } +} + +// ────────────────────────────────────────────────────────────────────── +// Wire format +// ────────────────────────────────────────────────────────────────────── + +internal sealed class RpcResponseSuccess +{ + [JsonPropertyName("jsonrpc")] + public string Jsonrpc => "2.0"; + + [JsonPropertyName("id")] + public JsonElement? Id { get; init; } + + [JsonPropertyName("result")] + public object? Result { get; init; } +} + +internal sealed class RpcResponseError +{ + [JsonPropertyName("jsonrpc")] + public string Jsonrpc => "2.0"; + + [JsonPropertyName("id")] + public JsonElement? Id { get; init; } + + [JsonPropertyName("error")] + public RpcErrorBody Error { get; init; } = null!; +} + +internal sealed class RpcErrorBody +{ + [JsonPropertyName("code")] + public int Code { get; init; } + + [JsonPropertyName("message")] + public string Message { get; init; } = ""; +} + +// ────────────────────────────────────────────────────────────────────── +// Error catalog +// ────────────────────────────────────────────────────────────────────── + +internal readonly record struct RpcError(int Code) +{ + public static readonly RpcError ParseError = new(-32700); + public static readonly RpcError InvalidRequest = new(-32600); + public static readonly RpcError MethodNotFound = new(-32601); + public static readonly RpcError InvalidParams = new(-32602); + public static readonly RpcError InternalError = new(-32603); + + // Bridge-specific (above -32000) + public static readonly RpcError WindowNotFound = new(-32010); + public static readonly RpcError ElementNotFound = new(-32011); + public static readonly RpcError UnsupportedPattern = new(-32012); + public static readonly RpcError ActionFailed = new(-32013); +} + +internal sealed class RpcException(RpcError error, string message) : Exception(message) +{ + public RpcError Error { get; } = error; +} + +// ────────────────────────────────────────────────────────────────────── +// Dispatcher +// ────────────────────────────────────────────────────────────────────── + +internal sealed class RpcDispatcher +{ + private static readonly string BridgeVersion = "0.4.0"; + + public object? Dispatch(string method, JsonElement @params) + { + return method switch + { + "ping" => HandlePing(), + "capabilities" => HandleCapabilities(), + _ => throw new RpcException( + RpcError.MethodNotFound, + $"Unknown method: {method}. Supported: ping, capabilities."), + }; + } + + private static object HandlePing() => new + { + pong = true, + version = BridgeVersion, + arch = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture.ToString().ToLowerInvariant(), + processId = Environment.ProcessId, + }; + + private static object HandleCapabilities() => new + { + bridge = "agentmark-bridge-windows", + version = BridgeVersion, + methods = new[] + { + "ping", + "capabilities", + // "capture" and "execute" land in Phase 0e2/0e3 + }, + uiaProvider = "FlaUI.UIA3", + platform = "windows", + }; +} diff --git a/apps/agent-runner/bridges/windows/README.md b/apps/agent-runner/bridges/windows/README.md new file mode 100644 index 0000000..7a47260 --- /dev/null +++ b/apps/agent-runner/bridges/windows/README.md @@ -0,0 +1,85 @@ +# AgentMark Windows UIA Bridge + +The Windows-side sidecar process for AgentMark Desktop. Walks the OS +accessibility tree via UIA (using [FlaUI](https://github.com/FlaUI/FlaUI), +MIT) and exposes capture/execute operations the Node-side +`WindowsUiaBackend` consumes through stdio JSON-RPC. + +This is one implementation of the `DesktopCaptureBackend` interface +defined in `@thinkfleet/agentmark` v0.4. macOS (AXAPI) and Linux (AT-SPI) +bridges follow the same protocol. + +## Build + +Requires the .NET 8 SDK. From this directory: + +```powershell +dotnet build +``` + +Output exe: `bin\Debug\net8.0-windows\agentmark-bridge-windows.exe`. + +## Run + smoke test + +The bridge speaks stdio JSON-RPC 2.0. One JSON message per line. + +```powershell +.\bin\Debug\net8.0-windows\agentmark-bridge-windows.exe +``` + +It blocks waiting on stdin. Paste a `ping`: + +``` +{"jsonrpc":"2.0","id":1,"method":"ping"} +``` + +Expected response (single line, here pretty-printed): + +```json +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "pong": true, + "version": "0.4.0", + "arch": "arm64", + "processId": 12345 + } +} +``` + +Ctrl-Z then Enter closes stdin and the bridge exits cleanly. + +## Protocol + +All responses are JSON-RPC 2.0. Diagnostic output goes to stderr only — +stdout is reserved for framed JSON messages. + +| Method | Status | Description | +|---|---|---| +| `ping` | Phase 0e1 ✅ | Liveness check, returns pong + bridge version | +| `capabilities` | Phase 0e1 ✅ | Lists supported methods and UIA provider info | +| `list_windows` | Phase 0e2 (planned) | Enumerate top-level windows visible to UIA | +| `capture` | Phase 0e2 (planned) | Walk a window's UIA tree → `DesktopCapture` JSON | +| `execute` | Phase 0e3 (planned) | Drive an action by element_id (click/type/select/etc.) | + +## Architecture notes + +- **Single-threaded for now.** UIA is COM-STA; once `capture` / `execute` + land we marshal those calls onto a dedicated STA thread. +- **UTF-8 forced on both pipes.** Windows defaults will mangle non-ASCII + in window titles / cell values otherwise. +- **All stderr goes to the parent process's stderr.** Useful for log + aggregation in the AgentMark MCP server when this bridge is spawned + as a subprocess. + +## Security posture + +This bridge runs as the logged-in user (never SYSTEM/admin). It can +only see/drive windows that user can already see/drive — no privilege +escalation. Stdio mode has zero network surface; only the parent +process that spawned this exe can write to its stdin. + +A future `--transport=ws` mode (Phase 0f) will add a localhost-only +WebSocket transport with auth-token gating, origin-header rejection, +and process-identity verification for multi-consumer scenarios. diff --git a/apps/agent-runner/bridges/windows/scripts/smoke-test.ps1 b/apps/agent-runner/bridges/windows/scripts/smoke-test.ps1 new file mode 100644 index 0000000..d8c67f4 --- /dev/null +++ b/apps/agent-runner/bridges/windows/scripts/smoke-test.ps1 @@ -0,0 +1,101 @@ +# Smoke test for the agentmark-bridge-windows binary. +# +# Spawns the bridge as a child process via .NET's Process API (NOT +# PowerShell's pipe), writes a few JSON-RPC requests, then closes +# stdin explicitly -- that is the only reliable way to signal EOF to +# the bridge so it exits cleanly. PowerShell's `... | & exe` pipe +# does NOT close stdin on the native exe; that's why the naive +# version hangs. + +$ErrorActionPreference = 'Stop' + +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$exe = Join-Path $scriptDir '..\bin\Debug\net8.0-windows\agentmark-bridge-windows.exe' + +if (-not (Test-Path $exe)) { + Write-Error "Bridge exe not built -- expected at $exe. Run 'dotnet build' first." + exit 1 +} + +Write-Host "Smoke testing $exe" + +$psi = New-Object System.Diagnostics.ProcessStartInfo +$psi.FileName = $exe +$psi.UseShellExecute = $false +$psi.RedirectStandardInput = $true +$psi.RedirectStandardOutput = $true +$psi.RedirectStandardError = $true +# StandardInputEncoding/OutputEncoding only exist on .NET Core/5+; +# Windows PowerShell 5.1 runs on .NET Framework which lacks them. +# Our JSON payloads are pure ASCII so the system codepage default works. +# The bridge process itself forces UTF-8 on its own streams (Program.cs). + +$proc = [System.Diagnostics.Process]::Start($psi) + +$requests = @( + '{"jsonrpc":"2.0","id":1,"method":"ping"}', + '{"jsonrpc":"2.0","id":2,"method":"capabilities"}' +) + +foreach ($req in $requests) { + $proc.StandardInput.WriteLine($req) +} +$proc.StandardInput.Close() + +# 10-second cap to keep the test sane. +if (-not $proc.WaitForExit(10000)) { + $proc.Kill() + Write-Error "Bridge did not exit within 10 seconds." + exit 1 +} + +$stdout = $proc.StandardOutput.ReadToEnd() +$stderr = $proc.StandardError.ReadToEnd() + +if ($stderr) { + Write-Host "--- bridge stderr ---" -ForegroundColor DarkGray + Write-Host $stderr -ForegroundColor DarkGray +} + +$responseLines = $stdout -split "`r?`n" | Where-Object { $_.Trim().Length -gt 0 } +Write-Host "Received $($responseLines.Count) response line(s)." + +if ($responseLines.Count -lt 2) { + Write-Error "Expected 2 responses; got $($responseLines.Count)." + Write-Host "stdout was:" + Write-Host $stdout + exit 1 +} + +foreach ($line in $responseLines) { + Write-Host " -> $line" +} + +$pingResp = $responseLines[0] | ConvertFrom-Json +$capResp = $responseLines[1] | ConvertFrom-Json + +if ($pingResp.result.pong -ne $true) { + Write-Error "ping: expected result.pong=true, got: $($pingResp | ConvertTo-Json -Compress)" + exit 1 +} +if ($pingResp.id -ne 1) { + Write-Error "ping: id mismatch (expected 1, got $($pingResp.id))" + exit 1 +} + +if ($capResp.result.bridge -ne 'agentmark-bridge-windows') { + Write-Error "capabilities: expected bridge=agentmark-bridge-windows, got: $($capResp | ConvertTo-Json -Compress)" + exit 1 +} +if ($capResp.id -ne 2) { + Write-Error "capabilities: id mismatch (expected 2, got $($capResp.id))" + exit 1 +} + +Write-Host "" +Write-Host "SMOKE TEST PASSED" -ForegroundColor Green +Write-Host " bridge version : $($pingResp.result.version)" +Write-Host " architecture : $($pingResp.result.arch)" +Write-Host " process id : $($pingResp.result.processId)" +Write-Host " methods : $($capResp.result.methods -join ', ')" +Write-Host " exit code : $($proc.ExitCode)"