From aa3dcc2357a71c67b61c410c57c9eefa84dc46fd Mon Sep 17 00:00:00 2001 From: rrader26 Date: Mon, 11 May 2026 16:30:16 -0400 Subject: [PATCH 1/3] feat(bridge-macos): Phase 0e1' scaffold + stdio JSON-RPC smoke test Adds the macOS-side sidecar process for AgentMark Desktop, mirroring the Windows bridge architecture exactly. Same stdio JSON-RPC 2.0 protocol; same DesktopCaptureBackend contract on the Node side. Uses Apple's Accessibility framework (AXAPI) as the capture / execute substrate when Phase 0e2'/0e3' land. This first commit ships the protocol substrate and two no-AXAPI methods so we can validate the cross-language pipeline before adding the AXAPI walking + action dispatch logic: - ping -> { pong, version, arch, processId } - capabilities -> supported methods + AXAPI provider info Built natively on Apple Silicon (arm64). Smoke test runs locally on the developer Mac in seconds. Layout (apps/agent-runner/bridges/macos/): Package.swift -- SPM manifest (macOS 13+) Sources/AgentMarkBridgeMacos/ main.swift -- entry + stdin read loop + BOM stripping + dispatch JsonRpc.swift -- request/response types + JSON-RPC 2.0 envelope encoder/decoder + error code catalog Dispatcher.swift -- method routing (mirrors the C# bridge's RpcDispatcher class) scripts/smoke-test.sh -- shell harness driving the binary via stdin; grep- based assertions, no jq dependency README.md -- build + protocol docs The protocol -- request envelope, error codes (parseError -32700 .. internalError -32603 plus bridge-specific 32010+), stderr discipline, BOM stripping -- is byte-identical to the Windows bridge. This is deliberate: the Node-side `MacosAxapiBackend` (Phase 0f') will be a near-exact mirror of `WindowsUiaBackend`, sharing 90% of the code through the abstract DesktopCaptureBackend interface. .gitignore extended for Swift Package Manager output (.build/, .swiftpm/, Packages/) alongside the existing .NET ignores. Why now: Windows-side debugging (Claude Desktop MCP config) is blocked on the user; macOS work unblocks local end-to-end validation while that resolves. After Phase 0e2'/0e3' and Phase 0f' ship, the user can demo AgentMark Desktop driving Pages / Excel-for-Mac / Numbers from Claude Desktop on their development machine -- no Windows VM dependency for the demo. Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 7 + apps/agent-runner/bridges/macos/Package.swift | 29 +++ apps/agent-runner/bridges/macos/README.md | 96 +++++++++ .../AgentMarkBridgeMacos/Dispatcher.swift | 47 +++++ .../AgentMarkBridgeMacos/JsonRpc.swift | 189 ++++++++++++++++++ .../Sources/AgentMarkBridgeMacos/main.swift | 91 +++++++++ .../bridges/macos/scripts/smoke-test.sh | 64 ++++++ 7 files changed, 523 insertions(+) create mode 100644 apps/agent-runner/bridges/macos/Package.swift create mode 100644 apps/agent-runner/bridges/macos/README.md create mode 100644 apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Dispatcher.swift create mode 100644 apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/JsonRpc.swift create mode 100644 apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/main.swift create mode 100755 apps/agent-runner/bridges/macos/scripts/smoke-test.sh diff --git a/.gitignore b/.gitignore index cbf0247..29b5def 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,13 @@ obj/ *.user *.userprefs +# Swift Package Manager output (used by apps/agent-runner/bridges/macos) +.build/ +.swiftpm/ +Packages/ +*.xcodeproj +DerivedData/ + # JetBrains .idea/ *.iml diff --git a/apps/agent-runner/bridges/macos/Package.swift b/apps/agent-runner/bridges/macos/Package.swift new file mode 100644 index 0000000..8760d86 --- /dev/null +++ b/apps/agent-runner/bridges/macos/Package.swift @@ -0,0 +1,29 @@ +// swift-tools-version:5.9 +// +// AgentMark macOS bridge. Mirrors the Windows bridge architecture — +// stdio JSON-RPC 2.0 server, Node-side `MacosAxapiBackend` spawns this +// and proxies capture/execute calls. Phase 0e'1 here ships ping + +// capabilities; AXAPI capture/execute land in 0e'2/0e'3. + +import PackageDescription + +let package = Package( + name: "AgentMarkBridgeMacos", + platforms: [ + // AXAPI requires macOS; setting a recent baseline keeps the + // Accessibility APIs we'll need (kAXMainAttribute etc.) safe. + .macOS(.v13), + ], + products: [ + .executable( + name: "agentmark-bridge-macos", + targets: ["AgentMarkBridgeMacos"] + ), + ], + targets: [ + .executableTarget( + name: "AgentMarkBridgeMacos", + path: "Sources/AgentMarkBridgeMacos" + ), + ] +) diff --git a/apps/agent-runner/bridges/macos/README.md b/apps/agent-runner/bridges/macos/README.md new file mode 100644 index 0000000..6ce1d1f --- /dev/null +++ b/apps/agent-runner/bridges/macos/README.md @@ -0,0 +1,96 @@ +# AgentMark macOS AXAPI Bridge + +The macOS-side sidecar process for AgentMark Desktop. Mirrors the +Windows UIA bridge — same stdio JSON-RPC 2.0 protocol, same +`DesktopCaptureBackend` contract on the Node side. Uses the system +Accessibility framework (AXAPI) to walk and drive native macOS app UIs. + +## Build + +Requires the Swift toolchain (ships with Xcode Command Line Tools, or +`xcode-select --install`). From this directory: + +```bash +swift build +``` + +Output binary: `.build/debug/agentmark-bridge-macos`. + +For a release build with optimisations: + +```bash +swift build -c release +# Binary at .build/release/agentmark-bridge-macos +``` + +## Run + smoke test + +The bridge speaks stdio JSON-RPC 2.0. One JSON message per line. + +```bash +./scripts/smoke-test.sh +``` + +That pipes two requests (`ping` + `capabilities`) into the binary, +validates the responses, prints the bridge's stderr diagnostics. + +For interactive testing: + +```bash +.build/debug/agentmark-bridge-macos +``` + +It blocks on stdin. Paste: + +``` +{"jsonrpc":"2.0","id":1,"method":"ping"} +``` + +Expected (single line): + +```json +{"jsonrpc":"2.0","id":1,"result":{"pong":true,"version":"0.4.0","arch":"arm64","processId":12345}} +``` + +Ctrl-D closes stdin and the bridge exits cleanly. + +## Protocol + +All responses are JSON-RPC 2.0. Diagnostic output goes to stderr only — +stdout is reserved for framed JSON messages. + +| Method | Status | Description | +|---|---|---| +| `ping` | ✅ this phase | Liveness check, returns pong + bridge version | +| `capabilities` | ✅ this phase | Lists supported methods + AXAPI provider info | +| `list_windows` | planned | Enumerate top-level AXAPI windows via NSWorkspace + AXUIElement | +| `capture` | planned | Walk a window's AXAPI tree → `DesktopCapture` JSON | +| `execute` | planned | Drive an action by element_id (AXPress / AXSetValue / etc.) | + +The protocol — request/response envelopes, error codes, BOM-stripping — +is byte-identical to the Windows bridge. The Node-side +`MacosAxapiBackend` will be a near-exact mirror of `WindowsUiaBackend`. + +## Accessibility permission + +macOS guards AXAPI behind **System Settings → Privacy & Security → +Accessibility**. When the bridge starts walking trees (Phase 0e'2+), +the parent process (or this binary, if launched directly) must be +granted. The bridge will probe `AXIsProcessTrusted()` and surface a +clear `accessibilityNotGranted` JSON-RPC error if not. + +For developer machines: grant the terminal you launch from (Terminal / +iTerm / Warp). For production / Claude Desktop integration: Claude +Desktop itself must be granted (it's the parent process that spawns +this bridge). + +## Architecture + +- **Single-threaded.** AXAPI calls are not strictly main-thread-only but + CFRunLoop integration matters; for capture/execute we'll route through + a dedicated serial DispatchQueue. For ping/capabilities the entry-point + thread is fine. +- **UTF-8 only.** stdin / stdout treated as UTF-8 throughout; BOM stripped + defensively from the first stdin write. +- **stderr for diagnostics, stdout for framed JSON.** Same discipline as + the Windows bridge so log aggregation works in the AgentMark MCP server. diff --git a/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Dispatcher.swift b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Dispatcher.swift new file mode 100644 index 0000000..5da8ffc --- /dev/null +++ b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Dispatcher.swift @@ -0,0 +1,47 @@ +// Method routing. New methods land here; the same name/shape as the +// Windows bridge so the Node-side `MacosAxapiBackend` (Phase 0f') can +// mirror `WindowsUiaBackend` exactly. + +import Foundation + +final class Dispatcher { + private let bridgeVersion: String + + init(bridgeVersion: String) { + self.bridgeVersion = bridgeVersion + } + + func dispatch(request: JsonRpcRequest) throws -> Any { + switch request.method { + case "ping": + return handlePing() + case "capabilities": + return handleCapabilities() + default: + throw RpcError( + code: .methodNotFound, + message: "Unknown method: \(request.method). Supported: ping, capabilities.", + requestId: request.id + ) + } + } + + private func handlePing() -> Any { + return [ + "pong": true, + "version": bridgeVersion, + "arch": currentArchString(), + "processId": ProcessInfo.processInfo.processIdentifier, + ] as [String: Any] + } + + private func handleCapabilities() -> Any { + return [ + "bridge": "agentmark-bridge-macos", + "version": bridgeVersion, + "methods": ["ping", "capabilities"], + "axapiProvider": "Accessibility (AXAPI)", + "platform": "macos", + ] as [String: Any] + } +} diff --git a/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/JsonRpc.swift b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/JsonRpc.swift new file mode 100644 index 0000000..3c6bded --- /dev/null +++ b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/JsonRpc.swift @@ -0,0 +1,189 @@ +// JSON-RPC 2.0 wire types. Identical envelope to the Windows bridge. + +import Foundation + +// MARK: - Request + +/// JSON-RPC 2.0 request. We accept any JSON value for `id` (number, +/// string, or null); store it raw and echo it back unmodified on the +/// response so different clients (number ids, uuid string ids) all work. +struct JsonRpcRequest { + let id: JsonRpcId + let method: String + let params: JsonValue? + + static func decode(jsonString: String) throws -> JsonRpcRequest { + guard let data = jsonString.data(using: .utf8) else { + throw RpcError(code: RpcErrorCode.parseError, message: "Input is not valid UTF-8") + } + guard let any = try? JSONSerialization.jsonObject(with: data, options: [.allowFragments]) else { + throw RpcError(code: RpcErrorCode.parseError, message: "Input is not valid JSON") + } + guard let obj = any as? [String: Any] else { + throw RpcError(code: RpcErrorCode.invalidRequest, message: "JSON-RPC request must be an object") + } + guard let method = obj["method"] as? String, !method.isEmpty else { + throw RpcError(code: RpcErrorCode.invalidRequest, message: "Missing `method`") + } + let id = JsonRpcId.fromAny(obj["id"]) + let params: JsonValue? = obj["params"].flatMap(JsonValue.fromAny) + return JsonRpcRequest(id: id, method: method, params: params) + } +} + +// MARK: - Response + +struct JsonRpcResponse { + let id: JsonRpcId + let kind: Kind + + enum Kind { + case success(result: Any) + case error(code: Int, message: String) + } + + static func success(id: JsonRpcId, result: Any) -> JsonRpcResponse { + JsonRpcResponse(id: id, kind: .success(result: result)) + } + + static func error(id: JsonRpcId, code: Int, message: String) -> JsonRpcResponse { + JsonRpcResponse(id: id, kind: .error(code: code, message: message)) + } + + func encode() throws -> String { + var dict: [String: Any] = [ + "jsonrpc": "2.0", + "id": id.toAny(), + ] + switch kind { + case .success(let result): + dict["result"] = result + case .error(let code, let message): + dict["error"] = [ + "code": code, + "message": message, + ] + } + let data = try JSONSerialization.data( + withJSONObject: dict, + options: [.withoutEscapingSlashes] + ) + guard let s = String(data: data, encoding: .utf8) else { + throw RpcError(code: RpcErrorCode.internalError, message: "Failed to serialise response as UTF-8") + } + return s + } +} + +// MARK: - JsonRpcId + +/// JSON-RPC ids can be number, string, or null. Hold the original +/// representation so we round-trip exactly. +enum JsonRpcId { + case number(Double) + case string(String) + case null + + static func fromAny(_ value: Any?) -> JsonRpcId { + if value == nil { return .null } + if let v = value as? NSNumber { + // NSNumber covers Int/Double/Bool from JSONSerialization + if CFGetTypeID(v) == CFBooleanGetTypeID() { return .null } + return .number(v.doubleValue) + } + if let s = value as? String { return .string(s) } + return .null + } + + func toAny() -> Any { + switch self { + case .number(let d): + // Emit ints as ints when possible so clients see id=1, not id=1.0. + if d.truncatingRemainder(dividingBy: 1) == 0 + && d >= Double(Int.min) && d <= Double(Int.max) { + return Int(d) + } + return d + case .string(let s): + return s + case .null: + return NSNull() + } + } +} + +// MARK: - JsonValue + +/// Lightweight wrapper for arbitrary JSON values plucked from the +/// request's `params` field. Most handlers project the few fields they +/// need out via the `dict`/`string(at:)` etc. helpers. +enum JsonValue { + case object([String: Any]) + case array([Any]) + case string(String) + case number(Double) + case bool(Bool) + case null + + static func fromAny(_ value: Any) -> JsonValue { + if let d = value as? [String: Any] { return .object(d) } + if let a = value as? [Any] { return .array(a) } + if let s = value as? String { return .string(s) } + if let n = value as? NSNumber { + if CFGetTypeID(n) == CFBooleanGetTypeID() { return .bool(n.boolValue) } + return .number(n.doubleValue) + } + return .null + } + + var dict: [String: Any]? { + if case .object(let d) = self { return d } + return nil + } + + func string(_ key: String) -> String? { + return (dict?[key] as? String) + } + + func int(_ key: String) -> Int? { + if let n = dict?[key] as? NSNumber { return n.intValue } + return nil + } + + func bool(_ key: String) -> Bool? { + if let n = dict?[key] as? NSNumber, + CFGetTypeID(n) == CFBooleanGetTypeID() { + return n.boolValue + } + return nil + } +} + +// MARK: - Errors + +enum RpcErrorCode: Int { + case parseError = -32700 + case invalidRequest = -32600 + case methodNotFound = -32601 + case invalidParams = -32602 + case internalError = -32603 + + // Bridge-specific (above -32000) + case windowNotFound = -32010 + case elementNotFound = -32011 + case unsupportedPattern = -32012 + case actionFailed = -32013 + case accessibilityNotGranted = -32020 +} + +struct RpcError: Error { + let code: Int + let message: String + var requestId: JsonRpcId = .null + + init(code: RpcErrorCode, message: String, requestId: JsonRpcId = .null) { + self.code = code.rawValue + self.message = message + self.requestId = requestId + } +} diff --git a/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/main.swift b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/main.swift new file mode 100644 index 0000000..67f6c97 --- /dev/null +++ b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/main.swift @@ -0,0 +1,91 @@ +// AgentMark macOS AXAPI Bridge +// ---------------------------- +// Stdio JSON-RPC 2.0 server. Parent process (typically the Node-side +// `MacosAxapiBackend` running inside the AgentMark MCP server) launches +// this binary and talks to it over stdin/stdout. One line of JSON per +// message. +// +// Protocol — identical to the Windows bridge: +// Request : { "jsonrpc": "2.0", "id": , "method": "", "params": {...} } +// Response: { "jsonrpc": "2.0", "id": , "result": {...} } on success +// { "jsonrpc": "2.0", "id": , "error": { "code": N, "message": "..." } } on failure +// +// All diagnostic output goes to stderr — never stdout — so the framing +// stays clean. +// +// Methods (this phase): +// ping — returns { pong, version, arch, processId } +// capabilities— returns { bridge, version, methods, axapiProvider, platform } +// +// Future: +// list_windows— enumerate top-level windows visible to AXAPI +// capture — walk AXAPI tree → DesktopCapture JSON +// execute — drive an action by element_id (AXPress / AXSetValue / etc.) +// +// AXAPI permission note: macOS guards Accessibility API access behind +// System Settings → Privacy & Security → Accessibility. The parent +// process (Claude Desktop, AgentMark MCP server) must be granted +// permission OR this bridge binary itself must be granted. We probe +// AXIsProcessTrusted() on startup and surface a clear error on first +// list_windows/capture if not granted. + +import Foundation + +let bridgeVersion = "0.4.0" + +let stderr = FileHandle.standardError +stderr.write("[bridge] agentmark-bridge-macos starting (pid=\(ProcessInfo.processInfo.processIdentifier), arch=\(currentArchString()))\n".data(using: .utf8)!) + +let dispatcher = Dispatcher(bridgeVersion: bridgeVersion) + +// Read newline-delimited JSON from stdin until EOF. +while let line = readLine(strippingNewline: true) { + let trimmed = stripBom(line.trimmingCharacters(in: .whitespacesAndNewlines)) + if trimmed.isEmpty { continue } + + do { + let request = try JsonRpcRequest.decode(jsonString: trimmed) + let result = try dispatcher.dispatch(request: request) + let response = JsonRpcResponse.success(id: request.id, result: result) + writeLine(try response.encode()) + } catch let rpcError as RpcError { + let response = JsonRpcResponse.error(id: rpcError.requestId, code: rpcError.code, message: rpcError.message) + writeLine((try? response.encode()) ?? "{}") + } catch { + // Last-resort fallback for genuinely unexpected exceptions. + let response = JsonRpcResponse.error( + id: .null, + code: RpcErrorCode.internalError.rawValue, + message: "Unhandled: \(error)" + ) + writeLine((try? response.encode()) ?? "{}") + stderr.write("[bridge] unhandled: \(error)\n".data(using: .utf8)!) + } +} + +stderr.write("[bridge] stdin closed; exiting\n".data(using: .utf8)!) + +// MARK: - I/O helpers + +func writeLine(_ s: String) { + FileHandle.standardOutput.write((s + "\n").data(using: .utf8)!) +} + +/// Strips the U+FEFF byte-order mark some clients (PowerShell, certain +/// shells) prepend to the first stdin write. Keeps JSON parsing happy. +func stripBom(_ s: String) -> String { + if s.first == "\u{FEFF}" { + return String(s.dropFirst()) + } + return s +} + +func currentArchString() -> String { + #if arch(arm64) + return "arm64" + #elseif arch(x86_64) + return "x86_64" + #else + return "unknown" + #endif +} diff --git a/apps/agent-runner/bridges/macos/scripts/smoke-test.sh b/apps/agent-runner/bridges/macos/scripts/smoke-test.sh new file mode 100755 index 0000000..3677439 --- /dev/null +++ b/apps/agent-runner/bridges/macos/scripts/smoke-test.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Smoke test for agentmark-bridge-macos. +# +# Feeds two requests through stdin (ping + capabilities), asserts both +# come back with the expected shape, exits non-zero on failure. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BIN="$SCRIPT_DIR/../.build/debug/agentmark-bridge-macos" + +if [[ ! -x "$BIN" ]]; then + echo "Bridge binary not built. Expected at $BIN" >&2 + echo "Run: cd $(dirname "$SCRIPT_DIR") && swift build" >&2 + exit 1 +fi + +echo "Smoke testing $BIN" +echo + +# Two requests separated by newlines, then close stdin. +REQUESTS='{"jsonrpc":"2.0","id":1,"method":"ping"} +{"jsonrpc":"2.0","id":2,"method":"capabilities"}' + +# Run with a 10-second wall-clock cap. +OUTPUT="$(echo "$REQUESTS" | "$BIN" 2>/tmp/agentmark-bridge-macos-stderr.log)" \ + || { echo "Bridge exited non-zero. stderr:"; cat /tmp/agentmark-bridge-macos-stderr.log; exit 1; } + +LINES=() +while IFS= read -r line; do + [[ -n "$line" ]] && LINES+=("$line") +done <<< "$OUTPUT" + +if [[ ${#LINES[@]} -ne 2 ]]; then + echo "Expected 2 response lines; got ${#LINES[@]}." >&2 + echo "Output was:" >&2 + printf '%s\n' "${LINES[@]}" >&2 + exit 1 +fi + +for L in "${LINES[@]}"; do echo " -> $L"; done +echo + +# Validate basic shape with grep — no jq dependency. +if ! grep -q '"pong":true' <<< "${LINES[0]}"; then + echo "ping: expected result.pong=true" >&2 + exit 1 +fi +if ! grep -q '"id":1' <<< "${LINES[0]}"; then + echo "ping: id mismatch (expected 1)" >&2 + exit 1 +fi +if ! grep -q '"bridge":"agentmark-bridge-macos"' <<< "${LINES[1]}"; then + echo "capabilities: expected bridge=agentmark-bridge-macos" >&2 + exit 1 +fi +if ! grep -q '"id":2' <<< "${LINES[1]}"; then + echo "capabilities: id mismatch (expected 2)" >&2 + exit 1 +fi + +echo "SMOKE TEST PASSED" +echo " bridge stderr:" +sed 's/^/ /' /tmp/agentmark-bridge-macos-stderr.log From 511a4fad5ed1f5291274776a3aac807c11ba7d48 Mon Sep 17 00:00:00 2001 From: rrader26 Date: Mon, 11 May 2026 16:35:19 -0400 Subject: [PATCH 2/3] feat(bridge-macos): Phase 0e2' list_windows + capture via AXAPI Adds the real AXAPI-driven methods to the macOS bridge. Builds clean on Apple Silicon arm64. Validated end-to-end against a live Mac: list_windows returned 22 real windows across 13 apps (Finder, Cursor, Chrome, VS Code, Claude Desktop, Teams, Outlook, Parallels, Terminal, etc.); capture against Finder's Copy progress dialog returned 11 elements at depth 2 with correct role mapping (window -> pane -> image / static_text / progress_bar / button) and live property values (progress=0.132, copy status text, traffic-light buttons). Mirrors the Windows bridge UiaCapturer pattern: defensive everywhere (every AXAPI read wrapped, swallows mid-walk failures), depth + element-count + deadline caps, per-capture session cache for element_id -> AXUIElement lookup (Phase 0e3' execute will resolve against this), same JSON wire format the Node-side body-builder already consumes. ## New module Sources/AgentMarkBridgeMacos/ Axapi.swift -- AccessibilityPermission probe (AXIsProcessTrusted), AxElement wrapper around AXUIElement with defensive attribute readers (string/bool/value/children/bounds), content-derived stableId() fallback (macOS has no AutomationId equivalent so we hash role+title+value+position), RoleMapper for AXAPI roles -> normalised DesktopRole vocab (AXButton -> button, AXTextField subrole AXSecureTextField -> password_input, AXScrollArea -> pane, AXOutline -> tree, AXOutlineRow -> tree_item, etc.) AxapiCapturer.swift -- ListWindows() walks NSWorkspace.runningApplications, filters to .regular activation policy, AXUIElementCreate- Application(pid) -> kAXWindowsAttribute, marks hasFocus by cross-referencing NSWorkspace.frontmostApplication. Capture(req) resolves target by windowId (axapi::), processId, processName (substring, .app-tolerant), windowTitle, or falls back to focused window. Walks tree depth-first with depth+count+ deadline caps, extracts value via the per-role correct attribute, screen bounds via kAXPosition + kAXSize, focused-element id via kAXFocusedUIElementAttribute + identity comparison against captured handles. requirePermission() surfaces a clear accessibilityNotGranted error (-32020) when the parent process hasn't been granted Accessibility in System Settings. ## Dispatcher.swift Routes list_windows + capture through AxapiCapturer (lazy init so ping-only smoke tests don't pay AXAPI startup cost). Adds accessibilityGranted bool to capabilities response so clients can detect permission issues without waiting for first capture. ## Validation Built locally on Apple Silicon arm64; tested against real Mac: list_windows -> 22 windows across 13 real apps (Finder, Cursor x3, Chrome x5 including the actual agentmark PR view, VS Code with claude_desktop_config.json open, Claude Desktop, Teams, Messages, Outlook, Parallels Desktop, Terminal, etc.). hasFocus correctly marked on the active Cursor window. capture processName=Finder -> Copy progress dialog with 11 elements, depth 2. Live progress value (0.132 -> 13%) extracted. Real screen coordinates. Correct role mapping. Traffic-light buttons surfaced with bounds at x=78/98/118 (close/minimize/maximize). Same JSON wire format as the Windows bridge -- the Node-side body builder we built for the FixtureBackend renders this transparently once Phase 0f' (MacosAxapiBackend) lands. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../Sources/AgentMarkBridgeMacos/Axapi.swift | 226 ++++++++++ .../AgentMarkBridgeMacos/AxapiCapturer.swift | 395 ++++++++++++++++++ .../AgentMarkBridgeMacos/Dispatcher.swift | 59 ++- 3 files changed, 668 insertions(+), 12 deletions(-) create mode 100644 apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Axapi.swift create mode 100644 apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/AxapiCapturer.swift diff --git a/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Axapi.swift b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Axapi.swift new file mode 100644 index 0000000..1684a37 --- /dev/null +++ b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Axapi.swift @@ -0,0 +1,226 @@ +// AXAPI plumbing — wraps AXUIElement, AXUIElementCopyAttributeValue, +// and friends in idiomatic Swift. Mirrors the Windows bridge's UIA +// patterns: defensive everywhere, swallows mid-walk failures, never +// throws on missing attributes. + +import Foundation +import ApplicationServices +import Cocoa + +// MARK: - Permission + +enum AccessibilityPermission { + /// True if the running process has been granted Accessibility + /// permission in System Settings → Privacy & Security → Accessibility. + /// Phase 0e2'/0e3' surfaces a clear JSON-RPC error if this is false. + static var isGranted: Bool { + return AXIsProcessTrusted() + } + + /// Triggers the system prompt (one-time) to grant Accessibility. + /// We don't call this from the bridge — the prompt would interrupt + /// the parent process. Useful for installers / first-run UI. + static func promptForAccess() -> Bool { + let options: NSDictionary = [ + kAXTrustedCheckOptionPrompt.takeUnretainedValue() as NSString: true + ] + return AXIsProcessTrustedWithOptions(options) + } +} + +// MARK: - AxElement + +/// Lightweight wrapper around an `AXUIElement`. Every attribute read +/// returns nil on any AXAPI error so a single stale element doesn't +/// abort the whole capture. +struct AxElement { + let element: AXUIElement + + init(_ element: AXUIElement) { + self.element = element + } + + // ---- Attribute readers ---- + + func string(_ attr: String) -> String? { + var value: CFTypeRef? + let err = AXUIElementCopyAttributeValue(element, attr as CFString, &value) + guard err == .success, let v = value else { return nil } + if CFGetTypeID(v) == CFStringGetTypeID() { + return v as? String + } + return nil + } + + func bool(_ attr: String) -> Bool? { + var value: CFTypeRef? + let err = AXUIElementCopyAttributeValue(element, attr as CFString, &value) + guard err == .success, let v = value else { return nil } + if CFGetTypeID(v) == CFBooleanGetTypeID() { + return (v as! CFBoolean) === kCFBooleanTrue + } + return nil + } + + /// Generic "value" extractor — AX exposes value as either string, + /// number, AXValue (CGPoint/Size/etc.), or bool depending on the role. + /// We canonicalise to a string representation that's useful for the + /// agent. + func value(_ attr: String = kAXValueAttribute as String) -> String? { + var value: CFTypeRef? + let err = AXUIElementCopyAttributeValue(element, attr as CFString, &value) + guard err == .success, let v = value else { return nil } + let typeId = CFGetTypeID(v) + if typeId == CFStringGetTypeID() { + return v as? String + } + if typeId == CFNumberGetTypeID() { + return "\(v as! NSNumber)" + } + if typeId == CFBooleanGetTypeID() { + return ((v as! CFBoolean) === kCFBooleanTrue) ? "true" : "false" + } + return nil + } + + func children(limit: Int = 5_000) -> [AxElement] { + var value: CFTypeRef? + let err = AXUIElementCopyAttributeValue(element, kAXChildrenAttribute as CFString, &value) + guard err == .success, let v = value else { return [] } + if CFGetTypeID(v) == CFArrayGetTypeID() { + let arr = v as! [AnyObject] + return arr.prefix(limit).compactMap { item in + // Each child should be an AXUIElement. CFGetTypeID() + // confirms; AXUIElementGetTypeID is the real test. + if CFGetTypeID(item) == AXUIElementGetTypeID() { + return AxElement(item as! AXUIElement) + } + return nil + } + } + return [] + } + + /// Screen-space bounds via kAXPositionAttribute + kAXSizeAttribute. + /// Returns nil when either is missing or fails to decode. + func bounds() -> (x: Double, y: Double, width: Double, height: Double)? { + var posRef: CFTypeRef? + let posErr = AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &posRef) + var sizeRef: CFTypeRef? + let sizeErr = AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeRef) + guard posErr == .success, let pos = posRef, + sizeErr == .success, let size = sizeRef else { return nil } + var point = CGPoint.zero + var sz = CGSize.zero + AXValueGetValue(pos as! AXValue, .cgPoint, &point) + AXValueGetValue(size as! AXValue, .cgSize, &sz) + if sz.width == 0 && sz.height == 0 { return nil } + return (Double(point.x), Double(point.y), Double(sz.width), Double(sz.height)) + } + + /// Stable id for the element. AXAPI has no AutomationId equivalent; + /// kAXIdentifierAttribute is occasionally populated but most apps + /// don't set it. Fall back to a content-derived hash so the same + /// element produces the same id across captures. + func stableId(role: String, fallbackIndex: Int) -> String { + if let id = string(kAXIdentifierAttribute as String), !id.isEmpty { + return id + } + // Hash inputs: role + title + value's first 32 chars + position. + let title = string(kAXTitleAttribute as String) ?? "" + let value = self.value() ?? "" + let trimmedValue = String(value.prefix(32)) + var pos = "?" + if let b = bounds() { + pos = "\(Int(b.x)),\(Int(b.y)),\(Int(b.width))x\(Int(b.height))" + } + let raw = "\(role)|\(title)|\(trimmedValue)|\(pos)|\(fallbackIndex)" + let hash = AxElement.djb2Hash(raw) + return String(format: "el_%08x", hash) + } + + static func djb2Hash(_ s: String) -> UInt32 { + var hash: UInt32 = 5381 + for byte in s.utf8 { + hash = ((hash << 5) &+ hash) &+ UInt32(byte) + } + return hash + } +} + +// MARK: - Role mapping + +enum RoleMapper { + /// Translates AXAPI role (and where helpful, subrole) into the + /// AgentMark v0.4 normalised `DesktopRole` vocabulary. Unknown + /// roles fall through to "other"; the body-builder on the Node + /// side renders these gracefully (recurses but doesn't surface + /// them as actions). + static func toRole(role: String?, subrole: String?) -> String { + guard let role = role else { return "other" } + + // Subrole takes priority for a few cases that change semantics. + if role == "AXTextField" && subrole == "AXSecureTextField" { + return "password_input" + } + if role == "AXTextField" && subrole == "AXSearchField" { + return "text_input" + } + if (role == "AXTextArea") || (role == "AXTextField" && subrole == "AXContentSearchField") { + return "text_area" + } + if role == "AXButton" && subrole == "AXToggleSubrole" { + // older apps use AXToggleSubrole for toggle buttons + return "button" + } + if role == "AXWindow" && (subrole == "AXDialog" || subrole == "AXSystemDialog") { + return "dialog" + } + + switch role { + case "AXWindow": return "window" + case "AXGroup": return "group" + case "AXSplitGroup": return "pane" + case "AXScrollArea": return "pane" + case "AXToolbar": return "toolbar" + case "AXMenuBar": return "menu" + case "AXMenu": return "menu" + case "AXMenuItem": return "menu_item" + case "AXMenuBarItem": return "menu_item" + case "AXMenuButton": return "split_button" + case "AXTabGroup": return "tab_list" + case "AXRadioGroup": return "group" + case "AXTab": return "tab" + case "AXOutline": return "tree" + case "AXOutlineRow": return "tree_item" + case "AXList": return "list" + case "AXListItem": return "list_item" + case "AXTable": return "table" + case "AXRow": return "row" + case "AXCell": return "cell" + case "AXColumn": return "column_header" + case "AXButton": return "button" + case "AXTextField": return "text_input" + case "AXTextArea": return "text_area" + case "AXCheckBox": return "check_box" + case "AXRadioButton": return "radio_button" + case "AXPopUpButton": return "combo_box" + case "AXComboBox": return "combo_box" + case "AXSlider": return "slider" + case "AXProgressIndicator": return "progress_bar" + case "AXLink": return "link" + case "AXStaticText": return "static_text" + case "AXImage": return "image" + case "AXSplitter": return "separator" + case "AXScrollBar": return "scroll_bar" + case "AXSheet": return "dialog" + case "AXDrawer": return "pane" + case "AXHelpTag": return "tooltip" + case "AXValueIndicator": return "static_text" + case "AXIncrementor": return "slider" + case "AXBusyIndicator": return "progress_bar" + case "AXDisclosureTriangle": return "button" + default: return "other" + } + } +} diff --git a/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/AxapiCapturer.swift b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/AxapiCapturer.swift new file mode 100644 index 0000000..fe30d47 --- /dev/null +++ b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/AxapiCapturer.swift @@ -0,0 +1,395 @@ +// Core AXAPI work — list_windows + capture. Mirrors UiaCapturer.cs +// from the Windows bridge in behaviour and output shape. +// +// macOS-specific architectural notes: +// +// 1. Window enumeration goes via NSWorkspace.runningApplications +// (gives us regular apps + their pids) followed by +// AXUIElementCreateApplication(pid) → kAXWindowsAttribute. There's +// no Win32 HWND analog so we encode window ids as +// `axapi::` and re-resolve on each call. +// +// 2. AXAPI access requires Accessibility permission. We probe +// AXIsProcessTrusted() once at the start of every call that +// actually touches AXAPI. Without permission the bridge returns a +// clear JSON-RPC error with code -32020 (accessibilityNotGranted) +// pointing the user at System Settings. +// +// 3. macOS has no "AutomationId" — every element gets a content-derived +// stable hash via AxElement.stableId(). Survives across captures as +// long as title/value/position stay stable; works fine for the +// capture→execute round-trip the agent needs. + +import Foundation +import ApplicationServices +import Cocoa + +final class AxapiCapturer { + + /// Per-capture session — maps element_id → live AXUIElement so + /// execute() (Phase 0e3') can find the same element again. + final class Session { + let rootApp: AXUIElement + let rootWindow: AXUIElement + var elements: [String: AXUIElement] = [:] + init(rootApp: AXUIElement, rootWindow: AXUIElement) { + self.rootApp = rootApp + self.rootWindow = rootWindow + } + } + + private(set) var lastSession: Session? + + // MARK: - list_windows + + /// Enumerate top-level visible windows the bridge can see. + func listWindows() throws -> [[String: Any]] { + try requirePermission() + let focusedHint = focusedAppPid() + var out: [[String: Any]] = [] + + for app in NSWorkspace.shared.runningApplications { + // Skip background-only / agent-style apps; they rarely have + // windows worth surfacing and clutter the list. + if app.activationPolicy != .regular { continue } + let pid = app.processIdentifier + if pid <= 0 { continue } + + let appAx = AXUIElementCreateApplication(pid) + guard let windows = appWindows(appAx) else { continue } + + for (idx, win) in windows.enumerated() { + let el = AxElement(win) + let title = el.string(kAXTitleAttribute as String) ?? "" + // Skip chrome-less ghost windows. + if title.isEmpty { continue } + + let isMain = el.bool(kAXMainAttribute as String) ?? false + let isFocused = el.bool(kAXFocusedAttribute as String) ?? false + let hasFocus = pid == focusedHint && (isFocused || isMain) + + var summary: [String: Any] = [ + "windowId": encodeWindowId(pid: pid, index: idx), + "windowTitle": title, + "processName": app.localizedName ?? "(unknown)", + "processId": Int(pid), + "hasFocus": hasFocus, + ] + if let role = el.string(kAXRoleAttribute as String) { + // Surface the AX role as a hint; many GUI tools want this. + summary["windowClass"] = role + } + out.append(summary) + } + } + return out + } + + // MARK: - capture + + struct CaptureRequest { + var processName: String? + var processId: Int? + var windowTitle: String? + var windowId: String? + var maxDepth: Int = 12 + var includeHidden: Bool = false + var timeoutMs: Int = 5000 + var maxElements: Int = 2000 + } + + func capture(_ req: CaptureRequest) throws -> [String: Any] { + try requirePermission() + + guard let target = resolveTarget(req) else { + throw RpcError( + code: .windowNotFound, + message: "No matching window found and no focused window available." + ) + } + let (appAx, windowAx, pid, app) = target + + let session = Session(rootApp: appAx, rootWindow: windowAx) + var ctx = WalkContext( + maxDepth: max(1, req.maxDepth), + maxElements: max(50, req.maxElements), + includeHidden: req.includeHidden, + deadline: Date().addingTimeInterval(Double(max(500, req.timeoutMs)) / 1000.0) + ) + + let rootDto = walk(AxElement(windowAx), depth: 0, ctx: &ctx, session: session) + self.lastSession = session + + let focusedElementId = resolveFocusedElementId(in: session) + + var out: [String: Any] = [ + "platform": "macos", + "processName": app.localizedName ?? "(unknown)", + "processId": Int(pid), + "windowTitle": AxElement(windowAx).string(kAXTitleAttribute as String) ?? "(untitled window)", + "windowId": encodeWindowId(pid: pid, index: indexOf(window: windowAx, app: appAx) ?? 0), + "treeDepth": ctx.maxDepthReached, + "elementCount": ctx.elementCount, + "root": rootDto, + ] + if let role = AxElement(windowAx).string(kAXRoleAttribute as String) { + out["windowClass"] = role + } + if let fid = focusedElementId { + out["focusedElementId"] = fid + } + return out + } + + // MARK: - Walk + + private struct WalkContext { + let maxDepth: Int + let maxElements: Int + let includeHidden: Bool + let deadline: Date + var elementCount: Int = 0 + var maxDepthReached: Int = 0 + var seenIds: Set = [] + } + + private func walk( + _ el: AxElement, + depth: Int, + ctx: inout WalkContext, + session: Session + ) -> [String: Any] { + ctx.elementCount += 1 + if depth > ctx.maxDepthReached { ctx.maxDepthReached = depth } + + let role = el.string(kAXRoleAttribute as String) + let subrole = el.string(kAXSubroleAttribute as String) + let mappedRole = RoleMapper.toRole(role: role, subrole: subrole) + + var id = el.stableId(role: mappedRole, fallbackIndex: ctx.elementCount) + // Disambiguate hash collisions across siblings. + if ctx.seenIds.contains(id) { + var n = 2 + while ctx.seenIds.contains("\(id)#\(n)") { n += 1 } + id = "\(id)#\(n)" + } + ctx.seenIds.insert(id) + session.elements[id] = el.element + + var dto: [String: Any] = [ + "id": id, + "role": mappedRole, + ] + if let name = el.string(kAXTitleAttribute as String), !name.isEmpty { + dto["name"] = name + } else if let alt = el.string("AXDescription"), !alt.isEmpty { + dto["name"] = alt + } + if let v = el.value(), !v.isEmpty { + dto["value"] = v + } + if let placeholder = el.string("AXPlaceholderValue"), !placeholder.isEmpty { + dto["placeholder"] = placeholder + } + if let enabled = el.bool(kAXEnabledAttribute as String) { + dto["enabled"] = enabled + } + if let selected = el.bool(kAXSelectedAttribute as String) { + dto["selected"] = selected + } + if let expanded = el.bool(kAXExpandedAttribute as String) { + dto["expanded"] = expanded + } + if let bounds = el.bounds() { + dto["bounds"] = [ + "x": bounds.x, "y": bounds.y, + "width": bounds.width, "height": bounds.height, + ] + } + + // Stop expanding at caps. Parent still includes its own data + // but children get truncated; body-builder on the Node side + // renders the markdown gracefully when children is null. + if depth >= ctx.maxDepth { return dto } + if ctx.elementCount >= ctx.maxElements { return dto } + if Date() > ctx.deadline { return dto } + + let children = el.children() + if children.isEmpty { return dto } + + var kids: [[String: Any]] = [] + kids.reserveCapacity(children.count) + for c in children { + if ctx.elementCount >= ctx.maxElements { break } + if Date() > ctx.deadline { break } + // Off-screen / zero-size filter — children with no bounds + // are rare but exist (hidden helper elements); skip unless + // includeHidden is set. + if !ctx.includeHidden { + if let b = c.bounds(), b.width == 0 || b.height == 0 { continue } + } + kids.append(walk(c, depth: depth + 1, ctx: &ctx, session: session)) + } + if !kids.isEmpty { + dto["children"] = kids + } + return dto + } + + // MARK: - Target resolution + + private func resolveTarget(_ req: CaptureRequest) -> (AXUIElement, AXUIElement, pid_t, NSRunningApplication)? { + // 1. window_id (axapi::) is the most precise match. + if let id = req.windowId, let decoded = decodeWindowId(id) { + if let app = NSWorkspace.shared.runningApplications.first(where: { $0.processIdentifier == decoded.pid }) { + let appAx = AXUIElementCreateApplication(decoded.pid) + if let windows = appWindows(appAx), decoded.index < windows.count { + return (appAx, windows[decoded.index], decoded.pid, app) + } + } + } + + // 2. processId match + if let pid = req.processId { + if let app = NSWorkspace.shared.runningApplications.first(where: { $0.processIdentifier == pid_t(pid) }) { + let appAx = AXUIElementCreateApplication(pid_t(pid)) + if let win = firstMeaningfulWindow(appAx) { + return (appAx, win, pid_t(pid), app) + } + } + } + + // 3. processName match (substring, case-insensitive, .app suffix tolerant) + if let needle = req.processName?.lowercased().replacingOccurrences(of: ".app", with: "") { + for app in NSWorkspace.shared.runningApplications { + if app.activationPolicy != .regular { continue } + let name = (app.localizedName ?? "").lowercased() + let bundle = (app.bundleIdentifier ?? "").lowercased() + if name.contains(needle) || bundle.contains(needle) { + let appAx = AXUIElementCreateApplication(app.processIdentifier) + if let win = firstMeaningfulWindow(appAx) { + return (appAx, win, app.processIdentifier, app) + } + } + } + } + + // 4. windowTitle match + if let titleNeedle = req.windowTitle?.lowercased() { + for app in NSWorkspace.shared.runningApplications { + if app.activationPolicy != .regular { continue } + let appAx = AXUIElementCreateApplication(app.processIdentifier) + guard let windows = appWindows(appAx) else { continue } + for win in windows { + if let t = AxElement(win).string(kAXTitleAttribute as String), + t.lowercased().contains(titleNeedle) { + return (appAx, win, app.processIdentifier, app) + } + } + } + } + + // 5. Default: focused window. + if let pid = focusedAppPid(), + let app = NSWorkspace.shared.runningApplications.first(where: { $0.processIdentifier == pid }) { + let appAx = AXUIElementCreateApplication(pid) + if let win = focusedWindow(appAx) ?? firstMeaningfulWindow(appAx) { + return (appAx, win, pid, app) + } + } + return nil + } + + // MARK: - Helpers + + private func requirePermission() throws { + if !AccessibilityPermission.isGranted { + throw RpcError( + code: .accessibilityNotGranted, + message: "Accessibility permission not granted. Open System Settings → Privacy & Security → Accessibility and enable the process that launched this bridge (Claude Desktop, Terminal during dev, or the agentmark MCP server)." + ) + } + } + + private func appWindows(_ appAx: AXUIElement) -> [AXUIElement]? { + var value: CFTypeRef? + let err = AXUIElementCopyAttributeValue(appAx, kAXWindowsAttribute as CFString, &value) + guard err == .success, let v = value else { return nil } + if CFGetTypeID(v) == CFArrayGetTypeID() { + let arr = v as! [AnyObject] + return arr.compactMap { item in + if CFGetTypeID(item) == AXUIElementGetTypeID() { + return (item as! AXUIElement) + } + return nil + } + } + return nil + } + + private func firstMeaningfulWindow(_ appAx: AXUIElement) -> AXUIElement? { + guard let windows = appWindows(appAx) else { return nil } + for win in windows { + let el = AxElement(win) + if let t = el.string(kAXTitleAttribute as String), !t.isEmpty { + return win + } + } + return windows.first + } + + private func focusedWindow(_ appAx: AXUIElement) -> AXUIElement? { + var value: CFTypeRef? + let err = AXUIElementCopyAttributeValue(appAx, kAXFocusedWindowAttribute as CFString, &value) + guard err == .success, let v = value, CFGetTypeID(v) == AXUIElementGetTypeID() else { + return nil + } + return (v as! AXUIElement) + } + + private func focusedAppPid() -> pid_t? { + // NSWorkspace.frontmostApplication is the highest-fidelity + // signal — it tracks the actually-active window owner. + return NSWorkspace.shared.frontmostApplication?.processIdentifier + } + + private func resolveFocusedElementId(in session: Session) -> String? { + var value: CFTypeRef? + let err = AXUIElementCopyAttributeValue(session.rootApp, kAXFocusedUIElementAttribute as CFString, &value) + guard err == .success, let v = value, CFGetTypeID(v) == AXUIElementGetTypeID() else { + return nil + } + let focused = v as! AXUIElement + // Try to find that element in the captured session. + for (id, el) in session.elements { + if CFEqual(el, focused) { + return id + } + } + return nil + } + + private func indexOf(window: AXUIElement, app: AXUIElement) -> Int? { + guard let windows = appWindows(app) else { return nil } + for (i, w) in windows.enumerated() where CFEqual(w, window) { + return i + } + return nil + } + + // MARK: - Window ID encoding + + func encodeWindowId(pid: pid_t, index: Int) -> String { + return "axapi:\(pid):\(index)" + } + + func decodeWindowId(_ s: String) -> (pid: pid_t, index: Int)? { + guard s.hasPrefix("axapi:") else { return nil } + let parts = s.dropFirst("axapi:".count).split(separator: ":") + guard parts.count == 2, + let pid = pid_t(parts[0]), + let idx = Int(parts[1]) + else { return nil } + return (pid, idx) + } +} diff --git a/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Dispatcher.swift b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Dispatcher.swift index 5da8ffc..b24500a 100644 --- a/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Dispatcher.swift +++ b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Dispatcher.swift @@ -7,22 +7,36 @@ import Foundation final class Dispatcher { private let bridgeVersion: String + // AXAPI is fine to call from the main thread; we don't currently + // need a dedicated serial queue. If we ever do, this is where the + // dispatch boundary goes. + private lazy var capturer = AxapiCapturer() + init(bridgeVersion: String) { self.bridgeVersion = bridgeVersion } func dispatch(request: JsonRpcRequest) throws -> Any { - switch request.method { - case "ping": - return handlePing() - case "capabilities": - return handleCapabilities() - default: - throw RpcError( - code: .methodNotFound, - message: "Unknown method: \(request.method). Supported: ping, capabilities.", - requestId: request.id - ) + do { + switch request.method { + case "ping": + return handlePing() + case "capabilities": + return handleCapabilities() + case "list_windows": + return try handleListWindows() + case "capture": + return try handleCapture(params: request.params) + default: + throw RpcError( + code: .methodNotFound, + message: "Unknown method: \(request.method). Supported: ping, capabilities, list_windows, capture.", + requestId: request.id + ) + } + } catch var rpcError as RpcError { + rpcError.requestId = request.id + throw rpcError } } @@ -39,9 +53,30 @@ final class Dispatcher { return [ "bridge": "agentmark-bridge-macos", "version": bridgeVersion, - "methods": ["ping", "capabilities"], + "methods": ["ping", "capabilities", "list_windows", "capture"], "axapiProvider": "Accessibility (AXAPI)", "platform": "macos", + "accessibilityGranted": AccessibilityPermission.isGranted, ] as [String: Any] } + + private func handleListWindows() throws -> Any { + let windows = try capturer.listWindows() + return ["windows": windows] as [String: Any] + } + + private func handleCapture(params: JsonValue?) throws -> Any { + var req = AxapiCapturer.CaptureRequest() + if let p = params { + if let s = p.string("processName") { req.processName = s } + if let i = p.int("processId") { req.processId = i } + if let s = p.string("windowTitle") { req.windowTitle = s } + if let s = p.string("windowId") { req.windowId = s } + if let i = p.int("maxDepth") { req.maxDepth = i } + if let b = p.bool("includeHidden") { req.includeHidden = b } + if let i = p.int("timeoutMs") { req.timeoutMs = i } + if let i = p.int("maxElements") { req.maxElements = i } + } + return try capturer.capture(req) + } } From 69744737c73511c5e623168dd9f90cbc6709e107 Mon Sep 17 00:00:00 2001 From: rrader26 Date: Mon, 11 May 2026 16:44:25 -0400 Subject: [PATCH 3/3] feat(bridge-macos): Phase 0e3' execute (AXAPI action dispatch) Adds the `execute` JSON-RPC method to the macOS bridge. Routes element-targeted actions through the right AXAPI pattern. Mirrors the Windows bridge's UiaCapturer.Execute, same JSON shape on the wire. ## Action types - click -> AXUIElementPerformAction(AXPress) with AXToggle / AXPick / AXConfirm / AXShowMenu fallbacks - type -> AXUIElementSetAttributeValue(kAXValueAttribute) with focus+keystrokes (CGEvent) fallback when AXValue is not writable; supports clearFirst (Cmd+A then Delete) - select -> AXPress on a selection item, or kAXValueAttribute set directly for popup buttons - check -> Toggle until state matches (capped at 3 iterations) via AXPress or AXToggle, mapping into kAXValue bool - expand -> kAXExpandedAttribute set or AXShowMenu / AXPress fallback for outline rows / disclosure triangles - focus -> kAXFocusedAttribute = true - scroll_to -> AXUIElementPerformAction(AXScrollToVisible) with focus fallback - key -> CGEvent keyboard simulation. Modifier-aware (ctrl / option / shift / cmd). Named keys: enter, tab, esc, space, delete, arrows, pgup/pgdn, home/end, F1..F12. Unknown names fall through to literal text typing. ## Element lookup Capture stashes each walked AXUIElement in a per-session [element_id -> AXUIElement] map. Execute resolves the request's elementId against that map. Errors are specific: - "No capture session active" -> caller must call capture first - "Unknown element_id" -> stale binding, re-capture - "Accessibility permission not granted" -> system settings error ## CGEvent typing strategy For the keystroke fallback we use CGEventKeyboardSetUnicodeString rather than per-character keyCode lookup. Handles Latin + accented + most BMP characters without maintaining a layout-specific table. Combined with AXSetValue as the primary path, this covers TextEdit / Mail / Cursor / VS Code / Pages / Numbers cleanly. ## Live demo (scripts/textedit-demo.py) Python harness that: 1. Launches TextEdit with a new document via osascript 2. list_windows -> find TextEdit 3. capture -> locate the text_area element (the AXTextArea inside the AXScrollArea inside the AXSplitGroup inside the AXWindow) 4. execute type -> "Hello from AgentMark Desktop -- macOS bridge phase 0e3'" 5. capture -> assert the typed text is in element.value Tested live on Apple Silicon arm64. PASSED first run: Step 2 -- list_windows: found TextEdit window axapi:21232:0 Step 3 -- capture: 47 elements, depth 3, editor id="First Text View" Step 4 -- execute type: ok=true, newValue echoes the sent text Step 5 -- re-capture: editor value matches "Hello from AgentMark..." LIVE DEMO PASSED Co-Authored-By: Claude Opus 4.7 (1M context) --- .../AgentMarkBridgeMacos/AxapiCapturer.swift | 277 ++++++++++++++++++ .../AgentMarkBridgeMacos/Dispatcher.swift | 27 +- .../bridges/macos/scripts/textedit-demo.py | 186 ++++++++++++ .../bridges/macos/scripts/textedit-demo.sh | 137 +++++++++ 4 files changed, 625 insertions(+), 2 deletions(-) create mode 100755 apps/agent-runner/bridges/macos/scripts/textedit-demo.py create mode 100755 apps/agent-runner/bridges/macos/scripts/textedit-demo.sh diff --git a/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/AxapiCapturer.swift b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/AxapiCapturer.swift index fe30d47..bd89072 100644 --- a/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/AxapiCapturer.swift +++ b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/AxapiCapturer.swift @@ -377,6 +377,283 @@ final class AxapiCapturer { return nil } + // MARK: - Execute + + struct ExecuteRequest { + var elementId: String = "" + var actionType: String = "click" + var text: String? + var value: String? + var checked: Bool? + var expanded: Bool? + var key: String? + var modifiers: [String]? + var clearFirst: Bool = false + var timeoutMs: Int = 5000 + } + + func execute(_ req: ExecuteRequest) throws -> [String: Any] { + try requirePermission() + + guard let session = lastSession else { + return [ + "ok": false, + "message": "No capture session active. Call `capture` before `execute` so the bridge can resolve element_ids.", + ] + } + guard let element = session.elements[req.elementId] else { + return [ + "ok": false, + "message": "Unknown element_id `\(req.elementId)` in the current capture session. Re-capture if the window has changed.", + ] + } + + do { + switch req.actionType { + case "click": return try doClick(element) + case "type": return try doType(element, text: req.text ?? "", clearFirst: req.clearFirst) + case "select": return try doSelect(element, value: req.value ?? "") + case "check": return try doCheck(element, want: req.checked ?? true) + case "expand": return try doExpand(element, want: req.expanded ?? true) + case "focus": return try doFocus(element) + case "scroll_to": return try doScrollTo(element) + case "key": return try doKey(element, key: req.key ?? "", modifiers: req.modifiers ?? []) + default: + return ["ok": false, "message": "Unknown action type `\(req.actionType)`."] + } + } catch let rpcError as RpcError { + return ["ok": false, "message": rpcError.message] + } catch { + return ["ok": false, "message": "\(error)"] + } + } + + // ---- Per-action helpers ---- + + private func doClick(_ el: AXUIElement) throws -> [String: Any] { + // Try AXPress first (buttons, menu items, links). + if performAction(el, "AXPress") { + return ["ok": true] + } + // Toggle action (some checkboxes). + if performAction(el, "AXToggle") { + return ["ok": true] + } + // Pick (some popups / list items). + if performAction(el, "AXPick") { + return ["ok": true] + } + // Confirm (default button in a dialog). + if performAction(el, "AXConfirm") { + return ["ok": true] + } + // Show menu (for popup-button-style controls). + if performAction(el, "AXShowMenu") { + return ["ok": true, "message": "Used AXShowMenu fallback (element exposed no AXPress)."] + } + return ["ok": false, "message": "Element does not support any clickable AX action."] + } + + private func doType(_ el: AXUIElement, text: String, clearFirst: Bool) throws -> [String: Any] { + // AXAPI's standard text input is kAXValueAttribute as a String. + // Setting it via AXUIElementSetAttributeValue replaces content + // atomically; honour `clearFirst` by setting "" first when + // requested. + _ = setFocus(el) // best-effort; some apps require focus before SetValue is honoured + + if clearFirst { + _ = AXUIElementSetAttributeValue(el, kAXValueAttribute as CFString, "" as CFTypeRef) + } + let err = AXUIElementSetAttributeValue(el, kAXValueAttribute as CFString, text as CFTypeRef) + if err == .success { + let newValue = AxElement(el).value() ?? text + return ["ok": true, "newValue": newValue] + } + + // Fallback: synthesise keystrokes via CGEvent. Only usable when + // the element accepts focus and the host app honours regular + // keyboard input on the focused element. + if !setFocus(el) { + return ["ok": false, "message": "SetAttributeValue failed (\(err)) and element refused focus."] + } + if clearFirst { + sendKeyCombo(keyCode: 0x00 /* a */, modifiers: [.maskCommand]) + sendKey(keyCode: 0x33 /* delete */) + } + typeString(text) + return ["ok": true, "message": "Used keyboard fallback (AXValue not writable)."] + } + + private func doSelect(_ el: AXUIElement, value: String) throws -> [String: Any] { + // For selection-item elements (rows, menu items, tabs), perform AXPress. + if performAction(el, "AXPress") { return ["ok": true, "newValue": AxElement(el).string(kAXTitleAttribute as String) ?? value] } + // For popup buttons, set kAXValue directly. + let err = AXUIElementSetAttributeValue(el, kAXValueAttribute as CFString, value as CFTypeRef) + if err == .success { return ["ok": true, "newValue": value] } + return ["ok": false, "message": "Element does not support a selection action."] + } + + private func doCheck(_ el: AXUIElement, want: Bool) throws -> [String: Any] { + // Toggle until state matches (cap at 3 to avoid loops on + // tri-state controls). + var guardCount = 3 + while guardCount > 0 { + guardCount -= 1 + let current = AxElement(el).bool(kAXValueAttribute as String) ?? false + if current == want { break } + // Try AXPress first, then AXToggle. + if !performAction(el, "AXPress") && !performAction(el, "AXToggle") { + return ["ok": false, "message": "Element does not support AXPress/AXToggle."] + } + } + let final = AxElement(el).bool(kAXValueAttribute as String) ?? false + return ["ok": final == want, "newValue": final ? "true" : "false"] + } + + private func doExpand(_ el: AXUIElement, want: Bool) throws -> [String: Any] { + // Expanded state lives in kAXExpandedAttribute or kAXDisclosing. + let current = AxElement(el).bool(kAXExpandedAttribute as String) ?? AxElement(el).bool("AXDisclosing") ?? false + if current == want { + return ["ok": true, "newValue": want ? "Expanded" : "Collapsed"] + } + // Try setting the attribute directly (works for outline rows). + let setErr = AXUIElementSetAttributeValue(el, kAXExpandedAttribute as CFString, want as CFTypeRef) + if setErr == .success { + return ["ok": true, "newValue": want ? "Expanded" : "Collapsed"] + } + // Fall back to AXShowMenu / AXPress to toggle. + if performAction(el, "AXShowMenu") || performAction(el, "AXPress") { + return ["ok": true, "newValue": want ? "Expanded" : "Collapsed", "message": "Used AXPress fallback."] + } + return ["ok": false, "message": "Element does not support expand/collapse."] + } + + private func doFocus(_ el: AXUIElement) throws -> [String: Any] { + if setFocus(el) { return ["ok": true] } + return ["ok": false, "message": "Element refused focus."] + } + + private func doScrollTo(_ el: AXUIElement) throws -> [String: Any] { + if performAction(el, "AXScrollToVisible") { + return ["ok": true] + } + // SetFocus often implies scroll-into-view for most controls. + if setFocus(el) { + return ["ok": true, "message": "Used focus fallback (no AXScrollToVisible)."] + } + return ["ok": false, "message": "Element does not support AXScrollToVisible and refused focus."] + } + + private func doKey(_ el: AXUIElement, key: String, modifiers: [String]) throws -> [String: Any] { + if key.isEmpty { + return ["ok": false, "message": "Missing `key` argument."] + } + _ = setFocus(el) + guard let code = virtualKeyCode(forName: key) else { + // Not a named key — type the literal text. + typeString(key) + return ["ok": true, "message": "Typed literal text `\(key)`."] + } + let mods = modifiers.compactMap(modifierFlag(forName:)) + sendKeyCombo(keyCode: code, modifiers: mods) + return ["ok": true] + } + + // MARK: - AXAPI action helpers + + /// AXUIElementPerformAction returns success on no-op as well as on + /// real activation, so we don't try to interpret partial-failure. + /// Caller chains alternatives if `false` returned. + private func performAction(_ el: AXUIElement, _ action: String) -> Bool { + return AXUIElementPerformAction(el, action as CFString) == .success + } + + /// Best-effort focus. Some elements ignore kAXFocusedAttribute + /// (Document role, AXStaticText); not strictly necessary for the + /// caller to act on the result. + private func setFocus(_ el: AXUIElement) -> Bool { + let err = AXUIElementSetAttributeValue(el, kAXFocusedAttribute as CFString, kCFBooleanTrue) + return err == .success + } + + // MARK: - Keyboard simulation (CGEvent) + + private func typeString(_ s: String) { + // CGEvent supports unicode payload directly via + // CGEventKeyboardSetUnicodeString — works for most Latin + // and accented characters without per-character key-code + // lookup. + for ch in s { + guard let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true), + let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false) else { continue } + let unicodeChars = Array(String(ch).utf16) + down.keyboardSetUnicodeString(stringLength: unicodeChars.count, unicodeString: unicodeChars) + up.keyboardSetUnicodeString(stringLength: unicodeChars.count, unicodeString: unicodeChars) + down.post(tap: .cghidEventTap) + up.post(tap: .cghidEventTap) + } + } + + private func sendKey(keyCode: CGKeyCode) { + sendKeyCombo(keyCode: keyCode, modifiers: []) + } + + private func sendKeyCombo(keyCode: CGKeyCode, modifiers: [CGEventFlags]) { + let combined: CGEventFlags = modifiers.reduce(CGEventFlags(rawValue: 0)) { CGEventFlags(rawValue: $0.rawValue | $1.rawValue) } + if let down = CGEvent(keyboardEventSource: nil, virtualKey: keyCode, keyDown: true) { + down.flags = combined + down.post(tap: .cghidEventTap) + } + if let up = CGEvent(keyboardEventSource: nil, virtualKey: keyCode, keyDown: false) { + up.flags = combined + up.post(tap: .cghidEventTap) + } + } + + /// Mapping of friendly key names to macOS virtual key codes. + /// Values from Carbon's `Events.h` (`kVK_*`) — Apple still ships + /// these as the canonical key-code constants on Apple Silicon. + private func virtualKeyCode(forName name: String) -> CGKeyCode? { + switch name.lowercased() { + case "return", "enter": return 0x24 + case "tab": return 0x30 + case "space", "spacebar": return 0x31 + case "delete", "backspace": return 0x33 + case "escape", "esc": return 0x35 + case "left": return 0x7B + case "right": return 0x7C + case "down": return 0x7D + case "up": return 0x7E + case "home": return 0x73 + case "end": return 0x77 + case "pageup", "pgup": return 0x74 + case "pagedown", "pgdn": return 0x79 + case "f1": return 0x7A + case "f2": return 0x78 + case "f3": return 0x63 + case "f4": return 0x76 + case "f5": return 0x60 + case "f6": return 0x61 + case "f7": return 0x62 + case "f8": return 0x64 + case "f9": return 0x65 + case "f10": return 0x6D + case "f11": return 0x67 + case "f12": return 0x6F + default: return nil + } + } + + private func modifierFlag(forName name: String) -> CGEventFlags? { + switch name.lowercased() { + case "ctrl", "control": return .maskControl + case "alt", "option": return .maskAlternate + case "shift": return .maskShift + case "meta", "cmd", "command", "win", "windows": return .maskCommand + default: return nil + } + } + // MARK: - Window ID encoding func encodeWindowId(pid: pid_t, index: Int) -> String { diff --git a/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Dispatcher.swift b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Dispatcher.swift index b24500a..72a0b6c 100644 --- a/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Dispatcher.swift +++ b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Dispatcher.swift @@ -27,10 +27,12 @@ final class Dispatcher { return try handleListWindows() case "capture": return try handleCapture(params: request.params) + case "execute": + return try handleExecute(params: request.params) default: throw RpcError( code: .methodNotFound, - message: "Unknown method: \(request.method). Supported: ping, capabilities, list_windows, capture.", + message: "Unknown method: \(request.method). Supported: ping, capabilities, list_windows, capture, execute.", requestId: request.id ) } @@ -53,7 +55,7 @@ final class Dispatcher { return [ "bridge": "agentmark-bridge-macos", "version": bridgeVersion, - "methods": ["ping", "capabilities", "list_windows", "capture"], + "methods": ["ping", "capabilities", "list_windows", "capture", "execute"], "axapiProvider": "Accessibility (AXAPI)", "platform": "macos", "accessibilityGranted": AccessibilityPermission.isGranted, @@ -79,4 +81,25 @@ final class Dispatcher { } return try capturer.capture(req) } + + private func handleExecute(params: JsonValue?) throws -> Any { + guard let p = params, + let elementId = p.string("elementId"), !elementId.isEmpty else { + throw RpcError(code: .invalidParams, message: "execute requires `elementId`.") + } + var req = AxapiCapturer.ExecuteRequest() + req.elementId = elementId + req.actionType = p.string("actionType") ?? "click" + req.text = p.string("text") + req.value = p.string("value") + req.checked = p.bool("checked") + req.expanded = p.bool("expanded") + req.key = p.string("key") + if let modsAny = p.dict?["modifiers"], case .array(let arr) = JsonValue.fromAny(modsAny) { + req.modifiers = arr.compactMap { $0 as? String } + } + req.clearFirst = p.bool("clearFirst") ?? false + if let i = p.int("timeoutMs") { req.timeoutMs = i } + return try capturer.execute(req) + } } diff --git a/apps/agent-runner/bridges/macos/scripts/textedit-demo.py b/apps/agent-runner/bridges/macos/scripts/textedit-demo.py new file mode 100755 index 0000000..fa79749 --- /dev/null +++ b/apps/agent-runner/bridges/macos/scripts/textedit-demo.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +""" +Live end-to-end demo: drive macOS TextEdit through the bridge. + +Sequence: + 1. Ensure TextEdit is running with a new document + 2. list_windows -> find the TextEdit window + 3. capture -> locate the text_area element + 4. execute type -> write a message into the editor + 5. capture -> verify the text landed in element.value + +Requires Accessibility permission granted to the terminal you're +running this from (System Settings -> Privacy & Security -> +Accessibility). +""" + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +SCRIPT_DIR = Path(__file__).parent.resolve() +BIN = SCRIPT_DIR.parent / ".build" / "debug" / "agentmark-bridge-macos" + +if not BIN.exists(): + print(f"Bridge not built. Expected at {BIN}\nRun: swift build", file=sys.stderr) + sys.exit(1) + + +def main() -> int: + # Step 1: ensure TextEdit is running with a doc. + print("Launching TextEdit...") + subprocess.run( + [ + "osascript", + "-e", 'tell application "TextEdit" to activate', + "-e", 'tell application "TextEdit" to make new document', + ], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + time.sleep(1) + + # Spawn bridge with bidirectional pipes. + bridge = subprocess.Popen( + [str(BIN)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + next_id = 1 + + def call(method: str, params: dict | None = None) -> dict: + nonlocal next_id + req = {"jsonrpc": "2.0", "id": next_id, "method": method} + if params is not None: + req["params"] = params + next_id += 1 + bridge.stdin.write(json.dumps(req) + "\n") + bridge.stdin.flush() + line = bridge.stdout.readline() + if not line: + err = bridge.stderr.read() + raise RuntimeError(f"Bridge closed unexpectedly. stderr:\n{err}") + return json.loads(line) + + def find_editor(node: dict) -> str | None: + if node.get("role") in ("text_area", "text_input"): + return node["id"] + for child in node.get("children") or []: + found = find_editor(child) + if found: + return found + return None + + def find_value(node: dict, target_id: str) -> str | None: + if node.get("id") == target_id: + return node.get("value") + for child in node.get("children") or []: + v = find_value(child, target_id) + if v is not None: + return v + return None + + try: + # Step 2: list_windows + print() + print("Step 2 -- list_windows...") + list_resp = call("list_windows") + windows = list_resp["result"]["windows"] + textedit = next( + (w for w in windows if w.get("processName") == "TextEdit"), + None, + ) + if textedit is None: + print("TextEdit window not found. Visible processes:") + for w in windows: + print(f" {w.get('processName'):25s} -- {w.get('windowTitle')}") + return 1 + window_id = textedit["windowId"] + print(f" found TextEdit window: {window_id} ({textedit.get('windowTitle')})") + + # Step 3: capture + print() + print("Step 3 -- capture TextEdit...") + cap = call("capture", { + "windowId": window_id, + "maxDepth": 8, + "maxElements": 400, + }) + if "error" in cap: + print(f" capture errored: {cap['error']}") + return 1 + result = cap["result"] + print(f" treeDepth={result['treeDepth']} elementCount={result['elementCount']}") + editor_id = find_editor(result["root"]) + if not editor_id: + print(" Could not find a text_area / text_input element. Tree top level:") + for c in (result["root"].get("children") or [])[:10]: + print(f" [{c.get('role'):15s}] id={c.get('id'):20s} name={c.get('name')}") + return 1 + print(f" editor element_id: {editor_id}") + + # Step 4: execute type + message = "Hello from AgentMark Desktop -- macOS bridge phase 0e3'" + print() + print("Step 4 -- execute type...") + exec_resp = call("execute", { + "elementId": editor_id, + "actionType": "type", + "text": message, + "clearFirst": True, + }) + if "error" in exec_resp: + print(f" execute errored: {exec_resp['error']}") + return 1 + r = exec_resp["result"] + print(f" ok: {r.get('ok')}") + if r.get("message"): + print(f" note: {r['message']}") + if r.get("newValue"): + preview = r["newValue"][:80] + print(f" newValue: {preview}") + + time.sleep(0.5) + + # Step 5: re-capture and verify + print() + print("Step 5 -- re-capture and verify...") + cap2 = call("capture", { + "windowId": window_id, + "maxDepth": 8, + "maxElements": 400, + }) + value_after = find_value(cap2["result"]["root"], editor_id) + print(f" editor value after type: {repr(value_after)[:120]}") + + ok = value_after and "Hello from AgentMark Desktop" in value_after + print() + if ok: + print("LIVE DEMO PASSED -- AgentMark Desktop drove real TextEdit end-to-end on macOS.") + return 0 + print("LIVE DEMO FAILED -- expected text not found in editor value.") + return 1 + finally: + try: + bridge.stdin.close() + bridge.wait(timeout=5) + except Exception: + bridge.kill() + stderr = bridge.stderr.read() + if stderr: + print() + print("--- bridge stderr ---") + for line in stderr.splitlines(): + print(f" {line}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/apps/agent-runner/bridges/macos/scripts/textedit-demo.sh b/apps/agent-runner/bridges/macos/scripts/textedit-demo.sh new file mode 100755 index 0000000..c157e39 --- /dev/null +++ b/apps/agent-runner/bridges/macos/scripts/textedit-demo.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# Live end-to-end demo: drive macOS TextEdit through the bridge. +# +# Sequence: +# 1. Ensure TextEdit is running with a new document +# 2. list_windows -> find the TextEdit window +# 3. capture -> locate the text_area element +# 4. execute type -> write a message into the editor +# 5. capture -> verify the text landed in element.value +# +# Requires Accessibility permission granted to the terminal you're +# running this from (System Settings -> Privacy & Security -> +# Accessibility). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BIN="$SCRIPT_DIR/../.build/debug/agentmark-bridge-macos" +TMP_FIFO="/tmp/agentmark-bridge-demo.$$" + +if [[ ! -x "$BIN" ]]; then + echo "Bridge not built. Run: swift build" >&2 + exit 1 +fi + +cleanup() { rm -f "$TMP_FIFO"; } +trap cleanup EXIT + +# ── Step 1: ensure TextEdit is running with a document ───────────────── +echo "Launching TextEdit..." +osascript -e 'tell application "TextEdit" to activate' -e 'tell application "TextEdit" to make new document' >/dev/null 2>&1 || true +sleep 1 + +# ── Spawn the bridge with a bidirectional FIFO ───────────────────────── +mkfifo "$TMP_FIFO" +# Background the bridge with stdin from the FIFO and stdout into a file. +"$BIN" < "$TMP_FIFO" > /tmp/agentmark-bridge-out.$$ 2>/tmp/agentmark-bridge-err.$$ & +BRIDGE_PID=$! + +# Open the FIFO for writing (keeps the pipe alive across calls). +exec 3>"$TMP_FIFO" + +send_request() { + echo "$1" >&3 + # Each response is a single line; wait for it to appear in the + # output file. + while ! tail -n 1 /tmp/agentmark-bridge-out.$$ 2>/dev/null | grep -q '"id":'"$2"; do + sleep 0.05 + done + tail -n 1 /tmp/agentmark-bridge-out.$$ +} + +teardown() { + exec 3>&- # close stdin to bridge + wait $BRIDGE_PID 2>/dev/null || true + cat /tmp/agentmark-bridge-err.$$ | sed 's/^/[bridge stderr] /' + rm -f /tmp/agentmark-bridge-out.$$ /tmp/agentmark-bridge-err.$$ +} +trap 'teardown; cleanup' EXIT + +# ── Step 2: list_windows ─────────────────────────────────────────────── +echo +echo "Step 2 -- list_windows..." +LIST=$(send_request '{"jsonrpc":"2.0","id":1,"method":"list_windows"}' 1) +WINDOW_ID=$(echo "$LIST" | python3 -c 'import sys,json; d=json.load(sys.stdin); w=[x for x in d["result"]["windows"] if x["processName"]=="TextEdit"]; print(w[0]["windowId"] if w else "")') +if [[ -z "$WINDOW_ID" ]]; then + echo "TextEdit window not found. Visible processes:" + echo "$LIST" | python3 -c 'import sys,json; d=json.load(sys.stdin); [print(" ", w["processName"], "--", w["windowTitle"]) for w in d["result"]["windows"]]' + exit 1 +fi +echo " found TextEdit window: $WINDOW_ID" + +# ── Step 3: capture ──────────────────────────────────────────────────── +echo +echo "Step 3 -- capture TextEdit..." +CAP=$(send_request "{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"capture\",\"params\":{\"windowId\":\"$WINDOW_ID\",\"maxDepth\":8,\"maxElements\":400}}" 2) +echo " treeDepth=$(echo "$CAP" | python3 -c 'import sys,json; print(json.load(sys.stdin)["result"]["treeDepth"])') elementCount=$(echo "$CAP" | python3 -c 'import sys,json; print(json.load(sys.stdin)["result"]["elementCount"])')" + +EDITOR_ID=$(echo "$CAP" | python3 - <<'PYEOF' +import sys, json +d = json.load(sys.stdin) +def find(node): + if node.get("role") in ("text_area", "text_input"): + return node["id"] + for c in node.get("children", []) or []: + r = find(c) + if r: return r + return None +print(find(d["result"]["root"]) or "") +PYEOF +) +if [[ -z "$EDITOR_ID" ]]; then + echo "Could not find text_area element. Tree:" + echo "$CAP" | python3 -m json.tool | head -100 + exit 1 +fi +echo " editor element_id: $EDITOR_ID" + +# ── Step 4: execute type ─────────────────────────────────────────────── +MESSAGE="Hello from AgentMark Desktop -- macOS bridge phase 0e3'" +echo +echo "Step 4 -- execute type..." +EXEC=$(send_request "{\"jsonrpc\":\"2.0\",\"id\":3,\"method\":\"execute\",\"params\":{\"elementId\":\"$EDITOR_ID\",\"actionType\":\"type\",\"text\":\"$MESSAGE\",\"clearFirst\":true}}" 3) +OK=$(echo "$EXEC" | python3 -c 'import sys,json; print(json.load(sys.stdin)["result"]["ok"])') +echo " ok: $OK" +echo " message: $(echo "$EXEC" | python3 -c 'import sys,json; r=json.load(sys.stdin)["result"]; print(r.get("message") or r.get("newValue") or "")')" + +sleep 0.5 + +# ── Step 5: re-capture and verify ───────────────────────────────────── +echo +echo "Step 5 -- re-capture and verify..." +CAP2=$(send_request "{\"jsonrpc\":\"2.0\",\"id\":4,\"method\":\"capture\",\"params\":{\"windowId\":\"$WINDOW_ID\",\"maxDepth\":8,\"maxElements\":400}}" 4) +VALUE=$(echo "$CAP2" | python3 - <