diff --git a/.gitignore b/.gitignore index cbf0247..29b5def 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,13 @@ obj/ *.user *.userprefs +# Swift Package Manager output (used by apps/agent-runner/bridges/macos) +.build/ +.swiftpm/ +Packages/ +*.xcodeproj +DerivedData/ + # JetBrains .idea/ *.iml diff --git a/apps/agent-runner/bridges/macos/Package.swift b/apps/agent-runner/bridges/macos/Package.swift new file mode 100644 index 0000000..8760d86 --- /dev/null +++ b/apps/agent-runner/bridges/macos/Package.swift @@ -0,0 +1,29 @@ +// swift-tools-version:5.9 +// +// AgentMark macOS bridge. Mirrors the Windows bridge architecture — +// stdio JSON-RPC 2.0 server, Node-side `MacosAxapiBackend` spawns this +// and proxies capture/execute calls. Phase 0e'1 here ships ping + +// capabilities; AXAPI capture/execute land in 0e'2/0e'3. + +import PackageDescription + +let package = Package( + name: "AgentMarkBridgeMacos", + platforms: [ + // AXAPI requires macOS; setting a recent baseline keeps the + // Accessibility APIs we'll need (kAXMainAttribute etc.) safe. + .macOS(.v13), + ], + products: [ + .executable( + name: "agentmark-bridge-macos", + targets: ["AgentMarkBridgeMacos"] + ), + ], + targets: [ + .executableTarget( + name: "AgentMarkBridgeMacos", + path: "Sources/AgentMarkBridgeMacos" + ), + ] +) diff --git a/apps/agent-runner/bridges/macos/README.md b/apps/agent-runner/bridges/macos/README.md new file mode 100644 index 0000000..6ce1d1f --- /dev/null +++ b/apps/agent-runner/bridges/macos/README.md @@ -0,0 +1,96 @@ +# AgentMark macOS AXAPI Bridge + +The macOS-side sidecar process for AgentMark Desktop. Mirrors the +Windows UIA bridge — same stdio JSON-RPC 2.0 protocol, same +`DesktopCaptureBackend` contract on the Node side. Uses the system +Accessibility framework (AXAPI) to walk and drive native macOS app UIs. + +## Build + +Requires the Swift toolchain (ships with Xcode Command Line Tools, or +`xcode-select --install`). From this directory: + +```bash +swift build +``` + +Output binary: `.build/debug/agentmark-bridge-macos`. + +For a release build with optimisations: + +```bash +swift build -c release +# Binary at .build/release/agentmark-bridge-macos +``` + +## Run + smoke test + +The bridge speaks stdio JSON-RPC 2.0. One JSON message per line. + +```bash +./scripts/smoke-test.sh +``` + +That pipes two requests (`ping` + `capabilities`) into the binary, +validates the responses, prints the bridge's stderr diagnostics. + +For interactive testing: + +```bash +.build/debug/agentmark-bridge-macos +``` + +It blocks on stdin. Paste: + +``` +{"jsonrpc":"2.0","id":1,"method":"ping"} +``` + +Expected (single line): + +```json +{"jsonrpc":"2.0","id":1,"result":{"pong":true,"version":"0.4.0","arch":"arm64","processId":12345}} +``` + +Ctrl-D closes stdin and the bridge exits cleanly. + +## Protocol + +All responses are JSON-RPC 2.0. Diagnostic output goes to stderr only — +stdout is reserved for framed JSON messages. + +| Method | Status | Description | +|---|---|---| +| `ping` | ✅ this phase | Liveness check, returns pong + bridge version | +| `capabilities` | ✅ this phase | Lists supported methods + AXAPI provider info | +| `list_windows` | planned | Enumerate top-level AXAPI windows via NSWorkspace + AXUIElement | +| `capture` | planned | Walk a window's AXAPI tree → `DesktopCapture` JSON | +| `execute` | planned | Drive an action by element_id (AXPress / AXSetValue / etc.) | + +The protocol — request/response envelopes, error codes, BOM-stripping — +is byte-identical to the Windows bridge. The Node-side +`MacosAxapiBackend` will be a near-exact mirror of `WindowsUiaBackend`. + +## Accessibility permission + +macOS guards AXAPI behind **System Settings → Privacy & Security → +Accessibility**. When the bridge starts walking trees (Phase 0e'2+), +the parent process (or this binary, if launched directly) must be +granted. The bridge will probe `AXIsProcessTrusted()` and surface a +clear `accessibilityNotGranted` JSON-RPC error if not. + +For developer machines: grant the terminal you launch from (Terminal / +iTerm / Warp). For production / Claude Desktop integration: Claude +Desktop itself must be granted (it's the parent process that spawns +this bridge). + +## Architecture + +- **Single-threaded.** AXAPI calls are not strictly main-thread-only but + CFRunLoop integration matters; for capture/execute we'll route through + a dedicated serial DispatchQueue. For ping/capabilities the entry-point + thread is fine. +- **UTF-8 only.** stdin / stdout treated as UTF-8 throughout; BOM stripped + defensively from the first stdin write. +- **stderr for diagnostics, stdout for framed JSON.** Same discipline as + the Windows bridge so log aggregation works in the AgentMark MCP server. diff --git a/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Axapi.swift b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Axapi.swift new file mode 100644 index 0000000..1684a37 --- /dev/null +++ b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Axapi.swift @@ -0,0 +1,226 @@ +// AXAPI plumbing — wraps AXUIElement, AXUIElementCopyAttributeValue, +// and friends in idiomatic Swift. Mirrors the Windows bridge's UIA +// patterns: defensive everywhere, swallows mid-walk failures, never +// throws on missing attributes. + +import Foundation +import ApplicationServices +import Cocoa + +// MARK: - Permission + +enum AccessibilityPermission { + /// True if the running process has been granted Accessibility + /// permission in System Settings → Privacy & Security → Accessibility. + /// Phase 0e2'/0e3' surfaces a clear JSON-RPC error if this is false. + static var isGranted: Bool { + return AXIsProcessTrusted() + } + + /// Triggers the system prompt (one-time) to grant Accessibility. + /// We don't call this from the bridge — the prompt would interrupt + /// the parent process. Useful for installers / first-run UI. + static func promptForAccess() -> Bool { + let options: NSDictionary = [ + kAXTrustedCheckOptionPrompt.takeUnretainedValue() as NSString: true + ] + return AXIsProcessTrustedWithOptions(options) + } +} + +// MARK: - AxElement + +/// Lightweight wrapper around an `AXUIElement`. Every attribute read +/// returns nil on any AXAPI error so a single stale element doesn't +/// abort the whole capture. +struct AxElement { + let element: AXUIElement + + init(_ element: AXUIElement) { + self.element = element + } + + // ---- Attribute readers ---- + + func string(_ attr: String) -> String? { + var value: CFTypeRef? + let err = AXUIElementCopyAttributeValue(element, attr as CFString, &value) + guard err == .success, let v = value else { return nil } + if CFGetTypeID(v) == CFStringGetTypeID() { + return v as? String + } + return nil + } + + func bool(_ attr: String) -> Bool? { + var value: CFTypeRef? + let err = AXUIElementCopyAttributeValue(element, attr as CFString, &value) + guard err == .success, let v = value else { return nil } + if CFGetTypeID(v) == CFBooleanGetTypeID() { + return (v as! CFBoolean) === kCFBooleanTrue + } + return nil + } + + /// Generic "value" extractor — AX exposes value as either string, + /// number, AXValue (CGPoint/Size/etc.), or bool depending on the role. + /// We canonicalise to a string representation that's useful for the + /// agent. + func value(_ attr: String = kAXValueAttribute as String) -> String? { + var value: CFTypeRef? + let err = AXUIElementCopyAttributeValue(element, attr as CFString, &value) + guard err == .success, let v = value else { return nil } + let typeId = CFGetTypeID(v) + if typeId == CFStringGetTypeID() { + return v as? String + } + if typeId == CFNumberGetTypeID() { + return "\(v as! NSNumber)" + } + if typeId == CFBooleanGetTypeID() { + return ((v as! CFBoolean) === kCFBooleanTrue) ? "true" : "false" + } + return nil + } + + func children(limit: Int = 5_000) -> [AxElement] { + var value: CFTypeRef? + let err = AXUIElementCopyAttributeValue(element, kAXChildrenAttribute as CFString, &value) + guard err == .success, let v = value else { return [] } + if CFGetTypeID(v) == CFArrayGetTypeID() { + let arr = v as! [AnyObject] + return arr.prefix(limit).compactMap { item in + // Each child should be an AXUIElement. CFGetTypeID() + // confirms; AXUIElementGetTypeID is the real test. + if CFGetTypeID(item) == AXUIElementGetTypeID() { + return AxElement(item as! AXUIElement) + } + return nil + } + } + return [] + } + + /// Screen-space bounds via kAXPositionAttribute + kAXSizeAttribute. + /// Returns nil when either is missing or fails to decode. + func bounds() -> (x: Double, y: Double, width: Double, height: Double)? { + var posRef: CFTypeRef? + let posErr = AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &posRef) + var sizeRef: CFTypeRef? + let sizeErr = AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeRef) + guard posErr == .success, let pos = posRef, + sizeErr == .success, let size = sizeRef else { return nil } + var point = CGPoint.zero + var sz = CGSize.zero + AXValueGetValue(pos as! AXValue, .cgPoint, &point) + AXValueGetValue(size as! AXValue, .cgSize, &sz) + if sz.width == 0 && sz.height == 0 { return nil } + return (Double(point.x), Double(point.y), Double(sz.width), Double(sz.height)) + } + + /// Stable id for the element. AXAPI has no AutomationId equivalent; + /// kAXIdentifierAttribute is occasionally populated but most apps + /// don't set it. Fall back to a content-derived hash so the same + /// element produces the same id across captures. + func stableId(role: String, fallbackIndex: Int) -> String { + if let id = string(kAXIdentifierAttribute as String), !id.isEmpty { + return id + } + // Hash inputs: role + title + value's first 32 chars + position. + let title = string(kAXTitleAttribute as String) ?? "" + let value = self.value() ?? "" + let trimmedValue = String(value.prefix(32)) + var pos = "?" + if let b = bounds() { + pos = "\(Int(b.x)),\(Int(b.y)),\(Int(b.width))x\(Int(b.height))" + } + let raw = "\(role)|\(title)|\(trimmedValue)|\(pos)|\(fallbackIndex)" + let hash = AxElement.djb2Hash(raw) + return String(format: "el_%08x", hash) + } + + static func djb2Hash(_ s: String) -> UInt32 { + var hash: UInt32 = 5381 + for byte in s.utf8 { + hash = ((hash << 5) &+ hash) &+ UInt32(byte) + } + return hash + } +} + +// MARK: - Role mapping + +enum RoleMapper { + /// Translates AXAPI role (and where helpful, subrole) into the + /// AgentMark v0.4 normalised `DesktopRole` vocabulary. Unknown + /// roles fall through to "other"; the body-builder on the Node + /// side renders these gracefully (recurses but doesn't surface + /// them as actions). + static func toRole(role: String?, subrole: String?) -> String { + guard let role = role else { return "other" } + + // Subrole takes priority for a few cases that change semantics. + if role == "AXTextField" && subrole == "AXSecureTextField" { + return "password_input" + } + if role == "AXTextField" && subrole == "AXSearchField" { + return "text_input" + } + if (role == "AXTextArea") || (role == "AXTextField" && subrole == "AXContentSearchField") { + return "text_area" + } + if role == "AXButton" && subrole == "AXToggleSubrole" { + // older apps use AXToggleSubrole for toggle buttons + return "button" + } + if role == "AXWindow" && (subrole == "AXDialog" || subrole == "AXSystemDialog") { + return "dialog" + } + + switch role { + case "AXWindow": return "window" + case "AXGroup": return "group" + case "AXSplitGroup": return "pane" + case "AXScrollArea": return "pane" + case "AXToolbar": return "toolbar" + case "AXMenuBar": return "menu" + case "AXMenu": return "menu" + case "AXMenuItem": return "menu_item" + case "AXMenuBarItem": return "menu_item" + case "AXMenuButton": return "split_button" + case "AXTabGroup": return "tab_list" + case "AXRadioGroup": return "group" + case "AXTab": return "tab" + case "AXOutline": return "tree" + case "AXOutlineRow": return "tree_item" + case "AXList": return "list" + case "AXListItem": return "list_item" + case "AXTable": return "table" + case "AXRow": return "row" + case "AXCell": return "cell" + case "AXColumn": return "column_header" + case "AXButton": return "button" + case "AXTextField": return "text_input" + case "AXTextArea": return "text_area" + case "AXCheckBox": return "check_box" + case "AXRadioButton": return "radio_button" + case "AXPopUpButton": return "combo_box" + case "AXComboBox": return "combo_box" + case "AXSlider": return "slider" + case "AXProgressIndicator": return "progress_bar" + case "AXLink": return "link" + case "AXStaticText": return "static_text" + case "AXImage": return "image" + case "AXSplitter": return "separator" + case "AXScrollBar": return "scroll_bar" + case "AXSheet": return "dialog" + case "AXDrawer": return "pane" + case "AXHelpTag": return "tooltip" + case "AXValueIndicator": return "static_text" + case "AXIncrementor": return "slider" + case "AXBusyIndicator": return "progress_bar" + case "AXDisclosureTriangle": return "button" + default: return "other" + } + } +} diff --git a/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/AxapiCapturer.swift b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/AxapiCapturer.swift new file mode 100644 index 0000000..bd89072 --- /dev/null +++ b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/AxapiCapturer.swift @@ -0,0 +1,672 @@ +// Core AXAPI work — list_windows + capture. Mirrors UiaCapturer.cs +// from the Windows bridge in behaviour and output shape. +// +// macOS-specific architectural notes: +// +// 1. Window enumeration goes via NSWorkspace.runningApplications +// (gives us regular apps + their pids) followed by +// AXUIElementCreateApplication(pid) → kAXWindowsAttribute. There's +// no Win32 HWND analog so we encode window ids as +// `axapi::` and re-resolve on each call. +// +// 2. AXAPI access requires Accessibility permission. We probe +// AXIsProcessTrusted() once at the start of every call that +// actually touches AXAPI. Without permission the bridge returns a +// clear JSON-RPC error with code -32020 (accessibilityNotGranted) +// pointing the user at System Settings. +// +// 3. macOS has no "AutomationId" — every element gets a content-derived +// stable hash via AxElement.stableId(). Survives across captures as +// long as title/value/position stay stable; works fine for the +// capture→execute round-trip the agent needs. + +import Foundation +import ApplicationServices +import Cocoa + +final class AxapiCapturer { + + /// Per-capture session — maps element_id → live AXUIElement so + /// execute() (Phase 0e3') can find the same element again. + final class Session { + let rootApp: AXUIElement + let rootWindow: AXUIElement + var elements: [String: AXUIElement] = [:] + init(rootApp: AXUIElement, rootWindow: AXUIElement) { + self.rootApp = rootApp + self.rootWindow = rootWindow + } + } + + private(set) var lastSession: Session? + + // MARK: - list_windows + + /// Enumerate top-level visible windows the bridge can see. + func listWindows() throws -> [[String: Any]] { + try requirePermission() + let focusedHint = focusedAppPid() + var out: [[String: Any]] = [] + + for app in NSWorkspace.shared.runningApplications { + // Skip background-only / agent-style apps; they rarely have + // windows worth surfacing and clutter the list. + if app.activationPolicy != .regular { continue } + let pid = app.processIdentifier + if pid <= 0 { continue } + + let appAx = AXUIElementCreateApplication(pid) + guard let windows = appWindows(appAx) else { continue } + + for (idx, win) in windows.enumerated() { + let el = AxElement(win) + let title = el.string(kAXTitleAttribute as String) ?? "" + // Skip chrome-less ghost windows. + if title.isEmpty { continue } + + let isMain = el.bool(kAXMainAttribute as String) ?? false + let isFocused = el.bool(kAXFocusedAttribute as String) ?? false + let hasFocus = pid == focusedHint && (isFocused || isMain) + + var summary: [String: Any] = [ + "windowId": encodeWindowId(pid: pid, index: idx), + "windowTitle": title, + "processName": app.localizedName ?? "(unknown)", + "processId": Int(pid), + "hasFocus": hasFocus, + ] + if let role = el.string(kAXRoleAttribute as String) { + // Surface the AX role as a hint; many GUI tools want this. + summary["windowClass"] = role + } + out.append(summary) + } + } + return out + } + + // MARK: - capture + + struct CaptureRequest { + var processName: String? + var processId: Int? + var windowTitle: String? + var windowId: String? + var maxDepth: Int = 12 + var includeHidden: Bool = false + var timeoutMs: Int = 5000 + var maxElements: Int = 2000 + } + + func capture(_ req: CaptureRequest) throws -> [String: Any] { + try requirePermission() + + guard let target = resolveTarget(req) else { + throw RpcError( + code: .windowNotFound, + message: "No matching window found and no focused window available." + ) + } + let (appAx, windowAx, pid, app) = target + + let session = Session(rootApp: appAx, rootWindow: windowAx) + var ctx = WalkContext( + maxDepth: max(1, req.maxDepth), + maxElements: max(50, req.maxElements), + includeHidden: req.includeHidden, + deadline: Date().addingTimeInterval(Double(max(500, req.timeoutMs)) / 1000.0) + ) + + let rootDto = walk(AxElement(windowAx), depth: 0, ctx: &ctx, session: session) + self.lastSession = session + + let focusedElementId = resolveFocusedElementId(in: session) + + var out: [String: Any] = [ + "platform": "macos", + "processName": app.localizedName ?? "(unknown)", + "processId": Int(pid), + "windowTitle": AxElement(windowAx).string(kAXTitleAttribute as String) ?? "(untitled window)", + "windowId": encodeWindowId(pid: pid, index: indexOf(window: windowAx, app: appAx) ?? 0), + "treeDepth": ctx.maxDepthReached, + "elementCount": ctx.elementCount, + "root": rootDto, + ] + if let role = AxElement(windowAx).string(kAXRoleAttribute as String) { + out["windowClass"] = role + } + if let fid = focusedElementId { + out["focusedElementId"] = fid + } + return out + } + + // MARK: - Walk + + private struct WalkContext { + let maxDepth: Int + let maxElements: Int + let includeHidden: Bool + let deadline: Date + var elementCount: Int = 0 + var maxDepthReached: Int = 0 + var seenIds: Set = [] + } + + private func walk( + _ el: AxElement, + depth: Int, + ctx: inout WalkContext, + session: Session + ) -> [String: Any] { + ctx.elementCount += 1 + if depth > ctx.maxDepthReached { ctx.maxDepthReached = depth } + + let role = el.string(kAXRoleAttribute as String) + let subrole = el.string(kAXSubroleAttribute as String) + let mappedRole = RoleMapper.toRole(role: role, subrole: subrole) + + var id = el.stableId(role: mappedRole, fallbackIndex: ctx.elementCount) + // Disambiguate hash collisions across siblings. + if ctx.seenIds.contains(id) { + var n = 2 + while ctx.seenIds.contains("\(id)#\(n)") { n += 1 } + id = "\(id)#\(n)" + } + ctx.seenIds.insert(id) + session.elements[id] = el.element + + var dto: [String: Any] = [ + "id": id, + "role": mappedRole, + ] + if let name = el.string(kAXTitleAttribute as String), !name.isEmpty { + dto["name"] = name + } else if let alt = el.string("AXDescription"), !alt.isEmpty { + dto["name"] = alt + } + if let v = el.value(), !v.isEmpty { + dto["value"] = v + } + if let placeholder = el.string("AXPlaceholderValue"), !placeholder.isEmpty { + dto["placeholder"] = placeholder + } + if let enabled = el.bool(kAXEnabledAttribute as String) { + dto["enabled"] = enabled + } + if let selected = el.bool(kAXSelectedAttribute as String) { + dto["selected"] = selected + } + if let expanded = el.bool(kAXExpandedAttribute as String) { + dto["expanded"] = expanded + } + if let bounds = el.bounds() { + dto["bounds"] = [ + "x": bounds.x, "y": bounds.y, + "width": bounds.width, "height": bounds.height, + ] + } + + // Stop expanding at caps. Parent still includes its own data + // but children get truncated; body-builder on the Node side + // renders the markdown gracefully when children is null. + if depth >= ctx.maxDepth { return dto } + if ctx.elementCount >= ctx.maxElements { return dto } + if Date() > ctx.deadline { return dto } + + let children = el.children() + if children.isEmpty { return dto } + + var kids: [[String: Any]] = [] + kids.reserveCapacity(children.count) + for c in children { + if ctx.elementCount >= ctx.maxElements { break } + if Date() > ctx.deadline { break } + // Off-screen / zero-size filter — children with no bounds + // are rare but exist (hidden helper elements); skip unless + // includeHidden is set. + if !ctx.includeHidden { + if let b = c.bounds(), b.width == 0 || b.height == 0 { continue } + } + kids.append(walk(c, depth: depth + 1, ctx: &ctx, session: session)) + } + if !kids.isEmpty { + dto["children"] = kids + } + return dto + } + + // MARK: - Target resolution + + private func resolveTarget(_ req: CaptureRequest) -> (AXUIElement, AXUIElement, pid_t, NSRunningApplication)? { + // 1. window_id (axapi::) is the most precise match. + if let id = req.windowId, let decoded = decodeWindowId(id) { + if let app = NSWorkspace.shared.runningApplications.first(where: { $0.processIdentifier == decoded.pid }) { + let appAx = AXUIElementCreateApplication(decoded.pid) + if let windows = appWindows(appAx), decoded.index < windows.count { + return (appAx, windows[decoded.index], decoded.pid, app) + } + } + } + + // 2. processId match + if let pid = req.processId { + if let app = NSWorkspace.shared.runningApplications.first(where: { $0.processIdentifier == pid_t(pid) }) { + let appAx = AXUIElementCreateApplication(pid_t(pid)) + if let win = firstMeaningfulWindow(appAx) { + return (appAx, win, pid_t(pid), app) + } + } + } + + // 3. processName match (substring, case-insensitive, .app suffix tolerant) + if let needle = req.processName?.lowercased().replacingOccurrences(of: ".app", with: "") { + for app in NSWorkspace.shared.runningApplications { + if app.activationPolicy != .regular { continue } + let name = (app.localizedName ?? "").lowercased() + let bundle = (app.bundleIdentifier ?? "").lowercased() + if name.contains(needle) || bundle.contains(needle) { + let appAx = AXUIElementCreateApplication(app.processIdentifier) + if let win = firstMeaningfulWindow(appAx) { + return (appAx, win, app.processIdentifier, app) + } + } + } + } + + // 4. windowTitle match + if let titleNeedle = req.windowTitle?.lowercased() { + for app in NSWorkspace.shared.runningApplications { + if app.activationPolicy != .regular { continue } + let appAx = AXUIElementCreateApplication(app.processIdentifier) + guard let windows = appWindows(appAx) else { continue } + for win in windows { + if let t = AxElement(win).string(kAXTitleAttribute as String), + t.lowercased().contains(titleNeedle) { + return (appAx, win, app.processIdentifier, app) + } + } + } + } + + // 5. Default: focused window. + if let pid = focusedAppPid(), + let app = NSWorkspace.shared.runningApplications.first(where: { $0.processIdentifier == pid }) { + let appAx = AXUIElementCreateApplication(pid) + if let win = focusedWindow(appAx) ?? firstMeaningfulWindow(appAx) { + return (appAx, win, pid, app) + } + } + return nil + } + + // MARK: - Helpers + + private func requirePermission() throws { + if !AccessibilityPermission.isGranted { + throw RpcError( + code: .accessibilityNotGranted, + message: "Accessibility permission not granted. Open System Settings → Privacy & Security → Accessibility and enable the process that launched this bridge (Claude Desktop, Terminal during dev, or the agentmark MCP server)." + ) + } + } + + private func appWindows(_ appAx: AXUIElement) -> [AXUIElement]? { + var value: CFTypeRef? + let err = AXUIElementCopyAttributeValue(appAx, kAXWindowsAttribute as CFString, &value) + guard err == .success, let v = value else { return nil } + if CFGetTypeID(v) == CFArrayGetTypeID() { + let arr = v as! [AnyObject] + return arr.compactMap { item in + if CFGetTypeID(item) == AXUIElementGetTypeID() { + return (item as! AXUIElement) + } + return nil + } + } + return nil + } + + private func firstMeaningfulWindow(_ appAx: AXUIElement) -> AXUIElement? { + guard let windows = appWindows(appAx) else { return nil } + for win in windows { + let el = AxElement(win) + if let t = el.string(kAXTitleAttribute as String), !t.isEmpty { + return win + } + } + return windows.first + } + + private func focusedWindow(_ appAx: AXUIElement) -> AXUIElement? { + var value: CFTypeRef? + let err = AXUIElementCopyAttributeValue(appAx, kAXFocusedWindowAttribute as CFString, &value) + guard err == .success, let v = value, CFGetTypeID(v) == AXUIElementGetTypeID() else { + return nil + } + return (v as! AXUIElement) + } + + private func focusedAppPid() -> pid_t? { + // NSWorkspace.frontmostApplication is the highest-fidelity + // signal — it tracks the actually-active window owner. + return NSWorkspace.shared.frontmostApplication?.processIdentifier + } + + private func resolveFocusedElementId(in session: Session) -> String? { + var value: CFTypeRef? + let err = AXUIElementCopyAttributeValue(session.rootApp, kAXFocusedUIElementAttribute as CFString, &value) + guard err == .success, let v = value, CFGetTypeID(v) == AXUIElementGetTypeID() else { + return nil + } + let focused = v as! AXUIElement + // Try to find that element in the captured session. + for (id, el) in session.elements { + if CFEqual(el, focused) { + return id + } + } + return nil + } + + private func indexOf(window: AXUIElement, app: AXUIElement) -> Int? { + guard let windows = appWindows(app) else { return nil } + for (i, w) in windows.enumerated() where CFEqual(w, window) { + return i + } + return nil + } + + // MARK: - Execute + + struct ExecuteRequest { + var elementId: String = "" + var actionType: String = "click" + var text: String? + var value: String? + var checked: Bool? + var expanded: Bool? + var key: String? + var modifiers: [String]? + var clearFirst: Bool = false + var timeoutMs: Int = 5000 + } + + func execute(_ req: ExecuteRequest) throws -> [String: Any] { + try requirePermission() + + guard let session = lastSession else { + return [ + "ok": false, + "message": "No capture session active. Call `capture` before `execute` so the bridge can resolve element_ids.", + ] + } + guard let element = session.elements[req.elementId] else { + return [ + "ok": false, + "message": "Unknown element_id `\(req.elementId)` in the current capture session. Re-capture if the window has changed.", + ] + } + + do { + switch req.actionType { + case "click": return try doClick(element) + case "type": return try doType(element, text: req.text ?? "", clearFirst: req.clearFirst) + case "select": return try doSelect(element, value: req.value ?? "") + case "check": return try doCheck(element, want: req.checked ?? true) + case "expand": return try doExpand(element, want: req.expanded ?? true) + case "focus": return try doFocus(element) + case "scroll_to": return try doScrollTo(element) + case "key": return try doKey(element, key: req.key ?? "", modifiers: req.modifiers ?? []) + default: + return ["ok": false, "message": "Unknown action type `\(req.actionType)`."] + } + } catch let rpcError as RpcError { + return ["ok": false, "message": rpcError.message] + } catch { + return ["ok": false, "message": "\(error)"] + } + } + + // ---- Per-action helpers ---- + + private func doClick(_ el: AXUIElement) throws -> [String: Any] { + // Try AXPress first (buttons, menu items, links). + if performAction(el, "AXPress") { + return ["ok": true] + } + // Toggle action (some checkboxes). + if performAction(el, "AXToggle") { + return ["ok": true] + } + // Pick (some popups / list items). + if performAction(el, "AXPick") { + return ["ok": true] + } + // Confirm (default button in a dialog). + if performAction(el, "AXConfirm") { + return ["ok": true] + } + // Show menu (for popup-button-style controls). + if performAction(el, "AXShowMenu") { + return ["ok": true, "message": "Used AXShowMenu fallback (element exposed no AXPress)."] + } + return ["ok": false, "message": "Element does not support any clickable AX action."] + } + + private func doType(_ el: AXUIElement, text: String, clearFirst: Bool) throws -> [String: Any] { + // AXAPI's standard text input is kAXValueAttribute as a String. + // Setting it via AXUIElementSetAttributeValue replaces content + // atomically; honour `clearFirst` by setting "" first when + // requested. + _ = setFocus(el) // best-effort; some apps require focus before SetValue is honoured + + if clearFirst { + _ = AXUIElementSetAttributeValue(el, kAXValueAttribute as CFString, "" as CFTypeRef) + } + let err = AXUIElementSetAttributeValue(el, kAXValueAttribute as CFString, text as CFTypeRef) + if err == .success { + let newValue = AxElement(el).value() ?? text + return ["ok": true, "newValue": newValue] + } + + // Fallback: synthesise keystrokes via CGEvent. Only usable when + // the element accepts focus and the host app honours regular + // keyboard input on the focused element. + if !setFocus(el) { + return ["ok": false, "message": "SetAttributeValue failed (\(err)) and element refused focus."] + } + if clearFirst { + sendKeyCombo(keyCode: 0x00 /* a */, modifiers: [.maskCommand]) + sendKey(keyCode: 0x33 /* delete */) + } + typeString(text) + return ["ok": true, "message": "Used keyboard fallback (AXValue not writable)."] + } + + private func doSelect(_ el: AXUIElement, value: String) throws -> [String: Any] { + // For selection-item elements (rows, menu items, tabs), perform AXPress. + if performAction(el, "AXPress") { return ["ok": true, "newValue": AxElement(el).string(kAXTitleAttribute as String) ?? value] } + // For popup buttons, set kAXValue directly. + let err = AXUIElementSetAttributeValue(el, kAXValueAttribute as CFString, value as CFTypeRef) + if err == .success { return ["ok": true, "newValue": value] } + return ["ok": false, "message": "Element does not support a selection action."] + } + + private func doCheck(_ el: AXUIElement, want: Bool) throws -> [String: Any] { + // Toggle until state matches (cap at 3 to avoid loops on + // tri-state controls). + var guardCount = 3 + while guardCount > 0 { + guardCount -= 1 + let current = AxElement(el).bool(kAXValueAttribute as String) ?? false + if current == want { break } + // Try AXPress first, then AXToggle. + if !performAction(el, "AXPress") && !performAction(el, "AXToggle") { + return ["ok": false, "message": "Element does not support AXPress/AXToggle."] + } + } + let final = AxElement(el).bool(kAXValueAttribute as String) ?? false + return ["ok": final == want, "newValue": final ? "true" : "false"] + } + + private func doExpand(_ el: AXUIElement, want: Bool) throws -> [String: Any] { + // Expanded state lives in kAXExpandedAttribute or kAXDisclosing. + let current = AxElement(el).bool(kAXExpandedAttribute as String) ?? AxElement(el).bool("AXDisclosing") ?? false + if current == want { + return ["ok": true, "newValue": want ? "Expanded" : "Collapsed"] + } + // Try setting the attribute directly (works for outline rows). + let setErr = AXUIElementSetAttributeValue(el, kAXExpandedAttribute as CFString, want as CFTypeRef) + if setErr == .success { + return ["ok": true, "newValue": want ? "Expanded" : "Collapsed"] + } + // Fall back to AXShowMenu / AXPress to toggle. + if performAction(el, "AXShowMenu") || performAction(el, "AXPress") { + return ["ok": true, "newValue": want ? "Expanded" : "Collapsed", "message": "Used AXPress fallback."] + } + return ["ok": false, "message": "Element does not support expand/collapse."] + } + + private func doFocus(_ el: AXUIElement) throws -> [String: Any] { + if setFocus(el) { return ["ok": true] } + return ["ok": false, "message": "Element refused focus."] + } + + private func doScrollTo(_ el: AXUIElement) throws -> [String: Any] { + if performAction(el, "AXScrollToVisible") { + return ["ok": true] + } + // SetFocus often implies scroll-into-view for most controls. + if setFocus(el) { + return ["ok": true, "message": "Used focus fallback (no AXScrollToVisible)."] + } + return ["ok": false, "message": "Element does not support AXScrollToVisible and refused focus."] + } + + private func doKey(_ el: AXUIElement, key: String, modifiers: [String]) throws -> [String: Any] { + if key.isEmpty { + return ["ok": false, "message": "Missing `key` argument."] + } + _ = setFocus(el) + guard let code = virtualKeyCode(forName: key) else { + // Not a named key — type the literal text. + typeString(key) + return ["ok": true, "message": "Typed literal text `\(key)`."] + } + let mods = modifiers.compactMap(modifierFlag(forName:)) + sendKeyCombo(keyCode: code, modifiers: mods) + return ["ok": true] + } + + // MARK: - AXAPI action helpers + + /// AXUIElementPerformAction returns success on no-op as well as on + /// real activation, so we don't try to interpret partial-failure. + /// Caller chains alternatives if `false` returned. + private func performAction(_ el: AXUIElement, _ action: String) -> Bool { + return AXUIElementPerformAction(el, action as CFString) == .success + } + + /// Best-effort focus. Some elements ignore kAXFocusedAttribute + /// (Document role, AXStaticText); not strictly necessary for the + /// caller to act on the result. + private func setFocus(_ el: AXUIElement) -> Bool { + let err = AXUIElementSetAttributeValue(el, kAXFocusedAttribute as CFString, kCFBooleanTrue) + return err == .success + } + + // MARK: - Keyboard simulation (CGEvent) + + private func typeString(_ s: String) { + // CGEvent supports unicode payload directly via + // CGEventKeyboardSetUnicodeString — works for most Latin + // and accented characters without per-character key-code + // lookup. + for ch in s { + guard let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true), + let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false) else { continue } + let unicodeChars = Array(String(ch).utf16) + down.keyboardSetUnicodeString(stringLength: unicodeChars.count, unicodeString: unicodeChars) + up.keyboardSetUnicodeString(stringLength: unicodeChars.count, unicodeString: unicodeChars) + down.post(tap: .cghidEventTap) + up.post(tap: .cghidEventTap) + } + } + + private func sendKey(keyCode: CGKeyCode) { + sendKeyCombo(keyCode: keyCode, modifiers: []) + } + + private func sendKeyCombo(keyCode: CGKeyCode, modifiers: [CGEventFlags]) { + let combined: CGEventFlags = modifiers.reduce(CGEventFlags(rawValue: 0)) { CGEventFlags(rawValue: $0.rawValue | $1.rawValue) } + if let down = CGEvent(keyboardEventSource: nil, virtualKey: keyCode, keyDown: true) { + down.flags = combined + down.post(tap: .cghidEventTap) + } + if let up = CGEvent(keyboardEventSource: nil, virtualKey: keyCode, keyDown: false) { + up.flags = combined + up.post(tap: .cghidEventTap) + } + } + + /// Mapping of friendly key names to macOS virtual key codes. + /// Values from Carbon's `Events.h` (`kVK_*`) — Apple still ships + /// these as the canonical key-code constants on Apple Silicon. + private func virtualKeyCode(forName name: String) -> CGKeyCode? { + switch name.lowercased() { + case "return", "enter": return 0x24 + case "tab": return 0x30 + case "space", "spacebar": return 0x31 + case "delete", "backspace": return 0x33 + case "escape", "esc": return 0x35 + case "left": return 0x7B + case "right": return 0x7C + case "down": return 0x7D + case "up": return 0x7E + case "home": return 0x73 + case "end": return 0x77 + case "pageup", "pgup": return 0x74 + case "pagedown", "pgdn": return 0x79 + case "f1": return 0x7A + case "f2": return 0x78 + case "f3": return 0x63 + case "f4": return 0x76 + case "f5": return 0x60 + case "f6": return 0x61 + case "f7": return 0x62 + case "f8": return 0x64 + case "f9": return 0x65 + case "f10": return 0x6D + case "f11": return 0x67 + case "f12": return 0x6F + default: return nil + } + } + + private func modifierFlag(forName name: String) -> CGEventFlags? { + switch name.lowercased() { + case "ctrl", "control": return .maskControl + case "alt", "option": return .maskAlternate + case "shift": return .maskShift + case "meta", "cmd", "command", "win", "windows": return .maskCommand + default: return nil + } + } + + // MARK: - Window ID encoding + + func encodeWindowId(pid: pid_t, index: Int) -> String { + return "axapi:\(pid):\(index)" + } + + func decodeWindowId(_ s: String) -> (pid: pid_t, index: Int)? { + guard s.hasPrefix("axapi:") else { return nil } + let parts = s.dropFirst("axapi:".count).split(separator: ":") + guard parts.count == 2, + let pid = pid_t(parts[0]), + let idx = Int(parts[1]) + else { return nil } + return (pid, idx) + } +} diff --git a/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Dispatcher.swift b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Dispatcher.swift new file mode 100644 index 0000000..72a0b6c --- /dev/null +++ b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Dispatcher.swift @@ -0,0 +1,105 @@ +// Method routing. New methods land here; the same name/shape as the +// Windows bridge so the Node-side `MacosAxapiBackend` (Phase 0f') can +// mirror `WindowsUiaBackend` exactly. + +import Foundation + +final class Dispatcher { + private let bridgeVersion: String + + // AXAPI is fine to call from the main thread; we don't currently + // need a dedicated serial queue. If we ever do, this is where the + // dispatch boundary goes. + private lazy var capturer = AxapiCapturer() + + init(bridgeVersion: String) { + self.bridgeVersion = bridgeVersion + } + + func dispatch(request: JsonRpcRequest) throws -> Any { + do { + switch request.method { + case "ping": + return handlePing() + case "capabilities": + return handleCapabilities() + case "list_windows": + return try handleListWindows() + case "capture": + return try handleCapture(params: request.params) + case "execute": + return try handleExecute(params: request.params) + default: + throw RpcError( + code: .methodNotFound, + message: "Unknown method: \(request.method). Supported: ping, capabilities, list_windows, capture, execute.", + requestId: request.id + ) + } + } catch var rpcError as RpcError { + rpcError.requestId = request.id + throw rpcError + } + } + + private func handlePing() -> Any { + return [ + "pong": true, + "version": bridgeVersion, + "arch": currentArchString(), + "processId": ProcessInfo.processInfo.processIdentifier, + ] as [String: Any] + } + + private func handleCapabilities() -> Any { + return [ + "bridge": "agentmark-bridge-macos", + "version": bridgeVersion, + "methods": ["ping", "capabilities", "list_windows", "capture", "execute"], + "axapiProvider": "Accessibility (AXAPI)", + "platform": "macos", + "accessibilityGranted": AccessibilityPermission.isGranted, + ] as [String: Any] + } + + private func handleListWindows() throws -> Any { + let windows = try capturer.listWindows() + return ["windows": windows] as [String: Any] + } + + private func handleCapture(params: JsonValue?) throws -> Any { + var req = AxapiCapturer.CaptureRequest() + if let p = params { + if let s = p.string("processName") { req.processName = s } + if let i = p.int("processId") { req.processId = i } + if let s = p.string("windowTitle") { req.windowTitle = s } + if let s = p.string("windowId") { req.windowId = s } + if let i = p.int("maxDepth") { req.maxDepth = i } + if let b = p.bool("includeHidden") { req.includeHidden = b } + if let i = p.int("timeoutMs") { req.timeoutMs = i } + if let i = p.int("maxElements") { req.maxElements = i } + } + return try capturer.capture(req) + } + + private func handleExecute(params: JsonValue?) throws -> Any { + guard let p = params, + let elementId = p.string("elementId"), !elementId.isEmpty else { + throw RpcError(code: .invalidParams, message: "execute requires `elementId`.") + } + var req = AxapiCapturer.ExecuteRequest() + req.elementId = elementId + req.actionType = p.string("actionType") ?? "click" + req.text = p.string("text") + req.value = p.string("value") + req.checked = p.bool("checked") + req.expanded = p.bool("expanded") + req.key = p.string("key") + if let modsAny = p.dict?["modifiers"], case .array(let arr) = JsonValue.fromAny(modsAny) { + req.modifiers = arr.compactMap { $0 as? String } + } + req.clearFirst = p.bool("clearFirst") ?? false + if let i = p.int("timeoutMs") { req.timeoutMs = i } + return try capturer.execute(req) + } +} diff --git a/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/JsonRpc.swift b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/JsonRpc.swift new file mode 100644 index 0000000..3c6bded --- /dev/null +++ b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/JsonRpc.swift @@ -0,0 +1,189 @@ +// JSON-RPC 2.0 wire types. Identical envelope to the Windows bridge. + +import Foundation + +// MARK: - Request + +/// JSON-RPC 2.0 request. We accept any JSON value for `id` (number, +/// string, or null); store it raw and echo it back unmodified on the +/// response so different clients (number ids, uuid string ids) all work. +struct JsonRpcRequest { + let id: JsonRpcId + let method: String + let params: JsonValue? + + static func decode(jsonString: String) throws -> JsonRpcRequest { + guard let data = jsonString.data(using: .utf8) else { + throw RpcError(code: RpcErrorCode.parseError, message: "Input is not valid UTF-8") + } + guard let any = try? JSONSerialization.jsonObject(with: data, options: [.allowFragments]) else { + throw RpcError(code: RpcErrorCode.parseError, message: "Input is not valid JSON") + } + guard let obj = any as? [String: Any] else { + throw RpcError(code: RpcErrorCode.invalidRequest, message: "JSON-RPC request must be an object") + } + guard let method = obj["method"] as? String, !method.isEmpty else { + throw RpcError(code: RpcErrorCode.invalidRequest, message: "Missing `method`") + } + let id = JsonRpcId.fromAny(obj["id"]) + let params: JsonValue? = obj["params"].flatMap(JsonValue.fromAny) + return JsonRpcRequest(id: id, method: method, params: params) + } +} + +// MARK: - Response + +struct JsonRpcResponse { + let id: JsonRpcId + let kind: Kind + + enum Kind { + case success(result: Any) + case error(code: Int, message: String) + } + + static func success(id: JsonRpcId, result: Any) -> JsonRpcResponse { + JsonRpcResponse(id: id, kind: .success(result: result)) + } + + static func error(id: JsonRpcId, code: Int, message: String) -> JsonRpcResponse { + JsonRpcResponse(id: id, kind: .error(code: code, message: message)) + } + + func encode() throws -> String { + var dict: [String: Any] = [ + "jsonrpc": "2.0", + "id": id.toAny(), + ] + switch kind { + case .success(let result): + dict["result"] = result + case .error(let code, let message): + dict["error"] = [ + "code": code, + "message": message, + ] + } + let data = try JSONSerialization.data( + withJSONObject: dict, + options: [.withoutEscapingSlashes] + ) + guard let s = String(data: data, encoding: .utf8) else { + throw RpcError(code: RpcErrorCode.internalError, message: "Failed to serialise response as UTF-8") + } + return s + } +} + +// MARK: - JsonRpcId + +/// JSON-RPC ids can be number, string, or null. Hold the original +/// representation so we round-trip exactly. +enum JsonRpcId { + case number(Double) + case string(String) + case null + + static func fromAny(_ value: Any?) -> JsonRpcId { + if value == nil { return .null } + if let v = value as? NSNumber { + // NSNumber covers Int/Double/Bool from JSONSerialization + if CFGetTypeID(v) == CFBooleanGetTypeID() { return .null } + return .number(v.doubleValue) + } + if let s = value as? String { return .string(s) } + return .null + } + + func toAny() -> Any { + switch self { + case .number(let d): + // Emit ints as ints when possible so clients see id=1, not id=1.0. + if d.truncatingRemainder(dividingBy: 1) == 0 + && d >= Double(Int.min) && d <= Double(Int.max) { + return Int(d) + } + return d + case .string(let s): + return s + case .null: + return NSNull() + } + } +} + +// MARK: - JsonValue + +/// Lightweight wrapper for arbitrary JSON values plucked from the +/// request's `params` field. Most handlers project the few fields they +/// need out via the `dict`/`string(at:)` etc. helpers. +enum JsonValue { + case object([String: Any]) + case array([Any]) + case string(String) + case number(Double) + case bool(Bool) + case null + + static func fromAny(_ value: Any) -> JsonValue { + if let d = value as? [String: Any] { return .object(d) } + if let a = value as? [Any] { return .array(a) } + if let s = value as? String { return .string(s) } + if let n = value as? NSNumber { + if CFGetTypeID(n) == CFBooleanGetTypeID() { return .bool(n.boolValue) } + return .number(n.doubleValue) + } + return .null + } + + var dict: [String: Any]? { + if case .object(let d) = self { return d } + return nil + } + + func string(_ key: String) -> String? { + return (dict?[key] as? String) + } + + func int(_ key: String) -> Int? { + if let n = dict?[key] as? NSNumber { return n.intValue } + return nil + } + + func bool(_ key: String) -> Bool? { + if let n = dict?[key] as? NSNumber, + CFGetTypeID(n) == CFBooleanGetTypeID() { + return n.boolValue + } + return nil + } +} + +// MARK: - Errors + +enum RpcErrorCode: Int { + case parseError = -32700 + case invalidRequest = -32600 + case methodNotFound = -32601 + case invalidParams = -32602 + case internalError = -32603 + + // Bridge-specific (above -32000) + case windowNotFound = -32010 + case elementNotFound = -32011 + case unsupportedPattern = -32012 + case actionFailed = -32013 + case accessibilityNotGranted = -32020 +} + +struct RpcError: Error { + let code: Int + let message: String + var requestId: JsonRpcId = .null + + init(code: RpcErrorCode, message: String, requestId: JsonRpcId = .null) { + self.code = code.rawValue + self.message = message + self.requestId = requestId + } +} diff --git a/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/main.swift b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/main.swift new file mode 100644 index 0000000..67f6c97 --- /dev/null +++ b/apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/main.swift @@ -0,0 +1,91 @@ +// AgentMark macOS AXAPI Bridge +// ---------------------------- +// Stdio JSON-RPC 2.0 server. Parent process (typically the Node-side +// `MacosAxapiBackend` running inside the AgentMark MCP server) launches +// this binary and talks to it over stdin/stdout. One line of JSON per +// message. +// +// Protocol — identical to the Windows bridge: +// Request : { "jsonrpc": "2.0", "id": , "method": "", "params": {...} } +// Response: { "jsonrpc": "2.0", "id": , "result": {...} } on success +// { "jsonrpc": "2.0", "id": , "error": { "code": N, "message": "..." } } on failure +// +// All diagnostic output goes to stderr — never stdout — so the framing +// stays clean. +// +// Methods (this phase): +// ping — returns { pong, version, arch, processId } +// capabilities— returns { bridge, version, methods, axapiProvider, platform } +// +// Future: +// list_windows— enumerate top-level windows visible to AXAPI +// capture — walk AXAPI tree → DesktopCapture JSON +// execute — drive an action by element_id (AXPress / AXSetValue / etc.) +// +// AXAPI permission note: macOS guards Accessibility API access behind +// System Settings → Privacy & Security → Accessibility. The parent +// process (Claude Desktop, AgentMark MCP server) must be granted +// permission OR this bridge binary itself must be granted. We probe +// AXIsProcessTrusted() on startup and surface a clear error on first +// list_windows/capture if not granted. + +import Foundation + +let bridgeVersion = "0.4.0" + +let stderr = FileHandle.standardError +stderr.write("[bridge] agentmark-bridge-macos starting (pid=\(ProcessInfo.processInfo.processIdentifier), arch=\(currentArchString()))\n".data(using: .utf8)!) + +let dispatcher = Dispatcher(bridgeVersion: bridgeVersion) + +// Read newline-delimited JSON from stdin until EOF. +while let line = readLine(strippingNewline: true) { + let trimmed = stripBom(line.trimmingCharacters(in: .whitespacesAndNewlines)) + if trimmed.isEmpty { continue } + + do { + let request = try JsonRpcRequest.decode(jsonString: trimmed) + let result = try dispatcher.dispatch(request: request) + let response = JsonRpcResponse.success(id: request.id, result: result) + writeLine(try response.encode()) + } catch let rpcError as RpcError { + let response = JsonRpcResponse.error(id: rpcError.requestId, code: rpcError.code, message: rpcError.message) + writeLine((try? response.encode()) ?? "{}") + } catch { + // Last-resort fallback for genuinely unexpected exceptions. + let response = JsonRpcResponse.error( + id: .null, + code: RpcErrorCode.internalError.rawValue, + message: "Unhandled: \(error)" + ) + writeLine((try? response.encode()) ?? "{}") + stderr.write("[bridge] unhandled: \(error)\n".data(using: .utf8)!) + } +} + +stderr.write("[bridge] stdin closed; exiting\n".data(using: .utf8)!) + +// MARK: - I/O helpers + +func writeLine(_ s: String) { + FileHandle.standardOutput.write((s + "\n").data(using: .utf8)!) +} + +/// Strips the U+FEFF byte-order mark some clients (PowerShell, certain +/// shells) prepend to the first stdin write. Keeps JSON parsing happy. +func stripBom(_ s: String) -> String { + if s.first == "\u{FEFF}" { + return String(s.dropFirst()) + } + return s +} + +func currentArchString() -> String { + #if arch(arm64) + return "arm64" + #elseif arch(x86_64) + return "x86_64" + #else + return "unknown" + #endif +} diff --git a/apps/agent-runner/bridges/macos/scripts/smoke-test.sh b/apps/agent-runner/bridges/macos/scripts/smoke-test.sh new file mode 100755 index 0000000..3677439 --- /dev/null +++ b/apps/agent-runner/bridges/macos/scripts/smoke-test.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Smoke test for agentmark-bridge-macos. +# +# Feeds two requests through stdin (ping + capabilities), asserts both +# come back with the expected shape, exits non-zero on failure. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BIN="$SCRIPT_DIR/../.build/debug/agentmark-bridge-macos" + +if [[ ! -x "$BIN" ]]; then + echo "Bridge binary not built. Expected at $BIN" >&2 + echo "Run: cd $(dirname "$SCRIPT_DIR") && swift build" >&2 + exit 1 +fi + +echo "Smoke testing $BIN" +echo + +# Two requests separated by newlines, then close stdin. +REQUESTS='{"jsonrpc":"2.0","id":1,"method":"ping"} +{"jsonrpc":"2.0","id":2,"method":"capabilities"}' + +# Run with a 10-second wall-clock cap. +OUTPUT="$(echo "$REQUESTS" | "$BIN" 2>/tmp/agentmark-bridge-macos-stderr.log)" \ + || { echo "Bridge exited non-zero. stderr:"; cat /tmp/agentmark-bridge-macos-stderr.log; exit 1; } + +LINES=() +while IFS= read -r line; do + [[ -n "$line" ]] && LINES+=("$line") +done <<< "$OUTPUT" + +if [[ ${#LINES[@]} -ne 2 ]]; then + echo "Expected 2 response lines; got ${#LINES[@]}." >&2 + echo "Output was:" >&2 + printf '%s\n' "${LINES[@]}" >&2 + exit 1 +fi + +for L in "${LINES[@]}"; do echo " -> $L"; done +echo + +# Validate basic shape with grep — no jq dependency. +if ! grep -q '"pong":true' <<< "${LINES[0]}"; then + echo "ping: expected result.pong=true" >&2 + exit 1 +fi +if ! grep -q '"id":1' <<< "${LINES[0]}"; then + echo "ping: id mismatch (expected 1)" >&2 + exit 1 +fi +if ! grep -q '"bridge":"agentmark-bridge-macos"' <<< "${LINES[1]}"; then + echo "capabilities: expected bridge=agentmark-bridge-macos" >&2 + exit 1 +fi +if ! grep -q '"id":2' <<< "${LINES[1]}"; then + echo "capabilities: id mismatch (expected 2)" >&2 + exit 1 +fi + +echo "SMOKE TEST PASSED" +echo " bridge stderr:" +sed 's/^/ /' /tmp/agentmark-bridge-macos-stderr.log diff --git a/apps/agent-runner/bridges/macos/scripts/textedit-demo.py b/apps/agent-runner/bridges/macos/scripts/textedit-demo.py new file mode 100755 index 0000000..fa79749 --- /dev/null +++ b/apps/agent-runner/bridges/macos/scripts/textedit-demo.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +""" +Live end-to-end demo: drive macOS TextEdit through the bridge. + +Sequence: + 1. Ensure TextEdit is running with a new document + 2. list_windows -> find the TextEdit window + 3. capture -> locate the text_area element + 4. execute type -> write a message into the editor + 5. capture -> verify the text landed in element.value + +Requires Accessibility permission granted to the terminal you're +running this from (System Settings -> Privacy & Security -> +Accessibility). +""" + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +SCRIPT_DIR = Path(__file__).parent.resolve() +BIN = SCRIPT_DIR.parent / ".build" / "debug" / "agentmark-bridge-macos" + +if not BIN.exists(): + print(f"Bridge not built. Expected at {BIN}\nRun: swift build", file=sys.stderr) + sys.exit(1) + + +def main() -> int: + # Step 1: ensure TextEdit is running with a doc. + print("Launching TextEdit...") + subprocess.run( + [ + "osascript", + "-e", 'tell application "TextEdit" to activate', + "-e", 'tell application "TextEdit" to make new document', + ], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + time.sleep(1) + + # Spawn bridge with bidirectional pipes. + bridge = subprocess.Popen( + [str(BIN)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + next_id = 1 + + def call(method: str, params: dict | None = None) -> dict: + nonlocal next_id + req = {"jsonrpc": "2.0", "id": next_id, "method": method} + if params is not None: + req["params"] = params + next_id += 1 + bridge.stdin.write(json.dumps(req) + "\n") + bridge.stdin.flush() + line = bridge.stdout.readline() + if not line: + err = bridge.stderr.read() + raise RuntimeError(f"Bridge closed unexpectedly. stderr:\n{err}") + return json.loads(line) + + def find_editor(node: dict) -> str | None: + if node.get("role") in ("text_area", "text_input"): + return node["id"] + for child in node.get("children") or []: + found = find_editor(child) + if found: + return found + return None + + def find_value(node: dict, target_id: str) -> str | None: + if node.get("id") == target_id: + return node.get("value") + for child in node.get("children") or []: + v = find_value(child, target_id) + if v is not None: + return v + return None + + try: + # Step 2: list_windows + print() + print("Step 2 -- list_windows...") + list_resp = call("list_windows") + windows = list_resp["result"]["windows"] + textedit = next( + (w for w in windows if w.get("processName") == "TextEdit"), + None, + ) + if textedit is None: + print("TextEdit window not found. Visible processes:") + for w in windows: + print(f" {w.get('processName'):25s} -- {w.get('windowTitle')}") + return 1 + window_id = textedit["windowId"] + print(f" found TextEdit window: {window_id} ({textedit.get('windowTitle')})") + + # Step 3: capture + print() + print("Step 3 -- capture TextEdit...") + cap = call("capture", { + "windowId": window_id, + "maxDepth": 8, + "maxElements": 400, + }) + if "error" in cap: + print(f" capture errored: {cap['error']}") + return 1 + result = cap["result"] + print(f" treeDepth={result['treeDepth']} elementCount={result['elementCount']}") + editor_id = find_editor(result["root"]) + if not editor_id: + print(" Could not find a text_area / text_input element. Tree top level:") + for c in (result["root"].get("children") or [])[:10]: + print(f" [{c.get('role'):15s}] id={c.get('id'):20s} name={c.get('name')}") + return 1 + print(f" editor element_id: {editor_id}") + + # Step 4: execute type + message = "Hello from AgentMark Desktop -- macOS bridge phase 0e3'" + print() + print("Step 4 -- execute type...") + exec_resp = call("execute", { + "elementId": editor_id, + "actionType": "type", + "text": message, + "clearFirst": True, + }) + if "error" in exec_resp: + print(f" execute errored: {exec_resp['error']}") + return 1 + r = exec_resp["result"] + print(f" ok: {r.get('ok')}") + if r.get("message"): + print(f" note: {r['message']}") + if r.get("newValue"): + preview = r["newValue"][:80] + print(f" newValue: {preview}") + + time.sleep(0.5) + + # Step 5: re-capture and verify + print() + print("Step 5 -- re-capture and verify...") + cap2 = call("capture", { + "windowId": window_id, + "maxDepth": 8, + "maxElements": 400, + }) + value_after = find_value(cap2["result"]["root"], editor_id) + print(f" editor value after type: {repr(value_after)[:120]}") + + ok = value_after and "Hello from AgentMark Desktop" in value_after + print() + if ok: + print("LIVE DEMO PASSED -- AgentMark Desktop drove real TextEdit end-to-end on macOS.") + return 0 + print("LIVE DEMO FAILED -- expected text not found in editor value.") + return 1 + finally: + try: + bridge.stdin.close() + bridge.wait(timeout=5) + except Exception: + bridge.kill() + stderr = bridge.stderr.read() + if stderr: + print() + print("--- bridge stderr ---") + for line in stderr.splitlines(): + print(f" {line}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/apps/agent-runner/bridges/macos/scripts/textedit-demo.sh b/apps/agent-runner/bridges/macos/scripts/textedit-demo.sh new file mode 100755 index 0000000..c157e39 --- /dev/null +++ b/apps/agent-runner/bridges/macos/scripts/textedit-demo.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# Live end-to-end demo: drive macOS TextEdit through the bridge. +# +# Sequence: +# 1. Ensure TextEdit is running with a new document +# 2. list_windows -> find the TextEdit window +# 3. capture -> locate the text_area element +# 4. execute type -> write a message into the editor +# 5. capture -> verify the text landed in element.value +# +# Requires Accessibility permission granted to the terminal you're +# running this from (System Settings -> Privacy & Security -> +# Accessibility). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BIN="$SCRIPT_DIR/../.build/debug/agentmark-bridge-macos" +TMP_FIFO="/tmp/agentmark-bridge-demo.$$" + +if [[ ! -x "$BIN" ]]; then + echo "Bridge not built. Run: swift build" >&2 + exit 1 +fi + +cleanup() { rm -f "$TMP_FIFO"; } +trap cleanup EXIT + +# ── Step 1: ensure TextEdit is running with a document ───────────────── +echo "Launching TextEdit..." +osascript -e 'tell application "TextEdit" to activate' -e 'tell application "TextEdit" to make new document' >/dev/null 2>&1 || true +sleep 1 + +# ── Spawn the bridge with a bidirectional FIFO ───────────────────────── +mkfifo "$TMP_FIFO" +# Background the bridge with stdin from the FIFO and stdout into a file. +"$BIN" < "$TMP_FIFO" > /tmp/agentmark-bridge-out.$$ 2>/tmp/agentmark-bridge-err.$$ & +BRIDGE_PID=$! + +# Open the FIFO for writing (keeps the pipe alive across calls). +exec 3>"$TMP_FIFO" + +send_request() { + echo "$1" >&3 + # Each response is a single line; wait for it to appear in the + # output file. + while ! tail -n 1 /tmp/agentmark-bridge-out.$$ 2>/dev/null | grep -q '"id":'"$2"; do + sleep 0.05 + done + tail -n 1 /tmp/agentmark-bridge-out.$$ +} + +teardown() { + exec 3>&- # close stdin to bridge + wait $BRIDGE_PID 2>/dev/null || true + cat /tmp/agentmark-bridge-err.$$ | sed 's/^/[bridge stderr] /' + rm -f /tmp/agentmark-bridge-out.$$ /tmp/agentmark-bridge-err.$$ +} +trap 'teardown; cleanup' EXIT + +# ── Step 2: list_windows ─────────────────────────────────────────────── +echo +echo "Step 2 -- list_windows..." +LIST=$(send_request '{"jsonrpc":"2.0","id":1,"method":"list_windows"}' 1) +WINDOW_ID=$(echo "$LIST" | python3 -c 'import sys,json; d=json.load(sys.stdin); w=[x for x in d["result"]["windows"] if x["processName"]=="TextEdit"]; print(w[0]["windowId"] if w else "")') +if [[ -z "$WINDOW_ID" ]]; then + echo "TextEdit window not found. Visible processes:" + echo "$LIST" | python3 -c 'import sys,json; d=json.load(sys.stdin); [print(" ", w["processName"], "--", w["windowTitle"]) for w in d["result"]["windows"]]' + exit 1 +fi +echo " found TextEdit window: $WINDOW_ID" + +# ── Step 3: capture ──────────────────────────────────────────────────── +echo +echo "Step 3 -- capture TextEdit..." +CAP=$(send_request "{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"capture\",\"params\":{\"windowId\":\"$WINDOW_ID\",\"maxDepth\":8,\"maxElements\":400}}" 2) +echo " treeDepth=$(echo "$CAP" | python3 -c 'import sys,json; print(json.load(sys.stdin)["result"]["treeDepth"])') elementCount=$(echo "$CAP" | python3 -c 'import sys,json; print(json.load(sys.stdin)["result"]["elementCount"])')" + +EDITOR_ID=$(echo "$CAP" | python3 - <<'PYEOF' +import sys, json +d = json.load(sys.stdin) +def find(node): + if node.get("role") in ("text_area", "text_input"): + return node["id"] + for c in node.get("children", []) or []: + r = find(c) + if r: return r + return None +print(find(d["result"]["root"]) or "") +PYEOF +) +if [[ -z "$EDITOR_ID" ]]; then + echo "Could not find text_area element. Tree:" + echo "$CAP" | python3 -m json.tool | head -100 + exit 1 +fi +echo " editor element_id: $EDITOR_ID" + +# ── Step 4: execute type ─────────────────────────────────────────────── +MESSAGE="Hello from AgentMark Desktop -- macOS bridge phase 0e3'" +echo +echo "Step 4 -- execute type..." +EXEC=$(send_request "{\"jsonrpc\":\"2.0\",\"id\":3,\"method\":\"execute\",\"params\":{\"elementId\":\"$EDITOR_ID\",\"actionType\":\"type\",\"text\":\"$MESSAGE\",\"clearFirst\":true}}" 3) +OK=$(echo "$EXEC" | python3 -c 'import sys,json; print(json.load(sys.stdin)["result"]["ok"])') +echo " ok: $OK" +echo " message: $(echo "$EXEC" | python3 -c 'import sys,json; r=json.load(sys.stdin)["result"]; print(r.get("message") or r.get("newValue") or "")')" + +sleep 0.5 + +# ── Step 5: re-capture and verify ───────────────────────────────────── +echo +echo "Step 5 -- re-capture and verify..." +CAP2=$(send_request "{\"jsonrpc\":\"2.0\",\"id\":4,\"method\":\"capture\",\"params\":{\"windowId\":\"$WINDOW_ID\",\"maxDepth\":8,\"maxElements\":400}}" 4) +VALUE=$(echo "$CAP2" | python3 - < void + reject: (error: Error) => void + method: string + timeout: NodeJS.Timeout +} + +interface JsonRpcResponse { + jsonrpc: '2.0' + id: number + result?: unknown + error?: { code: number; message: string } +} + +export class MacosAxapiBackend implements DesktopCaptureBackend { + readonly name = 'macos_axapi' + + private readonly bridgePath: string + private readonly logger: Logger + private readonly startupTimeoutMs: number + private readonly callTimeoutMs: number + + private proc: ChildProcessByStdio | null = null + private rl: ReadlineInterface | null = null + private pending = new Map() + private nextId = 1 + private starting: Promise | null = null + private closed = false + + constructor(opts: MacosAxapiBackendOptions = {}) { + if (process.platform !== 'darwin' && !opts.allowNonMac) { + throw new Error( + `MacosAxapiBackend requires macOS (process.platform=='darwin'). ` + + `Current platform: ${process.platform}. For tests that supply ` + + `a fake bridge, pass allowNonMac: true.`, + ) + } + this.bridgePath = opts.bridgePath ?? resolveBridgePath() + this.logger = opts.logger ?? noopLogger + this.startupTimeoutMs = opts.startupTimeoutMs ?? 10_000 + this.callTimeoutMs = opts.callTimeoutMs ?? 30_000 + } + + // ── DesktopCaptureBackend implementation ───────────────────────── + + async capture(opts: CaptureDesktopOptions = {}): Promise { + await this.ensureStarted() + const params = { + processName: opts.target?.process_name, + processId: opts.target?.process_id, + windowTitle: opts.target?.window_title, + windowId: opts.target?.window_id, + maxDepth: opts.maxDepth, + includeHidden: opts.includeHidden, + timeoutMs: opts.timeoutMs, + } + const result = (await this.call('capture', params)) as RawDesktopCapture + return mapCaptureResponse(result) + } + + async execute(opts: ExecuteDesktopOptions): Promise { + await this.ensureStarted() + const params = buildExecuteParams(opts.action) + if (opts.timeoutMs !== undefined) params.timeoutMs = opts.timeoutMs + + const result = (await this.call('execute', params)) as RawExecuteResult + return { + ok: !!result.ok, + message: result.message ?? undefined, + new_value: result.newValue ?? undefined, + } + } + + async close(): Promise { + this.closed = true + const proc = this.proc + if (!proc) return + + for (const [id, slot] of this.pending) { + clearTimeout(slot.timeout) + slot.reject(new Error(`Bridge closed before ${slot.method} (id=${id}) completed`)) + } + this.pending.clear() + + try { proc.stdin.end() } catch { /* swallow */ } + + await new Promise((resolve) => { + const timeout = setTimeout(() => { + try { proc.kill('SIGKILL') } catch { /* swallow */ } + resolve() + }, 2000) + proc.once('exit', () => { + clearTimeout(timeout) + resolve() + }) + }) + + this.proc = null + this.rl?.close() + this.rl = null + } + + // ── Lifecycle ───────────────────────────────────────────────────── + + private ensureStarted(): Promise { + if (this.closed) { + return Promise.reject(new Error('MacosAxapiBackend was closed; create a new instance.')) + } + if (this.proc) return Promise.resolve() + if (this.starting) return this.starting + + this.starting = this.spawnAndHandshake() + .catch((err) => { + this.starting = null + throw err + }) + .finally(() => { + if (this.proc) this.starting = null + }) + return this.starting + } + + private async spawnAndHandshake(): Promise { + this.logger.debug('macos-axapi.spawn', { bridgePath: this.bridgePath }) + + const proc = spawn(this.bridgePath, [], { + stdio: ['pipe', 'pipe', 'pipe'], + }) as ChildProcessByStdio + + this.proc = proc + + proc.on('error', (err) => { + this.logger.error('macos-axapi.spawn-error', { error: err.message }) + this.failAllPending(new Error(`Bridge process error: ${err.message}`)) + }) + + proc.stderr.setEncoding('utf8') + proc.stderr.on('data', (chunk: string) => { + const lines = chunk.split(/\r?\n/).filter((l) => l.length > 0) + for (const line of lines) { + this.logger.debug('macos-axapi.bridge-stderr', { line }) + } + }) + + proc.on('exit', (code, signal) => { + this.logger.info('macos-axapi.bridge-exit', { code, signal }) + this.failAllPending(new Error(`Bridge process exited (code=${code}, signal=${signal})`)) + this.proc = null + this.rl?.close() + this.rl = null + }) + + const rl = createInterface({ input: proc.stdout }) + this.rl = rl + rl.on('line', (line) => this.handleResponseLine(line)) + + try { + await this.callWithTimeout('ping', {}, this.startupTimeoutMs) + } catch (err) { + try { proc.kill('SIGKILL') } catch { /* swallow */ } + this.proc = null + this.rl?.close() + this.rl = null + throw new Error(`Bridge handshake failed: ${(err as Error).message}`) + } + } + + // ── JSON-RPC plumbing ───────────────────────────────────────────── + + private call(method: string, params: Record): Promise { + return this.callWithTimeout(method, params, this.callTimeoutMs) + } + + private callWithTimeout( + method: string, + params: Record, + timeoutMs: number, + ): Promise { + const proc = this.proc + if (!proc) return Promise.reject(new Error('Bridge process not started')) + + const id = this.nextId++ + const tidyParams: Record = {} + for (const [k, v] of Object.entries(params)) { + if (v !== undefined) tidyParams[k] = v + } + const frame = JSON.stringify({ jsonrpc: '2.0', id, method, params: tidyParams }) + '\n' + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + this.pending.delete(id) + reject(new Error(`Bridge call ${method} (id=${id}) timed out after ${timeoutMs}ms`)) + }, timeoutMs) + + this.pending.set(id, { resolve, reject, method, timeout }) + try { + proc.stdin.write(frame, (err) => { + if (err) { + clearTimeout(timeout) + this.pending.delete(id) + reject(err) + } + }) + } catch (err) { + clearTimeout(timeout) + this.pending.delete(id) + reject(err as Error) + } + }) + } + + private handleResponseLine(line: string): void { + const trimmed = line.trim() + if (trimmed.length === 0) return + + let msg: JsonRpcResponse + try { + msg = JSON.parse(trimmed) as JsonRpcResponse + } catch (err) { + this.logger.warn('macos-axapi.invalid-frame', { line: trimmed.slice(0, 200) }) + return + } + + const slot = this.pending.get(msg.id) + if (!slot) { + this.logger.warn('macos-axapi.orphan-response', { id: msg.id }) + return + } + this.pending.delete(msg.id) + clearTimeout(slot.timeout) + + if (msg.error) { + slot.reject(new Error(`[bridge ${msg.error.code}] ${msg.error.message}`)) + return + } + slot.resolve(msg.result) + } + + private failAllPending(err: Error): void { + for (const [, slot] of this.pending) { + clearTimeout(slot.timeout) + slot.reject(err) + } + this.pending.clear() + } +} + +// ────────────────────────────────────────────────────────────────────── +// Param mapping (identical to the Windows backend — bridges share +// the same wire format) +// ────────────────────────────────────────────────────────────────────── + +function buildExecuteParams(action: ExecuteDesktopAction): Record { + const base: Record = { + actionType: action.type, + elementId: 'element_id' in action ? action.element_id : undefined, + } + + switch (action.type) { + case 'click': + case 'focus': + case 'scroll_to': + break + case 'type': + base.text = action.text + if (action.clear_first) base.clearFirst = true + break + case 'select': + base.value = action.value + break + case 'check': + base.checked = action.checked + break + case 'expand': + base.expanded = action.expanded + break + case 'key': + base.key = action.key + if (action.modifiers && action.modifiers.length > 0) { + base.modifiers = action.modifiers as readonly KeyModifier[] + } + break + default: { + const _exhaustive: never = action + void _exhaustive + } + } + return base +} + +// ────────────────────────────────────────────────────────────────────── +// Bridge path resolution +// ────────────────────────────────────────────────────────────────────── + +const BRIDGE_BIN = 'agentmark-bridge-macos' +const RELATIVE_BRIDGE_PATHS = [ + path.join('apps', 'agent-runner', 'bridges', 'macos', '.build', 'release', BRIDGE_BIN), + path.join('apps', 'agent-runner', 'bridges', 'macos', '.build', 'debug', BRIDGE_BIN), + // Swift Package Manager also writes to architecture-specific + // subpaths (e.g. .build/arm64-apple-macosx/release/) depending on + // build configuration. Probe both. + path.join('apps', 'agent-runner', 'bridges', 'macos', '.build', 'arm64-apple-macosx', 'release', BRIDGE_BIN), + path.join('apps', 'agent-runner', 'bridges', 'macos', '.build', 'arm64-apple-macosx', 'debug', BRIDGE_BIN), + path.join('apps', 'agent-runner', 'bridges', 'macos', '.build', 'x86_64-apple-macosx', 'release', BRIDGE_BIN), + path.join('apps', 'agent-runner', 'bridges', 'macos', '.build', 'x86_64-apple-macosx', 'debug', BRIDGE_BIN), +] + +function resolveBridgePath(): string { + const fromEnv = process.env.AGENTMARK_BRIDGE_PATH + if (fromEnv && fs.existsSync(fromEnv)) return fromEnv + + const startDir = __dirname + + let current = startDir + for (let i = 0; i < 8; i++) { + for (const rel of RELATIVE_BRIDGE_PATHS) { + const candidate = path.join(current, rel) + if (fs.existsSync(candidate)) return candidate + } + const parent = path.dirname(current) + if (parent === current) break + current = parent + } + + throw new Error( + `Cannot find ${BRIDGE_BIN}. Tried AGENTMARK_BRIDGE_PATH env var and ` + + `walked up from ${startDir}. Build the bridge with:\n` + + ` cd apps/agent-runner/bridges/macos && swift build\n` + + `Or set AGENTMARK_BRIDGE_PATH to an existing binary.`, + ) +} + +// ────────────────────────────────────────────────────────────────────── +// Wire format mapping (shared with Windows backend) +// ────────────────────────────────────────────────────────────────────── + +interface RawDesktopCapture { + platform: 'windows' | 'macos' | 'linux' + processName?: string | null + processId?: number | null + windowTitle: string + windowClass?: string | null + windowId: string + focusedElementId?: string | null + treeDepth: number + elementCount: number + root: RawDesktopElement +} + +interface RawDesktopElement { + id: string + role: string + name?: string | null + value?: string | null + placeholder?: string | null + enabled?: boolean | null + selected?: boolean | null + readOnly?: boolean | null + expanded?: boolean | null + aria?: { + pressed?: boolean | null + checked?: boolean | 'mixed' | null + required?: boolean | null + invalid?: boolean | null + } | null + bounds?: { x: number; y: number; width: number; height: number } | null + children?: RawDesktopElement[] | null +} + +interface RawExecuteResult { + ok: boolean + message?: string | null + newValue?: string | null +} + +function mapCaptureResponse(raw: RawDesktopCapture): DesktopCapture { + return { + platform: raw.platform, + process_name: raw.processName ?? undefined, + process_id: raw.processId ?? undefined, + window_title: raw.windowTitle, + window_class: raw.windowClass ?? undefined, + window_id: raw.windowId, + focused_element_id: raw.focusedElementId ?? undefined, + tree_depth: raw.treeDepth, + element_count: raw.elementCount, + root: mapElement(raw.root), + } +} + +function mapElement(raw: RawDesktopElement): import('./types').DesktopElement { + return { + id: raw.id, + role: raw.role as import('./types').DesktopRole, + name: raw.name ?? undefined, + value: raw.value ?? undefined, + placeholder: raw.placeholder ?? undefined, + enabled: raw.enabled ?? undefined, + selected: raw.selected ?? undefined, + read_only: raw.readOnly ?? undefined, + expanded: raw.expanded ?? undefined, + aria: raw.aria ? { + pressed: raw.aria.pressed ?? undefined, + checked: raw.aria.checked ?? undefined, + required: raw.aria.required ?? undefined, + invalid: raw.aria.invalid ?? undefined, + } : undefined, + bounds: raw.bounds ?? undefined, + children: raw.children ? raw.children.map(mapElement) : undefined, + } +} diff --git a/src/index.ts b/src/index.ts index 6755dbf..c2841a5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -214,11 +214,18 @@ export type { // ── v0.4 spec / v0.12 lib: Desktop support (kind: 'desktop') ───────────── -export { convertDesktop, FixtureBackend, WindowsUiaBackend, buildDesktopBody } from './desktop' +export { + convertDesktop, + FixtureBackend, + WindowsUiaBackend, + MacosAxapiBackend, + buildDesktopBody, +} from './desktop' export type { ConvertDesktopOptions, FixtureBackendOptions, WindowsUiaBackendOptions, + MacosAxapiBackendOptions, BuildDesktopBodyResult, DesktopCaptureBackend, CaptureDesktopOptions, diff --git a/src/mcp/dispatcher.ts b/src/mcp/dispatcher.ts index 7cf9df5..0f1f355 100644 --- a/src/mcp/dispatcher.ts +++ b/src/mcp/dispatcher.ts @@ -14,6 +14,7 @@ import { createBrowser, convertDesktop, FixtureBackend, + MacosAxapiBackend, WindowsUiaBackend, openPdfDocument, isAgentMarkError, @@ -359,12 +360,17 @@ async function openDesktop(state: DispatcherState, args: Record backend = new WindowsUiaBackend({ bridgePath }) break case 'macos_axapi': - return { - text: - `Backend "macos_axapi" is not yet bundled with this build of agentmark. ` - + 'Use backend="fixture" for in-memory testing while the macOS bridge ships.', - isError: true, + if (process.platform !== 'darwin') { + return { + text: + `Backend "macos_axapi" requires macOS (process.platform=='darwin'). ` + + `Current platform: ${process.platform}. Use backend="fixture" for ` + + `in-memory testing, or run agentmark on a Mac host.`, + isError: true, + } } + backend = new MacosAxapiBackend({ bridgePath }) + break default: return { text: `Unknown desktop backend: ${requested}`, isError: true } } diff --git a/test/desktop/macos-axapi-backend.test.ts b/test/desktop/macos-axapi-backend.test.ts new file mode 100644 index 0000000..e99424a --- /dev/null +++ b/test/desktop/macos-axapi-backend.test.ts @@ -0,0 +1,109 @@ +import { describe, it, expect, afterEach } from 'vitest' +import * as path from 'node:path' +import { MacosAxapiBackend } from '../../src/desktop/macos-axapi-backend' + +// Reuse the same fake-bridge.cjs the Windows backend tests use. The +// wire protocol is byte-identical between the two bridges, so a +// single fake substitutes for both. +const FAKE_BRIDGE = path.join(__dirname, 'fixtures', 'fake-bridge.cjs') + +function makeBackend(extra: Partial[0]> = {}) { + return new MacosAxapiBackend({ + bridgePath: FAKE_BRIDGE, + allowNonMac: true, + startupTimeoutMs: 5000, + callTimeoutMs: 5000, + ...extra, + }) +} + +let backend: MacosAxapiBackend | null = null + +afterEach(async () => { + if (backend) { + try { await backend.close() } catch { /* swallow */ } + backend = null + } +}) + +describe('MacosAxapiBackend', () => { + it('refuses to construct on non-macOS without allowNonMac', () => { + if (process.platform === 'darwin') return // not applicable + expect(() => new MacosAxapiBackend({ bridgePath: FAKE_BRIDGE })).toThrow(/requires macOS/) + }) + + it('reports `macos_axapi` as its backend name', () => { + backend = makeBackend() + expect(backend.name).toBe('macos_axapi') + }) + + it('captures via the bridge and maps the response to DesktopCapture', async () => { + backend = makeBackend() + const cap = await backend.capture({}) + + // The fake bridge always reports platform=windows in its preset + // (it's a shared fixture); the backend should pass that through + // verbatim. We assert on the structural fields the macOS backend + // is responsible for mapping. + expect(cap.window_title).toBe('Fake Window 1') + expect(cap.tree_depth).toBe(2) + expect(cap.element_count).toBe(3) + expect(cap.root.role).toBe('window') + expect(cap.root.children).toHaveLength(2) + expect(cap.root.children?.[0].role).toBe('text_input') + expect(cap.root.children?.[0].value).toBe('Acme') + }) + + it('forwards capture target to the bridge', async () => { + backend = makeBackend() + const cap = await backend.capture({ + target: { window_id: 'axapi:1234:0', process_name: 'TextEdit' }, + }) + expect(cap.window_id).toBe('axapi:1234:0') + expect(cap.process_name).toBe('TextEdit') + }) + + it('executes a type action and maps newValue back to new_value', async () => { + backend = makeBackend() + await backend.capture({}) + const res = await backend.execute({ + action: { type: 'type', element_id: 'in_company', text: 'Beta Industries' }, + }) + expect(res.ok).toBe(true) + expect(res.new_value).toBe('Beta Industries') + }) + + it('reuses the same bridge process across multiple calls', async () => { + backend = makeBackend() + const r1 = await backend.capture({}) + const r2 = await backend.capture({}) + expect(r1.window_id).toBe(r2.window_id) + }) + + it('reports a clear error when the bridge fails its handshake', async () => { + backend = makeBackend() + await backend.close() + process.env.AGENTMARK_FAKE_BRIDGE_FAIL_PING = '1' + try { + backend = makeBackend() + await expect(backend.capture({})).rejects.toThrow(/handshake failed/i) + } finally { + delete process.env.AGENTMARK_FAKE_BRIDGE_FAIL_PING + } + }) + + it('closes cleanly and rejects subsequent calls', async () => { + backend = makeBackend() + await backend.capture({}) + await backend.close() + await expect(backend.capture({})).rejects.toThrow(/closed/i) + }) + + it('defers bridge-path validation until first call', () => { + // Constructor should not probe the path (matches WindowsUiaBackend behaviour). + expect(() => new MacosAxapiBackend({ + bridgePath: '/definitely/does/not/exist/agentmark-bridge-macos', + allowNonMac: true, + })).not.toThrow() + }) +}) diff --git a/test/mcp/desktop-dispatcher.test.ts b/test/mcp/desktop-dispatcher.test.ts index d6866a9..d169468 100644 --- a/test/mcp/desktop-dispatcher.test.ts +++ b/test/mcp/desktop-dispatcher.test.ts @@ -67,10 +67,15 @@ describe('MCP — desktop tools', () => { expect(result.text).toMatch(/requires Windows/i) }) - it('agentmark_desktop_open with macos_axapi still reports not-yet-bundled (bridge ships later)', async () => { + it('agentmark_desktop_open with macos_axapi refuses to start on non-macOS platforms', async () => { + // On non-macOS: clear OS-mismatch error. + // On macOS: the dispatcher would attempt to spawn the bridge; that + // path is exercised in test/desktop/macos-axapi-backend.test.ts. + if (process.platform === 'darwin') return + const result = await dispatch(state, 'agentmark_desktop_open', { backend: 'macos_axapi' }) expect(result.isError).toBe(true) - expect(result.text).toContain('not yet bundled') + expect(result.text).toMatch(/requires macOS/i) }) it('agentmark_desktop_snapshot returns a valid v0.4 desktop snapshot', async () => {