feat(desktop): batch execute — one round-trip for N actions

apps/agent-runner/bridges/macos/Sources/AgentMarkBridgeMacos/Dispatcher.swift

@@ -29,10 +29,12 @@ final class Dispatcher {
                 return try handleCapture(params: request.params)
             case "execute":
                 return try handleExecute(params: request.params)
+            case "execute_batch":
+                return try handleExecuteBatch(params: request.params)
             default:
                 throw RpcError(
                     code: .methodNotFound,
-                    message: "Unknown method: \(request.method). Supported: ping, capabilities, list_windows, capture, execute.",
+                    message: "Unknown method: \(request.method). Supported: ping, capabilities, list_windows, capture, execute, execute_batch.",
                     requestId: request.id
                 )
             }
@@ -55,7 +57,7 @@ final class Dispatcher {
         return [
             "bridge": "agentmark-bridge-macos",
             "version": bridgeVersion,
-            "methods": ["ping", "capabilities", "list_windows", "capture", "execute"],
+            "methods": ["ping", "capabilities", "list_windows", "capture", "execute", "execute_batch"],
             "axapiProvider": "Accessibility (AXAPI)",
             "platform": "macos",
             "accessibilityGranted": AccessibilityPermission.isGranted,
@@ -83,9 +85,60 @@ final class Dispatcher {
     }
 
     private func handleExecute(params: JsonValue?) throws -> Any {
+        guard let p = params else {
+            throw RpcError(code: .invalidParams, message: "execute requires params.")
+        }
+        let req = try buildExecuteRequest(p)
+        return try capturer.execute(req)
+    }
+
+    /// Run N actions in one in-process loop so the entire batch costs one
+    /// stdio round-trip. `onError: stop` (default) aborts on the first
+    /// failure; `onError: continue` runs the full array regardless.
+    private func handleExecuteBatch(params: JsonValue?) throws -> Any {
         guard let p = params,
-              let elementId = p.string("elementId"), !elementId.isEmpty else {
-            throw RpcError(code: .invalidParams, message: "execute requires `elementId`.")
+              let actionsAny = p.dict?["actions"],
+              case .array(let actionArray) = JsonValue.fromAny(actionsAny) else {
+            throw RpcError(code: .invalidParams, message: "execute_batch requires an `actions` array.")
+        }
+
+        let stopOnError = (p.string("onError") ?? "stop") != "continue"
+
+        var results: [[String: Any]] = []
+        results.reserveCapacity(actionArray.count)
+        var allOk = true
+
+        for actionAny in actionArray {
+            let actionParam = JsonValue.fromAny(actionAny)
+            let req: AxapiCapturer.ExecuteRequest
+            do {
+                req = try buildExecuteRequest(actionParam)
+            } catch {
+                results.append(["ok": false, "message": "\(error)"])
+                allOk = false
+                if stopOnError { break }
+                continue
+            }
+            let r = try capturer.execute(req)
+            // `capturer.execute` returns `[String: Any]` with `ok` / `message` / `newValue`.
+            let normalized = r as? [String: Any] ?? [:]
+            results.append(normalized)
+            if let ok = normalized["ok"] as? Bool, !ok {
+                allOk = false
+                if stopOnError { break }
+            }
+        }
+
+        return [
+            "results": results,
+            "allOk": allOk,
+            "executedCount": results.count,
+        ] as [String: Any]
+    }
+
+    private func buildExecuteRequest(_ p: JsonValue) throws -> AxapiCapturer.ExecuteRequest {
+        guard let elementId = p.string("elementId"), !elementId.isEmpty else {
+            throw RpcError(code: .invalidParams, message: "execute action requires `elementId`.")
         }
         var req = AxapiCapturer.ExecuteRequest()
         req.elementId = elementId
@@ -100,6 +153,6 @@ final class Dispatcher {
         }
         req.clearFirst = p.bool("clearFirst") ?? false
         if let i = p.int("timeoutMs") { req.timeoutMs = i }
-        return try capturer.execute(req)
+        return req
     }
 }

apps/agent-runner/bridges/windows/Program.cs

@@ -205,14 +205,15 @@ public RpcDispatcher()
     {
         return method switch
         {
-            "ping"         => HandlePing(),
-            "capabilities" => HandleCapabilities(),
-            "list_windows" => HandleListWindows(),
-            "capture"      => HandleCapture(@params),
-            "execute"      => HandleExecute(@params),
+            "ping"          => HandlePing(),
+            "capabilities"  => HandleCapabilities(),
+            "list_windows"  => HandleListWindows(),
+            "capture"       => HandleCapture(@params),
+            "execute"       => HandleExecute(@params),
+            "execute_batch" => HandleExecuteBatch(@params),
             _ => throw new RpcException(
                 RpcError.MethodNotFound,
-                $"Unknown method: {method}. Supported: ping, capabilities, list_windows, capture, execute."),
+                $"Unknown method: {method}. Supported: ping, capabilities, list_windows, capture, execute, execute_batch."),
         };
     }
 
@@ -247,6 +248,7 @@ public void Dispose()
             "list_windows",
             "capture",
             "execute",
+            "execute_batch",
         },
         uiaProvider = "FlaUI.UIA3",
         platform = "windows",
@@ -288,22 +290,7 @@ private object HandleCapture(JsonElement @params)
 
     private object HandleExecute(JsonElement @params)
     {
-        var modifiers = ReadStringArray(@params, "modifiers");
-
-        var req = new UiaCapturer.ExecuteRequest
-        {
-            ElementId = ReadString(@params, "elementId")
-                ?? throw new RpcException(RpcError.InvalidParams, "execute requires `elementId`."),
-            ActionType = ReadString(@params, "actionType") ?? "click",
-            Text = ReadString(@params, "text"),
-            Value = ReadString(@params, "value"),
-            Checked = ReadBool(@params, "checked"),
-            Expanded = ReadBool(@params, "expanded"),
-            Key = ReadString(@params, "key"),
-            Modifiers = modifiers,
-            ClearFirst = ReadBool(@params, "clearFirst") ?? false,
-            TimeoutMs = ReadInt(@params, "timeoutMs") ?? 5000,
-        };
+        var req = BuildExecuteRequest(@params);
 
         try
         {
@@ -323,6 +310,82 @@ private object HandleExecute(JsonElement @params)
         }
     }
 
+    /// <summary>
+    /// Run a sequence of actions inside the same STA invocation so the entire
+    /// batch costs one stdio round-trip. Honours `onError: stop | continue`
+    /// to abort or keep going past failures.
+    /// </summary>
+    private object HandleExecuteBatch(JsonElement @params)
+    {
+        if (!@params.TryGetProperty("actions", out var actionsEl) || actionsEl.ValueKind != JsonValueKind.Array)
+        {
+            throw new RpcException(RpcError.InvalidParams, "execute_batch requires an `actions` array.");
+        }
+
+        var stopOnError = (ReadString(@params, "onError") ?? "stop") != "continue";
+
+        var requests = new List<UiaCapturer.ExecuteRequest>(actionsEl.GetArrayLength());
+        foreach (var actionEl in actionsEl.EnumerateArray())
+        {
+            requests.Add(BuildExecuteRequest(actionEl));
+        }
+
+        var results = new List<object>(requests.Count);
+        var allOk = true;
+
+        try
+        {
+            var capturer = _capturer.Value;
+            var sta = _staWorker.Value;
+            sta.Invoke(() =>
+            {
+                foreach (var req in requests)
+                {
+                    var r = capturer.Execute(req);
+                    results.Add(new { ok = r.Ok, message = r.Message, newValue = r.NewValue });
+                    if (!r.Ok)
+                    {
+                        allOk = false;
+                        if (stopOnError) break;
+                    }
+                }
+            });
+        }
+        catch (InvalidOperationException ex)
+        {
+            throw new RpcException(RpcError.InternalError, ex.Message);
+        }
+
+        return new
+        {
+            results = results.ToArray(),
+            allOk,
+            executedCount = results.Count,
+        };
+    }
+
+    /// <summary>
+    /// Pull an ExecuteRequest out of a JSON params blob. Shared between the
+    /// single-execute and batch-execute paths.
+    /// </summary>
+    private static UiaCapturer.ExecuteRequest BuildExecuteRequest(JsonElement @params)
+    {
+        return new UiaCapturer.ExecuteRequest
+        {
+            ElementId = ReadString(@params, "elementId")
+                ?? throw new RpcException(RpcError.InvalidParams, "execute action requires `elementId`."),
+            ActionType = ReadString(@params, "actionType") ?? "click",
+            Text = ReadString(@params, "text"),
+            Value = ReadString(@params, "value"),
+            Checked = ReadBool(@params, "checked"),
+            Expanded = ReadBool(@params, "expanded"),
+            Key = ReadString(@params, "key"),
+            Modifiers = ReadStringArray(@params, "modifiers"),
+            ClearFirst = ReadBool(@params, "clearFirst") ?? false,
+            TimeoutMs = ReadInt(@params, "timeoutMs") ?? 5000,
+        };
+    }
+
     private static string[]? ReadStringArray(JsonElement parent, string name)
     {
         if (parent.ValueKind != JsonValueKind.Object) return null;

src/desktop/fixture-backend.ts

@@ -18,7 +18,11 @@ import type {
     DesktopCaptureBackend,
     CaptureDesktopOptions,
     DesktopElement,
+    DesktopTarget,
     DesktopTargetSummary,
+    ExecuteDesktopAction,
+    ExecuteDesktopBatchOptions,
+    ExecuteDesktopBatchResult,
     ExecuteDesktopOptions,
     ExecuteDesktopResult,
 } from './types'
@@ -104,6 +108,47 @@ export class FixtureBackend implements DesktopCaptureBackend {
                 return { ok: true }
         }
     }
+
+    async executeBatch(opts: ExecuteDesktopBatchOptions): Promise<ExecuteDesktopBatchResult> {
+        // The fixture pays its `latencyMs` once for the whole batch — modelling
+        // a bridge that processes the whole array inside its sidecar. The
+        // looped-execute fallback in WindowsUiaBackend / MacosAxapiBackend
+        // pays it per call.
+        if (this.latencyMs) await delay(this.latencyMs)
+
+        const results: ExecuteDesktopResult[] = []
+        const onError = opts.on_error ?? 'stop'
+        for (const action of opts.actions) {
+            const result = await this.executeAction(opts.target, action)
+            results.push(result)
+            if (!result.ok && onError === 'stop') break
+        }
+        return {
+            results,
+            all_ok: results.every((r) => r.ok),
+            executed_count: results.length,
+        }
+    }
+
+    private async executeAction(
+        target: DesktopTarget | undefined,
+        action: ExecuteDesktopAction,
+    ): Promise<ExecuteDesktopResult> {
+        this.executed.push({ target, action })
+        switch (action.type) {
+            case 'type':
+                this.elementValues.set(action.element_id, action.text)
+                return { ok: true, new_value: action.text }
+            case 'check':
+                this.elementValues.set(action.element_id, action.checked ? 'true' : 'false')
+                return { ok: true, new_value: String(action.checked) }
+            case 'select':
+                this.elementValues.set(action.element_id, action.value)
+                return { ok: true, new_value: action.value }
+            default:
+                return { ok: true }
+        }
+    }
 }
 
 function delay(ms: number): Promise<void> {

src/desktop/macos-axapi-backend.ts

@@ -35,6 +35,8 @@ import type {
     DesktopCaptureBackend,
     DesktopTargetSummary,
     ExecuteDesktopAction,
+    ExecuteDesktopBatchOptions,
+    ExecuteDesktopBatchResult,
     ExecuteDesktopOptions,
     ExecuteDesktopResult,
     KeyModifier,
@@ -149,6 +151,32 @@ export class MacosAxapiBackend implements DesktopCaptureBackend {
         }
     }
 
+    async executeBatch(opts: ExecuteDesktopBatchOptions): Promise<ExecuteDesktopBatchResult> {
+        await this.ensureStarted()
+        const params: Record<string, unknown> = {
+            actions: opts.actions.map((a) => buildExecuteParams(a)),
+            onError: opts.on_error ?? 'stop',
+        }
+        if (opts.timeoutMs !== undefined) params.timeoutMs = opts.timeoutMs
+
+        const raw = (await this.callWithTimeout(
+            'execute_batch',
+            params,
+            opts.timeoutMs ?? Math.max(5000, opts.actions.length * 50),
+        )) as { results?: RawExecuteResult[]; allOk?: boolean; executedCount?: number }
+
+        const results = (raw.results ?? []).map((r) => ({
+            ok: !!r.ok,
+            message: r.message ?? undefined,
+            new_value: r.newValue ?? undefined,
+        }))
+        return {
+            results,
+            all_ok: typeof raw.allOk === 'boolean' ? raw.allOk : results.every((r) => r.ok),
+            executed_count: typeof raw.executedCount === 'number' ? raw.executedCount : results.length,
+        }
+    }
+
     async close(): Promise<void> {
         this.closed = true
         const proc = this.proc

src/desktop/types.ts

@@ -35,6 +35,13 @@ export interface DesktopCaptureBackend {
      *  is identified by `element_id` which comes from the capture tree. */
     execute(opts: ExecuteDesktopOptions): Promise<ExecuteDesktopResult>
 
+    /** Execute a sequence of actions in one round-trip. Bridges that
+     *  implement this natively run the entire batch inside the sidecar
+     *  process; the default fallback below calls `execute()` in a loop
+     *  (still saves the MCP dispatch overhead but pays N stdio round-trips
+     *  to the bridge). */
+    executeBatch?(opts: ExecuteDesktopBatchOptions): Promise<ExecuteDesktopBatchResult>
+
     /** Optional teardown — release native handles, close sidecar process. */
     close?(): Promise<void>
 }
@@ -231,3 +238,40 @@ export interface ExecuteDesktopResult {
      *  re-capture. */
     new_value?: string
 }
+
+/**
+ * Batched execution — run N actions in one MCP / bridge round-trip.
+ *
+ * Designed for bulk-input workflows ("fill 1000 form fields", "write 10k
+ * Excel cells") where the per-call dispatch overhead dominates the
+ * actual SetValue/Click work. Bridges that implement this natively run
+ * the whole array inside their sidecar; the dispatcher's default loop
+ * fallback still wins by avoiding the MCP round-trips.
+ */
+export interface ExecuteDesktopBatchOptions {
+    /** Which window the actions target. Same semantics as `execute()`. */
+    target?: DesktopTarget
+    /** Sequence of actions to run. Executed in array order. */
+    actions: ReadonlyArray<ExecuteDesktopAction>
+    /**
+     * What to do on failure of any single action:
+     *   - 'stop' (default): abort the remainder, return results so far
+     *      plus an error result for the failure.
+     *   - 'continue': keep running, return one result per action.
+     */
+    on_error?: 'stop' | 'continue'
+    /** Per-batch timeout (ms). Default: max(5000, 50 * actions.length). */
+    timeoutMs?: number
+}
+
+export interface ExecuteDesktopBatchResult {
+    /** One result per attempted action, in input order. Length may be
+     *  less than `actions.length` when `on_error: 'stop'` and a failure
+     *  occurred before the end. */
+    results: ReadonlyArray<ExecuteDesktopResult>
+    /** Aggregate flag: true when every result has `ok: true`. */
+    all_ok: boolean
+    /** Number of actions that ran (including the failing one when
+     *  on_error: 'stop'). */
+    executed_count: number
+}