feat(desktop): snapshot diff — return only what changed

src/desktop/desktop-converter.ts

@@ -77,7 +77,19 @@ export interface ConvertDesktopOptions {
     vendorExtensions?: Record<string, unknown>
 }
 
-export async function convertDesktop(options: ConvertDesktopOptions): Promise<ConversionResult> {
+/**
+ * Desktop-specific extension of `ConversionResult` that also exposes the
+ * raw `DesktopCapture` tree. Callers that want to diff snapshots or
+ * inspect the structured tree without re-parsing the AgentMark string
+ * read it off this field.
+ */
+export interface DesktopConversionResult extends ConversionResult {
+    /** The raw capture returned by the backend, before AgentMark
+     *  serialisation. Same object the binding refers into. */
+    capture: DesktopCapture
+}
+
+export async function convertDesktop(options: ConvertDesktopOptions): Promise<DesktopConversionResult> {
     const logger = options.logger ?? noopLogger
     const ttlMs = options.ttlMs ?? 15_000
 
@@ -167,7 +179,7 @@ export async function convertDesktop(options: ConvertDesktopOptions): Promise<Co
         bytes: text.length,
     })
 
-    return { agentmark: text, binding }
+    return { agentmark: text, binding, capture }
 }
 
 function synthesiseDesktopUrl(capture: DesktopCapture): string {

src/desktop/diff.ts

@@ -0,0 +1,191 @@
+/**
+ * Diff two `DesktopCapture` trees.
+ *
+ * Used by `agentmark_desktop_diff` to let agents check "did my action
+ * take effect?" or "did a new dialog appear?" with a much smaller token
+ * payload than re-snapshotting the whole window.
+ *
+ * Matching is by stable accessibility id (`DesktopElement.id`). Roles,
+ * names, bounds are reported as auxiliary fields on each change entry,
+ * not used for matching.
+ *
+ * Out of scope (v1): structural moves (element reparented). Treated
+ * as "removed from old parent, added to new parent" — agents that care
+ * can re-snapshot.
+ */
+import type { DesktopCapture, DesktopElement } from './types'
+
+export interface DesktopElementChange {
+    /** Stable id present in both before and after. */
+    id: string
+    role: string
+    name?: string
+    /** Map of field → { from, to } for every primitive property that changed. */
+    changes: Record<string, { from: unknown; to: unknown }>
+}
+
+export interface DesktopElementSummary {
+    id: string
+    role: string
+    name?: string
+    value?: string
+}
+
+export interface DesktopDiff {
+    /** True when nothing meaningful changed (all arrays empty + title/focus same). */
+    no_changes: boolean
+    /** Title changes — common after navigation / dialog open. */
+    window_title_changed: { from: string; to: string } | null
+    /** Focus change — different element now has keyboard focus. */
+    focus_changed: { from: string | undefined; to: string | undefined } | null
+    /** Counts so agents can decide cheaply whether to re-snapshot. */
+    summary: {
+        added: number
+        removed: number
+        changed: number
+    }
+    /** Elements present in `after` but not `before`. */
+    elements_added: DesktopElementSummary[]
+    /** Elements present in `before` but not `after`. */
+    elements_removed: DesktopElementSummary[]
+    /** Elements present in both with one or more primitive field changes. */
+    elements_changed: DesktopElementChange[]
+}
+
+/**
+ * Compare two captures and return a structured diff. The function is
+ * pure — neither argument is mutated. Order of the arguments matters:
+ * `before` is the older snapshot, `after` is the newer one.
+ */
+export function diffDesktopCaptures(
+    before: DesktopCapture,
+    after: DesktopCapture,
+): DesktopDiff {
+    const beforeMap = indexById(before.root)
+    const afterMap = indexById(after.root)
+
+    const addedIds: string[] = []
+    const removedIds: string[] = []
+    const sharedIds: string[] = []
+
+    for (const id of afterMap.keys()) {
+        if (beforeMap.has(id)) sharedIds.push(id)
+        else addedIds.push(id)
+    }
+    for (const id of beforeMap.keys()) {
+        if (!afterMap.has(id)) removedIds.push(id)
+    }
+
+    const elements_added: DesktopElementSummary[] = addedIds.map((id) => summarise(afterMap.get(id)!))
+    const elements_removed: DesktopElementSummary[] = removedIds.map((id) => summarise(beforeMap.get(id)!))
+    const elements_changed: DesktopElementChange[] = []
+
+    for (const id of sharedIds) {
+        const a = beforeMap.get(id)!
+        const b = afterMap.get(id)!
+        const changes = compareElement(a, b)
+        if (Object.keys(changes).length > 0) {
+            elements_changed.push({
+                id,
+                role: b.role,
+                name: b.name,
+                changes,
+            })
+        }
+    }
+
+    const window_title_changed =
+        before.window_title !== after.window_title
+            ? { from: before.window_title, to: after.window_title }
+            : null
+
+    const focus_changed =
+        before.focused_element_id !== after.focused_element_id
+            ? { from: before.focused_element_id, to: after.focused_element_id }
+            : null
+
+    const summary = {
+        added: elements_added.length,
+        removed: elements_removed.length,
+        changed: elements_changed.length,
+    }
+
+    const no_changes =
+        summary.added === 0
+        && summary.removed === 0
+        && summary.changed === 0
+        && window_title_changed === null
+        && focus_changed === null
+
+    return {
+        no_changes,
+        window_title_changed,
+        focus_changed,
+        summary,
+        elements_added,
+        elements_removed,
+        elements_changed,
+    }
+}
+
+/** Build a flat id → element index by walking the tree. */
+function indexById(root: DesktopElement): Map<string, DesktopElement> {
+    const out = new Map<string, DesktopElement>()
+    const stack: DesktopElement[] = [root]
+    while (stack.length > 0) {
+        const el = stack.pop()!
+        if (!out.has(el.id)) out.set(el.id, el)
+        if (el.children) {
+            for (let i = el.children.length - 1; i >= 0; i--) stack.push(el.children[i])
+        }
+    }
+    return out
+}
+
+function summarise(el: DesktopElement): DesktopElementSummary {
+    return {
+        id: el.id,
+        role: el.role,
+        name: el.name,
+        value: el.value,
+    }
+}
+
+/** Compare two elements' primitive fields. Children are diffed at the
+ *  tree level via add/remove sets; not recursed into here. */
+function compareElement(a: DesktopElement, b: DesktopElement): Record<string, { from: unknown; to: unknown }> {
+    const changes: Record<string, { from: unknown; to: unknown }> = {}
+
+    const scalarFields = ['role', 'name', 'value', 'placeholder', 'enabled', 'selected', 'read_only', 'expanded'] as const
+    for (const field of scalarFields) {
+        const av = a[field]
+        const bv = b[field]
+        if (av !== bv) changes[field] = { from: av, to: bv }
+    }
+
+    // aria — small shallow object; compare each known key.
+    const ariaA = a.aria ?? {}
+    const ariaB = b.aria ?? {}
+    const ariaKeys: Array<keyof NonNullable<DesktopElement['aria']>> = ['pressed', 'checked', 'required', 'invalid']
+    for (const key of ariaKeys) {
+        if (ariaA[key] !== ariaB[key]) {
+            changes[`aria.${key}`] = { from: ariaA[key], to: ariaB[key] }
+        }
+    }
+
+    // bounds — only report if any coordinate shifted by more than 1px
+    // (sub-pixel jitter from compositor isn't actionable).
+    if (a.bounds && b.bounds) {
+        const dx = Math.abs(a.bounds.x - b.bounds.x)
+        const dy = Math.abs(a.bounds.y - b.bounds.y)
+        const dw = Math.abs(a.bounds.width - b.bounds.width)
+        const dh = Math.abs(a.bounds.height - b.bounds.height)
+        if (dx > 1 || dy > 1 || dw > 1 || dh > 1) {
+            changes.bounds = { from: a.bounds, to: b.bounds }
+        }
+    } else if (a.bounds !== b.bounds) {
+        changes.bounds = { from: a.bounds, to: b.bounds }
+    }
+
+    return changes
+}

src/desktop/index.ts

@@ -3,7 +3,14 @@
  */
 
 export { convertDesktop } from './desktop-converter'
-export type { ConvertDesktopOptions } from './desktop-converter'
+export type { ConvertDesktopOptions, DesktopConversionResult } from './desktop-converter'
+
+export { diffDesktopCaptures } from './diff'
+export type {
+    DesktopDiff,
+    DesktopElementChange,
+    DesktopElementSummary,
+} from './diff'
 
 export { FixtureBackend } from './fixture-backend'
 export type { FixtureBackendOptions } from './fixture-backend'
@@ -23,6 +30,8 @@ export type {
     DesktopTarget,
     ExecuteDesktopOptions,
     ExecuteDesktopAction,
+    ExecuteDesktopBatchOptions,
+    ExecuteDesktopBatchResult,
     ExecuteDesktopResult,
     KeyModifier,
 } from './types'

src/index.ts

@@ -216,6 +216,7 @@ export type {
 
 export {
     convertDesktop,
+    diffDesktopCaptures,
     FixtureBackend,
     WindowsUiaBackend,
     MacosAxapiBackend,
@@ -230,11 +231,17 @@ export type {
     DesktopCaptureBackend,
     CaptureDesktopOptions,
     DesktopCapture,
+    DesktopConversionResult,
+    DesktopDiff,
     DesktopElement,
+    DesktopElementChange,
+    DesktopElementSummary,
     DesktopRole,
     DesktopTarget,
     ExecuteDesktopOptions,
     ExecuteDesktopAction,
+    ExecuteDesktopBatchOptions,
+    ExecuteDesktopBatchResult,
     ExecuteDesktopResult,
     KeyModifier,
 } from './desktop'

src/mcp/plugins/desktop.ts

@@ -6,6 +6,7 @@
  */
 import {
     convertDesktop,
+    diffDesktopCaptures,
     FixtureBackend,
     MacosAxapiBackend,
     WindowsUiaBackend,
@@ -146,6 +147,45 @@ const DESKTOP_TOOLS: McpToolDef[] = [
             required: ['desktop_id', 'action_id'],
         },
     },
+    {
+        name: 'agentmark_desktop_diff',
+        description:
+            'Take a fresh capture of the target window and return only what '
+            + 'CHANGED since the last snapshot or diff on this session. '
+            + 'Much smaller payload than a full re-snapshot — useful for '
+            + 'verifying "did my action take effect?" or detecting a new '
+            + 'dialog without paying for the whole tree.\n'
+            + '\nReturns: { no_changes, summary {added/removed/changed}, '
+            + 'window_title_changed, focus_changed, elements_added[], '
+            + 'elements_removed[], elements_changed[] }. Each `elements_changed` '
+            + 'entry lists the specific fields that moved (value, enabled, '
+            + 'aria.checked, bounds, etc.) with from/to pairs.\n'
+            + '\nAfter the diff completes, the session\'s cached capture is '
+            + 'updated so the NEXT diff is against this new state. The '
+            + 'action-id binding from the most recent full snapshot is NOT '
+            + 'changed — call agentmark_desktop_snapshot when you need '
+            + 'fresh action_ids after a structural change.',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                desktop_id: { type: 'string' },
+                target: {
+                    type: 'object',
+                    description: 'Optional target override (same shape as snapshot).',
+                    properties: {
+                        process_name: { type: 'string' },
+                        process_id: { type: 'number' },
+                        window_title: { type: 'string' },
+                        window_id: { type: 'string' },
+                    },
+                },
+                max_depth: { type: 'number' },
+                include_hidden: { type: 'boolean' },
+                timeout_ms: { type: 'number' },
+            },
+            required: ['desktop_id'],
+        },
+    },
     {
         name: 'agentmark_desktop_execute_batch',
         description:
@@ -289,7 +329,7 @@ export function createDesktopPlugin(): DesktopPlugin {
             const includeHidden = args.include_hidden === true
             const timeoutMs = typeof args.timeout_ms === 'number' ? args.timeout_ms : undefined
 
-            const { agentmark, binding } = await convertDesktop({
+            const { agentmark, binding, capture } = await convertDesktop({
                 backend: session.backend,
                 target,
                 maxDepth,
@@ -299,6 +339,10 @@ export function createDesktopPlugin(): DesktopPlugin {
 
             session.lastTarget = target
             session.lastBinding = binding
+            // Cache the raw capture so agentmark_desktop_diff can compare
+            // a future snapshot against the most recent one without forcing
+            // the agent to re-fetch the previous baseline.
+            session.lastCapture = capture
             const snap = parseSnapshot(agentmark)
             session.lastActionTypes = new Map(
                 Object.entries(snap.actions ?? {}).map(([k, def]) => [k, def.type]),
@@ -307,6 +351,41 @@ export function createDesktopPlugin(): DesktopPlugin {
             return { text: agentmark }
         },
 
+        agentmark_desktop_diff: async (args): Promise<DispatchResult> => {
+            const id = requireString(args, 'desktop_id')
+            const session = requireDesktop(id)
+
+            if (!session.lastCapture) {
+                return {
+                    text:
+                        `No cached capture for desktop_id ${id}. Call `
+                        + `agentmark_desktop_snapshot first so the diff has a baseline.`,
+                    isError: true,
+                }
+            }
+
+            const target = parseTarget(args.target) ?? session.lastTarget
+            const maxDepth = typeof args.max_depth === 'number' ? args.max_depth : undefined
+            const includeHidden = args.include_hidden === true
+            const timeoutMs = typeof args.timeout_ms === 'number' ? args.timeout_ms : 5000
+
+            const fresh = await session.backend.capture({
+                target,
+                maxDepth,
+                includeHidden,
+                timeoutMs,
+            })
+
+            const diff = diffDesktopCaptures(session.lastCapture, fresh)
+
+            // Move the baseline forward so the NEXT diff is against this
+            // capture rather than the original snapshot.
+            session.lastCapture = fresh
+            session.lastTarget = target
+
+            return { text: JSON.stringify(diff, null, 2) }
+        },
+
         agentmark_desktop_execute: async (args): Promise<DispatchResult> => {
             const id = requireString(args, 'desktop_id')
             const actionId = requireString(args, 'action_id')