diff --git a/src/desktop/desktop-converter.ts b/src/desktop/desktop-converter.ts index 7793bae..7555826 100644 --- a/src/desktop/desktop-converter.ts +++ b/src/desktop/desktop-converter.ts @@ -77,7 +77,19 @@ export interface ConvertDesktopOptions { vendorExtensions?: Record } -export async function convertDesktop(options: ConvertDesktopOptions): Promise { +/** + * Desktop-specific extension of `ConversionResult` that also exposes the + * raw `DesktopCapture` tree. Callers that want to diff snapshots or + * inspect the structured tree without re-parsing the AgentMark string + * read it off this field. + */ +export interface DesktopConversionResult extends ConversionResult { + /** The raw capture returned by the backend, before AgentMark + * serialisation. Same object the binding refers into. */ + capture: DesktopCapture +} + +export async function convertDesktop(options: ConvertDesktopOptions): Promise { const logger = options.logger ?? noopLogger const ttlMs = options.ttlMs ?? 15_000 @@ -167,7 +179,7 @@ export async function convertDesktop(options: ConvertDesktopOptions): Promise +} + +export interface DesktopElementSummary { + id: string + role: string + name?: string + value?: string +} + +export interface DesktopDiff { + /** True when nothing meaningful changed (all arrays empty + title/focus same). */ + no_changes: boolean + /** Title changes — common after navigation / dialog open. */ + window_title_changed: { from: string; to: string } | null + /** Focus change — different element now has keyboard focus. */ + focus_changed: { from: string | undefined; to: string | undefined } | null + /** Counts so agents can decide cheaply whether to re-snapshot. */ + summary: { + added: number + removed: number + changed: number + } + /** Elements present in `after` but not `before`. */ + elements_added: DesktopElementSummary[] + /** Elements present in `before` but not `after`. */ + elements_removed: DesktopElementSummary[] + /** Elements present in both with one or more primitive field changes. */ + elements_changed: DesktopElementChange[] +} + +/** + * Compare two captures and return a structured diff. The function is + * pure — neither argument is mutated. Order of the arguments matters: + * `before` is the older snapshot, `after` is the newer one. + */ +export function diffDesktopCaptures( + before: DesktopCapture, + after: DesktopCapture, +): DesktopDiff { + const beforeMap = indexById(before.root) + const afterMap = indexById(after.root) + + const addedIds: string[] = [] + const removedIds: string[] = [] + const sharedIds: string[] = [] + + for (const id of afterMap.keys()) { + if (beforeMap.has(id)) sharedIds.push(id) + else addedIds.push(id) + } + for (const id of beforeMap.keys()) { + if (!afterMap.has(id)) removedIds.push(id) + } + + const elements_added: DesktopElementSummary[] = addedIds.map((id) => summarise(afterMap.get(id)!)) + const elements_removed: DesktopElementSummary[] = removedIds.map((id) => summarise(beforeMap.get(id)!)) + const elements_changed: DesktopElementChange[] = [] + + for (const id of sharedIds) { + const a = beforeMap.get(id)! + const b = afterMap.get(id)! + const changes = compareElement(a, b) + if (Object.keys(changes).length > 0) { + elements_changed.push({ + id, + role: b.role, + name: b.name, + changes, + }) + } + } + + const window_title_changed = + before.window_title !== after.window_title + ? { from: before.window_title, to: after.window_title } + : null + + const focus_changed = + before.focused_element_id !== after.focused_element_id + ? { from: before.focused_element_id, to: after.focused_element_id } + : null + + const summary = { + added: elements_added.length, + removed: elements_removed.length, + changed: elements_changed.length, + } + + const no_changes = + summary.added === 0 + && summary.removed === 0 + && summary.changed === 0 + && window_title_changed === null + && focus_changed === null + + return { + no_changes, + window_title_changed, + focus_changed, + summary, + elements_added, + elements_removed, + elements_changed, + } +} + +/** Build a flat id → element index by walking the tree. */ +function indexById(root: DesktopElement): Map { + const out = new Map() + const stack: DesktopElement[] = [root] + while (stack.length > 0) { + const el = stack.pop()! + if (!out.has(el.id)) out.set(el.id, el) + if (el.children) { + for (let i = el.children.length - 1; i >= 0; i--) stack.push(el.children[i]) + } + } + return out +} + +function summarise(el: DesktopElement): DesktopElementSummary { + return { + id: el.id, + role: el.role, + name: el.name, + value: el.value, + } +} + +/** Compare two elements' primitive fields. Children are diffed at the + * tree level via add/remove sets; not recursed into here. */ +function compareElement(a: DesktopElement, b: DesktopElement): Record { + const changes: Record = {} + + const scalarFields = ['role', 'name', 'value', 'placeholder', 'enabled', 'selected', 'read_only', 'expanded'] as const + for (const field of scalarFields) { + const av = a[field] + const bv = b[field] + if (av !== bv) changes[field] = { from: av, to: bv } + } + + // aria — small shallow object; compare each known key. + const ariaA = a.aria ?? {} + const ariaB = b.aria ?? {} + const ariaKeys: Array> = ['pressed', 'checked', 'required', 'invalid'] + for (const key of ariaKeys) { + if (ariaA[key] !== ariaB[key]) { + changes[`aria.${key}`] = { from: ariaA[key], to: ariaB[key] } + } + } + + // bounds — only report if any coordinate shifted by more than 1px + // (sub-pixel jitter from compositor isn't actionable). + if (a.bounds && b.bounds) { + const dx = Math.abs(a.bounds.x - b.bounds.x) + const dy = Math.abs(a.bounds.y - b.bounds.y) + const dw = Math.abs(a.bounds.width - b.bounds.width) + const dh = Math.abs(a.bounds.height - b.bounds.height) + if (dx > 1 || dy > 1 || dw > 1 || dh > 1) { + changes.bounds = { from: a.bounds, to: b.bounds } + } + } else if (a.bounds !== b.bounds) { + changes.bounds = { from: a.bounds, to: b.bounds } + } + + return changes +} diff --git a/src/desktop/index.ts b/src/desktop/index.ts index 9cae2ae..92f0880 100644 --- a/src/desktop/index.ts +++ b/src/desktop/index.ts @@ -3,7 +3,14 @@ */ export { convertDesktop } from './desktop-converter' -export type { ConvertDesktopOptions } from './desktop-converter' +export type { ConvertDesktopOptions, DesktopConversionResult } from './desktop-converter' + +export { diffDesktopCaptures } from './diff' +export type { + DesktopDiff, + DesktopElementChange, + DesktopElementSummary, +} from './diff' export { FixtureBackend } from './fixture-backend' export type { FixtureBackendOptions } from './fixture-backend' @@ -23,6 +30,8 @@ export type { DesktopTarget, ExecuteDesktopOptions, ExecuteDesktopAction, + ExecuteDesktopBatchOptions, + ExecuteDesktopBatchResult, ExecuteDesktopResult, KeyModifier, } from './types' diff --git a/src/index.ts b/src/index.ts index c2841a5..e466d91 100644 --- a/src/index.ts +++ b/src/index.ts @@ -216,6 +216,7 @@ export type { export { convertDesktop, + diffDesktopCaptures, FixtureBackend, WindowsUiaBackend, MacosAxapiBackend, @@ -230,11 +231,17 @@ export type { DesktopCaptureBackend, CaptureDesktopOptions, DesktopCapture, + DesktopConversionResult, + DesktopDiff, DesktopElement, + DesktopElementChange, + DesktopElementSummary, DesktopRole, DesktopTarget, ExecuteDesktopOptions, ExecuteDesktopAction, + ExecuteDesktopBatchOptions, + ExecuteDesktopBatchResult, ExecuteDesktopResult, KeyModifier, } from './desktop' diff --git a/src/mcp/plugins/desktop.ts b/src/mcp/plugins/desktop.ts index 8bf582e..1563d2b 100644 --- a/src/mcp/plugins/desktop.ts +++ b/src/mcp/plugins/desktop.ts @@ -6,6 +6,7 @@ */ import { convertDesktop, + diffDesktopCaptures, FixtureBackend, MacosAxapiBackend, WindowsUiaBackend, @@ -146,6 +147,45 @@ const DESKTOP_TOOLS: McpToolDef[] = [ required: ['desktop_id', 'action_id'], }, }, + { + name: 'agentmark_desktop_diff', + description: + 'Take a fresh capture of the target window and return only what ' + + 'CHANGED since the last snapshot or diff on this session. ' + + 'Much smaller payload than a full re-snapshot — useful for ' + + 'verifying "did my action take effect?" or detecting a new ' + + 'dialog without paying for the whole tree.\n' + + '\nReturns: { no_changes, summary {added/removed/changed}, ' + + 'window_title_changed, focus_changed, elements_added[], ' + + 'elements_removed[], elements_changed[] }. Each `elements_changed` ' + + 'entry lists the specific fields that moved (value, enabled, ' + + 'aria.checked, bounds, etc.) with from/to pairs.\n' + + '\nAfter the diff completes, the session\'s cached capture is ' + + 'updated so the NEXT diff is against this new state. The ' + + 'action-id binding from the most recent full snapshot is NOT ' + + 'changed — call agentmark_desktop_snapshot when you need ' + + 'fresh action_ids after a structural change.', + inputSchema: { + type: 'object', + properties: { + desktop_id: { type: 'string' }, + target: { + type: 'object', + description: 'Optional target override (same shape as snapshot).', + properties: { + process_name: { type: 'string' }, + process_id: { type: 'number' }, + window_title: { type: 'string' }, + window_id: { type: 'string' }, + }, + }, + max_depth: { type: 'number' }, + include_hidden: { type: 'boolean' }, + timeout_ms: { type: 'number' }, + }, + required: ['desktop_id'], + }, + }, { name: 'agentmark_desktop_execute_batch', description: @@ -289,7 +329,7 @@ export function createDesktopPlugin(): DesktopPlugin { const includeHidden = args.include_hidden === true const timeoutMs = typeof args.timeout_ms === 'number' ? args.timeout_ms : undefined - const { agentmark, binding } = await convertDesktop({ + const { agentmark, binding, capture } = await convertDesktop({ backend: session.backend, target, maxDepth, @@ -299,6 +339,10 @@ export function createDesktopPlugin(): DesktopPlugin { session.lastTarget = target session.lastBinding = binding + // Cache the raw capture so agentmark_desktop_diff can compare + // a future snapshot against the most recent one without forcing + // the agent to re-fetch the previous baseline. + session.lastCapture = capture const snap = parseSnapshot(agentmark) session.lastActionTypes = new Map( Object.entries(snap.actions ?? {}).map(([k, def]) => [k, def.type]), @@ -307,6 +351,41 @@ export function createDesktopPlugin(): DesktopPlugin { return { text: agentmark } }, + agentmark_desktop_diff: async (args): Promise => { + const id = requireString(args, 'desktop_id') + const session = requireDesktop(id) + + if (!session.lastCapture) { + return { + text: + `No cached capture for desktop_id ${id}. Call ` + + `agentmark_desktop_snapshot first so the diff has a baseline.`, + isError: true, + } + } + + const target = parseTarget(args.target) ?? session.lastTarget + const maxDepth = typeof args.max_depth === 'number' ? args.max_depth : undefined + const includeHidden = args.include_hidden === true + const timeoutMs = typeof args.timeout_ms === 'number' ? args.timeout_ms : 5000 + + const fresh = await session.backend.capture({ + target, + maxDepth, + includeHidden, + timeoutMs, + }) + + const diff = diffDesktopCaptures(session.lastCapture, fresh) + + // Move the baseline forward so the NEXT diff is against this + // capture rather than the original snapshot. + session.lastCapture = fresh + session.lastTarget = target + + return { text: JSON.stringify(diff, null, 2) } + }, + agentmark_desktop_execute: async (args): Promise => { const id = requireString(args, 'desktop_id') const actionId = requireString(args, 'action_id') diff --git a/test/desktop/diff.test.ts b/test/desktop/diff.test.ts new file mode 100644 index 0000000..35ce14f --- /dev/null +++ b/test/desktop/diff.test.ts @@ -0,0 +1,160 @@ +/** + * Tests for diffDesktopCaptures — the pure function comparing two + * DesktopCapture trees by stable element ID. + */ +import { describe, it, expect } from 'vitest' +import { diffDesktopCaptures } from '../../src/desktop/diff' +import type { DesktopCapture, DesktopElement } from '../../src/desktop/types' + +function makeCapture(overrides: Partial & { root: DesktopElement }): DesktopCapture { + return { + platform: 'windows', + window_title: 'Test Window', + tree_depth: 2, + element_count: 0, // recomputed below + root: overrides.root, + ...overrides, + } +} + +function el(id: string, role: string, fields: Partial = {}, children?: DesktopElement[]): DesktopElement { + return { id, role: role as DesktopElement['role'], children, ...fields } +} + +describe('diffDesktopCaptures — no-op cases', () => { + it('returns no_changes=true when two identical captures are compared', () => { + const root = el('root', 'window', { name: 'App' }, [ + el('a', 'button', { name: 'Save' }), + el('b', 'text_input', { name: 'Email', value: 'x@y.com' }), + ]) + const before = makeCapture({ root }) + const after = makeCapture({ root: structuredClone(root) }) + + const diff = diffDesktopCaptures(before, after) + expect(diff.no_changes).toBe(true) + expect(diff.summary).toEqual({ added: 0, removed: 0, changed: 0 }) + expect(diff.window_title_changed).toBeNull() + }) +}) + +describe('diffDesktopCaptures — value + state changes', () => { + it('reports a single value change on an input', () => { + const before = makeCapture({ + root: el('root', 'window', {}, [ + el('email', 'text_input', { value: 'old@x.com' }), + ]), + }) + const after = makeCapture({ + root: el('root', 'window', {}, [ + el('email', 'text_input', { value: 'new@x.com' }), + ]), + }) + + const diff = diffDesktopCaptures(before, after) + expect(diff.no_changes).toBe(false) + expect(diff.summary.changed).toBe(1) + expect(diff.elements_changed[0]).toMatchObject({ + id: 'email', + role: 'text_input', + changes: { value: { from: 'old@x.com', to: 'new@x.com' } }, + }) + }) + + it('reports enabled flips and aria.checked flips', () => { + const before = makeCapture({ + root: el('root', 'window', {}, [ + el('btn', 'button', { enabled: false, aria: { checked: false } }), + ]), + }) + const after = makeCapture({ + root: el('root', 'window', {}, [ + el('btn', 'button', { enabled: true, aria: { checked: true } }), + ]), + }) + + const diff = diffDesktopCaptures(before, after) + const change = diff.elements_changed.find((c) => c.id === 'btn') + expect(change?.changes.enabled).toEqual({ from: false, to: true }) + expect(change?.changes['aria.checked']).toEqual({ from: false, to: true }) + }) + + it('ignores sub-pixel bounds jitter (<=1px)', () => { + const before = makeCapture({ + root: el('root', 'window', { bounds: { x: 100, y: 200, width: 800, height: 600 } }), + }) + const after = makeCapture({ + root: el('root', 'window', { bounds: { x: 100.5, y: 200, width: 800, height: 600.5 } }), + }) + + const diff = diffDesktopCaptures(before, after) + expect(diff.no_changes).toBe(true) + }) + + it('reports bounds change when the shift exceeds 1px', () => { + const before = makeCapture({ + root: el('root', 'window', { bounds: { x: 100, y: 200, width: 800, height: 600 } }), + }) + const after = makeCapture({ + root: el('root', 'window', { bounds: { x: 100, y: 200, width: 1200, height: 600 } }), + }) + + const diff = diffDesktopCaptures(before, after) + const rootChange = diff.elements_changed.find((c) => c.id === 'root') + expect(rootChange?.changes.bounds).toBeDefined() + }) +}) + +describe('diffDesktopCaptures — added + removed', () => { + it('reports elements added in the after tree', () => { + const before = makeCapture({ + root: el('root', 'window', {}, [el('a', 'button', { name: 'A' })]), + }) + const after = makeCapture({ + root: el('root', 'window', {}, [ + el('a', 'button', { name: 'A' }), + el('b', 'button', { name: 'B (new)' }), + ]), + }) + + const diff = diffDesktopCaptures(before, after) + expect(diff.summary).toMatchObject({ added: 1, removed: 0 }) + expect(diff.elements_added[0]).toMatchObject({ id: 'b', name: 'B (new)' }) + }) + + it('reports elements removed when a dialog closes', () => { + const before = makeCapture({ + root: el('root', 'window', {}, [ + el('main', 'pane'), + el('dialog', 'dialog', { name: 'Confirm' }, [ + el('ok', 'button', { name: 'OK' }), + el('cancel', 'button', { name: 'Cancel' }), + ]), + ]), + }) + const after = makeCapture({ + root: el('root', 'window', {}, [el('main', 'pane')]), + }) + + const diff = diffDesktopCaptures(before, after) + expect(diff.summary.removed).toBe(3) // dialog + ok + cancel + const removedIds = diff.elements_removed.map((e) => e.id).sort() + expect(removedIds).toEqual(['cancel', 'dialog', 'ok']) + }) +}) + +describe('diffDesktopCaptures — window-level changes', () => { + it('reports window_title_changed', () => { + const before = makeCapture({ root: el('root', 'window'), window_title: 'Untitled' }) + const after = makeCapture({ root: el('root', 'window'), window_title: 'Untitled — Saved' }) + const diff = diffDesktopCaptures(before, after) + expect(diff.no_changes).toBe(false) + expect(diff.window_title_changed).toEqual({ from: 'Untitled', to: 'Untitled — Saved' }) + }) + + it('reports focus_changed', () => { + const before = makeCapture({ root: el('root', 'window'), focused_element_id: 'a' }) + const after = makeCapture({ root: el('root', 'window'), focused_element_id: 'b' }) + const diff = diffDesktopCaptures(before, after) + expect(diff.focus_changed).toEqual({ from: 'a', to: 'b' }) + }) +}) diff --git a/test/mcp/desktop-dispatcher.test.ts b/test/mcp/desktop-dispatcher.test.ts index c5d89b5..6d98325 100644 --- a/test/mcp/desktop-dispatcher.test.ts +++ b/test/mcp/desktop-dispatcher.test.ts @@ -279,6 +279,54 @@ describe('MCP — desktop tools', () => { expect(batch.text).toContain('No cached snapshot') }) + it('agentmark_desktop_diff returns no_changes when nothing happened between snapshots', async () => { + const open = await dispatch(state, 'agentmark_desktop_open', {}) + const { desktop_id } = JSON.parse(open.text) + + await dispatch(state, 'agentmark_desktop_snapshot', { + desktop_id, + target: { window_id: 'nowcerts_customer' }, + }) + + const diff = await dispatch(state, 'agentmark_desktop_diff', { desktop_id }) + expect(diff.isError).toBeFalsy() + const body = JSON.parse(diff.text) + expect(body.no_changes).toBe(true) + }) + + it('agentmark_desktop_diff surfaces the value change after an execute', async () => { + const open = await dispatch(state, 'agentmark_desktop_open', {}) + const { desktop_id } = JSON.parse(open.text) + + await dispatch(state, 'agentmark_desktop_snapshot', { + desktop_id, + target: { window_id: 'nowcerts_customer' }, + }) + + await dispatch(state, 'agentmark_desktop_execute', { + desktop_id, + action_id: 'act_in_company', + value: 'Globex Corp', + }) + + const diff = await dispatch(state, 'agentmark_desktop_diff', { desktop_id }) + expect(diff.isError).toBeFalsy() + const body = JSON.parse(diff.text) + expect(body.no_changes).toBe(false) + expect(body.summary.changed).toBeGreaterThanOrEqual(1) + const companyChange = body.elements_changed.find((c: { id: string }) => c.id === 'in_company') + expect(companyChange?.changes?.value?.to).toBe('Globex Corp') + }) + + it('agentmark_desktop_diff errors when no snapshot has been captured yet', async () => { + const open = await dispatch(state, 'agentmark_desktop_open', {}) + const { desktop_id } = JSON.parse(open.text) + + const diff = await dispatch(state, 'agentmark_desktop_diff', { desktop_id }) + expect(diff.isError).toBe(true) + expect(diff.text).toContain('No cached capture') + }) + it('agentmark_desktop_close removes the session', async () => { const open = await dispatch(state, 'agentmark_desktop_open', {}) const { desktop_id } = JSON.parse(open.text)