diff --git a/src/desktop/fingerprint.ts b/src/desktop/fingerprint.ts new file mode 100644 index 0000000..cabb5b2 --- /dev/null +++ b/src/desktop/fingerprint.ts @@ -0,0 +1,195 @@ +/** + * Self-healing element resolution via structural fingerprints. + * + * The problem: Recipe steps reference elements by `action_id`, which the + * binding resolves to a backend `element_id`. Those IDs are sometimes + * stable (UIA AutomationId), sometimes generated, sometimes drift + * between app versions. An agent that learned "click element_id=btn_save_4711" + * yesterday may find that ID missing today even though the Save button + * is right there. + * + * The fix: at snapshot time, compute a structural fingerprint per + * element from its role + name + neighbours + parent. At replay time, + * if the original element_id is gone, search the current tree for the + * best fingerprint match. + * + * v0 scope: the primitive only — `computeFingerprint` + `findByFingerprint` + * exposed as MCP tools the agent calls explicitly. Automatic healing + * inside the execute handler comes in a follow-up. + */ +import type { DesktopCapture, DesktopElement } from './types' + +/** + * Compact structural signature for an element. Hand-tuned for tolerance: + * primary keys are role + name; siblings + parent are secondary signals + * that help disambiguate when name alone isn't enough. + */ +export interface ElementFingerprint { + role: string + name?: string + value?: string + placeholder?: string + /** Depth from the root (0 = root). */ + depth: number + /** Parent's role + name for context. */ + parent_role?: string + parent_name?: string + /** Names + roles of the immediate siblings on either side. */ + preceding_sibling?: { role: string; name?: string } + following_sibling?: { role: string; name?: string } +} + +/** + * Walk the capture tree to find the element with the given id; return + * its fingerprint. Returns null when the id isn't found. + */ +export function computeFingerprint( + capture: DesktopCapture, + elementId: string, +): ElementFingerprint | null { + const target = findElement(capture.root, elementId) + if (!target) return null + return fingerprintFor(target.element, target.parent, target.depth, target.indexInParent) +} + +/** + * Find the element in `capture` that best matches `fp`. Returns the + * matching element_id + a confidence score (0–100). Returns null when + * no candidate scores above `minScore`. + */ +export function findByFingerprint( + capture: DesktopCapture, + fp: ElementFingerprint, + options: { minScore?: number } = {}, +): { element_id: string; score: number } | null { + const minScore = options.minScore ?? 60 + + interface Candidate { id: string; score: number } + const candidates: Candidate[] = [] + walkWithContext(capture.root, undefined, 0, 0, (el, parent, depth, indexInParent) => { + if (el.role !== fp.role) return // hard requirement + const candidate = fingerprintFor(el, parent, depth, indexInParent) + const score = scoreFingerprintMatch(fp, candidate) + candidates.push({ id: el.id, score }) + }) + + candidates.sort((a, b) => b.score - a.score) + const best = candidates[0] + if (!best || best.score < minScore) return null + return { element_id: best.id, score: best.score } +} + +/** + * Score how well two fingerprints match (0–100). Tuned so that an + * exact role+name match scores ~70 (clears the default threshold) and + * full sibling+parent context pushes it toward 100. + */ +export function scoreFingerprintMatch( + target: ElementFingerprint, + candidate: ElementFingerprint, +): number { + if (target.role !== candidate.role) return 0 + let score = 10 // baseline for matching role + + if (target.name && target.name === candidate.name) score += 60 + else if (target.name && candidate.name && stringSimilarity(target.name, candidate.name) > 0.8) score += 40 + + if (target.parent_name && target.parent_name === candidate.parent_name) score += 10 + if (target.parent_role && target.parent_role === candidate.parent_role) score += 5 + + if (siblingMatches(target.preceding_sibling, candidate.preceding_sibling)) score += 5 + if (siblingMatches(target.following_sibling, candidate.following_sibling)) score += 5 + + if (target.placeholder && target.placeholder === candidate.placeholder) score += 5 + + return Math.min(100, score) +} + +function siblingMatches( + a: ElementFingerprint['preceding_sibling'], + b: ElementFingerprint['preceding_sibling'], +): boolean { + if (!a && !b) return true + if (!a || !b) return false + return a.role === b.role && a.name === b.name +} + +/** + * Very-cheap string similarity (Jaccard over character bigrams). Used + * to give partial credit when names drift slightly ("Save" vs "Save..." + * or "Email" vs "Email Address"). + */ +function stringSimilarity(a: string, b: string): number { + if (a === b) return 1 + if (a.length < 2 || b.length < 2) return 0 + const grams = (s: string) => { + const set = new Set() + for (let i = 0; i < s.length - 1; i++) set.add(s.slice(i, i + 2).toLowerCase()) + return set + } + const A = grams(a) + const B = grams(b) + let intersection = 0 + for (const g of A) if (B.has(g)) intersection++ + const union = A.size + B.size - intersection + return union === 0 ? 0 : intersection / union +} + +function fingerprintFor( + el: DesktopElement, + parent: DesktopElement | undefined, + depth: number, + indexInParent: number, +): ElementFingerprint { + const siblings = parent?.children ?? [] + const prec = indexInParent > 0 ? siblings[indexInParent - 1] : undefined + const foll = indexInParent < siblings.length - 1 ? siblings[indexInParent + 1] : undefined + + return { + role: el.role, + name: el.name, + value: el.value, + placeholder: el.placeholder, + depth, + parent_role: parent?.role, + parent_name: parent?.name, + preceding_sibling: prec ? { role: prec.role, name: prec.name } : undefined, + following_sibling: foll ? { role: foll.role, name: foll.name } : undefined, + } +} + +interface FoundElement { + element: DesktopElement + parent?: DesktopElement + depth: number + indexInParent: number +} + +function findElement(root: DesktopElement, id: string): FoundElement | null { + let found: FoundElement | null = null + walkWithContext(root, undefined, 0, 0, (el, parent, depth, indexInParent) => { + if (!found && el.id === id) { + found = { element: el, parent, depth, indexInParent } + } + }) + return found +} + +function walkWithContext( + el: DesktopElement, + parent: DesktopElement | undefined, + depth: number, + indexInParent: number, + visit: ( + el: DesktopElement, + parent: DesktopElement | undefined, + depth: number, + indexInParent: number, + ) => void, +): void { + visit(el, parent, depth, indexInParent) + const children = el.children ?? [] + for (let i = 0; i < children.length; i++) { + walkWithContext(children[i], el, depth + 1, i, visit) + } +} diff --git a/src/desktop/index.ts b/src/desktop/index.ts index 92f0880..8aff818 100644 --- a/src/desktop/index.ts +++ b/src/desktop/index.ts @@ -12,6 +12,13 @@ export type { DesktopElementSummary, } from './diff' +export { + computeFingerprint, + findByFingerprint, + scoreFingerprintMatch, +} from './fingerprint' +export type { ElementFingerprint } from './fingerprint' + export { FixtureBackend } from './fixture-backend' export type { FixtureBackendOptions } from './fixture-backend' diff --git a/src/index.ts b/src/index.ts index e466d91..869ae53 100644 --- a/src/index.ts +++ b/src/index.ts @@ -217,6 +217,9 @@ export type { export { convertDesktop, diffDesktopCaptures, + computeFingerprint, + findByFingerprint, + scoreFingerprintMatch, FixtureBackend, WindowsUiaBackend, MacosAxapiBackend, @@ -238,6 +241,7 @@ export type { DesktopElementSummary, DesktopRole, DesktopTarget, + ElementFingerprint, ExecuteDesktopOptions, ExecuteDesktopAction, ExecuteDesktopBatchOptions, diff --git a/src/mcp/plugins/desktop.ts b/src/mcp/plugins/desktop.ts index 1563d2b..fb8e2c6 100644 --- a/src/mcp/plugins/desktop.ts +++ b/src/mcp/plugins/desktop.ts @@ -5,14 +5,17 @@ * owns its own DesktopSession map. */ import { + computeFingerprint, convertDesktop, diffDesktopCaptures, + findByFingerprint, FixtureBackend, MacosAxapiBackend, WindowsUiaBackend, parseSnapshot, type DesktopCaptureBackend, type DesktopTarget, + type ElementFingerprint, type ExecuteDesktopAction, type KeyModifier, } from '../../index' @@ -239,6 +242,54 @@ const DESKTOP_TOOLS: McpToolDef[] = [ required: ['desktop_id', 'actions'], }, }, + { + name: 'agentmark_desktop_fingerprint', + description: + 'Compute a structural fingerprint for an element in the current ' + + 'snapshot. The fingerprint captures role + name + parent context + ' + + 'adjacent siblings — stable across most UI changes that drift ' + + 'native element IDs. Save fingerprints with your Recipes so ' + + 'agentmark_desktop_find_by_fingerprint can recover when an ' + + 'element_id goes stale.\n' + + '\nPass `action_id` to resolve via the snapshot binding, or ' + + '`element_id` directly. Returns null when the target isn\'t in ' + + 'the current snapshot.', + inputSchema: { + type: 'object', + properties: { + desktop_id: { type: 'string' }, + action_id: { type: 'string', description: 'Resolved via the snapshot binding (action_id → element_id).' }, + element_id: { type: 'string', description: 'Raw backend element id, used when you have it directly.' }, + }, + required: ['desktop_id'], + }, + }, + { + name: 'agentmark_desktop_find_by_fingerprint', + description: + 'Search the current snapshot for the element best matching a ' + + 'previously-computed fingerprint. Returns the matched element_id ' + + 'and a confidence score (0–100). Returns null when no candidate ' + + 'clears the minimum score (default 60).\n' + + '\nScoring favours role+name matches; full sibling + parent ' + + 'context pushes scores toward 100. Use for Recipe replay when ' + + 'a saved element_id no longer resolves.', + inputSchema: { + type: 'object', + properties: { + desktop_id: { type: 'string' }, + fingerprint: { + type: 'object', + description: 'Output of agentmark_desktop_fingerprint.', + }, + min_score: { + type: 'number', + description: 'Minimum confidence (0–100). Default: 60.', + }, + }, + required: ['desktop_id', 'fingerprint'], + }, + }, ] export interface DesktopPlugin extends AgentMarkPlugin { @@ -351,6 +402,80 @@ export function createDesktopPlugin(): DesktopPlugin { return { text: agentmark } }, + agentmark_desktop_fingerprint: async (args): Promise => { + const id = requireString(args, 'desktop_id') + const session = requireDesktop(id) + + if (!session.lastCapture) { + return { + text: + `No cached capture for desktop_id ${id}. Call ` + + `agentmark_desktop_snapshot first.`, + isError: true, + } + } + + // Resolve to a backend element_id. The action_id path is + // more agent-friendly (action ids appear in snapshots); the + // element_id path is for callers that already have the raw + // backend id. + let elementId: string | undefined + if (typeof args.action_id === 'string') { + if (!session.lastBinding) { + return { text: `No snapshot binding; capture first.`, isError: true } + } + elementId = session.lastBinding.get(args.action_id) + if (!elementId) { + return { text: `Unknown action_id: ${args.action_id}`, isError: true } + } + } else if (typeof args.element_id === 'string') { + elementId = args.element_id + } else { + return { text: '`action_id` or `element_id` is required.', isError: true } + } + + const fingerprint = computeFingerprint(session.lastCapture, elementId) + if (!fingerprint) { + return { + text: JSON.stringify({ found: false, element_id: elementId }, null, 2), + isError: true, + } + } + return { text: JSON.stringify({ found: true, element_id: elementId, fingerprint }, null, 2) } + }, + + agentmark_desktop_find_by_fingerprint: async (args): Promise => { + const id = requireString(args, 'desktop_id') + const session = requireDesktop(id) + + if (!session.lastCapture) { + return { + text: + `No cached capture for desktop_id ${id}. Call ` + + `agentmark_desktop_snapshot first.`, + isError: true, + } + } + + if (!args.fingerprint || typeof args.fingerprint !== 'object') { + return { text: '`fingerprint` must be an object.', isError: true } + } + const fp = args.fingerprint as ElementFingerprint + if (typeof fp.role !== 'string') { + return { text: '`fingerprint.role` is required.', isError: true } + } + const minScore = typeof args.min_score === 'number' ? args.min_score : 60 + + const match = findByFingerprint(session.lastCapture, fp, { minScore }) + if (!match) { + return { + text: JSON.stringify({ found: false, min_score: minScore }, null, 2), + isError: true, + } + } + return { text: JSON.stringify({ found: true, ...match }, null, 2) } + }, + agentmark_desktop_diff: async (args): Promise => { const id = requireString(args, 'desktop_id') const session = requireDesktop(id) diff --git a/test/desktop/fingerprint.test.ts b/test/desktop/fingerprint.test.ts new file mode 100644 index 0000000..5b4936b --- /dev/null +++ b/test/desktop/fingerprint.test.ts @@ -0,0 +1,154 @@ +/** + * Tests for the self-healing fingerprint primitives. + */ +import { describe, it, expect } from 'vitest' +import { + computeFingerprint, + findByFingerprint, + scoreFingerprintMatch, +} from '../../src/desktop/fingerprint' +import type { DesktopCapture, DesktopElement } from '../../src/desktop/types' + +function el(id: string, role: string, fields: Partial = {}, children?: DesktopElement[]): DesktopElement { + return { id, role: role as DesktopElement['role'], children, ...fields } +} + +function capture(root: DesktopElement, overrides: Partial = {}): DesktopCapture { + return { + platform: 'windows', + window_title: 'Test', + tree_depth: 3, + element_count: 0, + root, + ...overrides, + } +} + +describe('computeFingerprint', () => { + it('captures role + name + parent + sibling context', () => { + const tree = capture( + el('root', 'window', { name: 'Form' }, [ + el('label_email', 'label', { name: 'Email' }), + el('input_email', 'text_input', { name: 'Email', placeholder: 'you@example.com' }), + el('label_phone', 'label', { name: 'Phone' }), + ]), + ) + const fp = computeFingerprint(tree, 'input_email') + expect(fp).toEqual({ + role: 'text_input', + name: 'Email', + value: undefined, + placeholder: 'you@example.com', + depth: 1, + parent_role: 'window', + parent_name: 'Form', + preceding_sibling: { role: 'label', name: 'Email' }, + following_sibling: { role: 'label', name: 'Phone' }, + }) + }) + + it('returns null for unknown ids', () => { + const tree = capture(el('root', 'window')) + expect(computeFingerprint(tree, 'nope')).toBeNull() + }) +}) + +describe('scoreFingerprintMatch', () => { + it('returns 0 when roles differ (hard requirement)', () => { + const a = { role: 'button', name: 'Save', depth: 1 } + const b = { role: 'text_input', name: 'Save', depth: 1 } + expect(scoreFingerprintMatch(a, b)).toBe(0) + }) + + it('exact role + name clears the default 60 threshold', () => { + const fp = { role: 'button', name: 'Save', depth: 1 } + expect(scoreFingerprintMatch(fp, fp)).toBeGreaterThanOrEqual(60) + }) + + it('partial-name match gives partial credit', () => { + const target = { role: 'button', name: 'Save' as const, depth: 1 } + const candidate = { role: 'button', name: 'Save...' as const, depth: 1 } + const score = scoreFingerprintMatch(target, candidate) + expect(score).toBeGreaterThan(0) + expect(score).toBeLessThan(scoreFingerprintMatch(target, target)) + }) + + it('parent context + siblings push toward 100', () => { + const target = { + role: 'button', + name: 'Save', + depth: 2, + parent_role: 'pane', + parent_name: 'Sidebar', + preceding_sibling: { role: 'button', name: 'Cancel' }, + following_sibling: { role: 'button', name: 'Delete' }, + } + const score = scoreFingerprintMatch(target, target) + expect(score).toBeGreaterThanOrEqual(95) + }) +}) + +describe('findByFingerprint', () => { + it('finds the same element after the ID changed but structure is identical', () => { + const yesterday = capture( + el('root', 'window', { name: 'Form' }, [ + el('label_email', 'label', { name: 'Email' }), + el('input_email_OLD', 'text_input', { name: 'Email' }), + ]), + ) + const today = capture( + el('root_v2', 'window', { name: 'Form' }, [ + el('label_email_v2', 'label', { name: 'Email' }), + el('input_email_NEW_v8', 'text_input', { name: 'Email' }), + ]), + ) + + const fp = computeFingerprint(yesterday, 'input_email_OLD')! + const match = findByFingerprint(today, fp) + expect(match).not.toBeNull() + expect(match!.element_id).toBe('input_email_NEW_v8') + expect(match!.score).toBeGreaterThanOrEqual(60) + }) + + it('returns null when no candidate clears the threshold', () => { + const tree = capture( + el('root', 'window', {}, [el('a', 'button', { name: 'Foo' })]), + ) + const fp = { role: 'text_input', name: 'Email', depth: 1 } + expect(findByFingerprint(tree, fp)).toBeNull() + }) + + it('disambiguates between similar elements via sibling context', () => { + // Two "Email" inputs in different sections — siblings should + // resolve which one is which. + const tree = capture( + el('root', 'window', {}, [ + el('billing_section', 'pane', { name: 'Billing' }, [ + el('billing_label', 'label', { name: 'Email' }), + el('billing_email', 'text_input', { name: 'Email' }), + ]), + el('shipping_section', 'pane', { name: 'Shipping' }, [ + el('shipping_label', 'label', { name: 'Email' }), + el('shipping_email', 'text_input', { name: 'Email' }), + ]), + ]), + ) + + const shippingFp = computeFingerprint(tree, 'shipping_email')! + // Pretend IDs all rotated by re-fingerprinting against the same tree. + const found = findByFingerprint(tree, shippingFp) + expect(found!.element_id).toBe('shipping_email') + }) + + it('respects custom min_score threshold', () => { + const tree = capture( + el('root', 'window', {}, [el('btn', 'button', { name: 'Save' })]), + ) + const fp = { role: 'button', name: 'Different', depth: 1 } + // Default threshold rejects role-only match. + expect(findByFingerprint(tree, fp)).toBeNull() + // Lower threshold accepts the role-only baseline score. + const match = findByFingerprint(tree, fp, { minScore: 5 }) + expect(match).not.toBeNull() + }) +}) diff --git a/test/mcp/desktop-dispatcher.test.ts b/test/mcp/desktop-dispatcher.test.ts index 6d98325..fa22e48 100644 --- a/test/mcp/desktop-dispatcher.test.ts +++ b/test/mcp/desktop-dispatcher.test.ts @@ -327,6 +327,79 @@ describe('MCP — desktop tools', () => { expect(diff.text).toContain('No cached capture') }) + it('agentmark_desktop_fingerprint returns a structural signature for an action_id', async () => { + const open = await dispatch(state, 'agentmark_desktop_open', {}) + const { desktop_id } = JSON.parse(open.text) + + await dispatch(state, 'agentmark_desktop_snapshot', { + desktop_id, + target: { window_id: 'nowcerts_customer' }, + }) + + const result = await dispatch(state, 'agentmark_desktop_fingerprint', { + desktop_id, + action_id: 'act_in_company', + }) + expect(result.isError).toBeFalsy() + const body = JSON.parse(result.text) + expect(body.found).toBe(true) + expect(body.element_id).toBe('in_company') + expect(body.fingerprint.role).toBe('text_input') + }) + + it('agentmark_desktop_fingerprint errors when neither action_id nor element_id is supplied', async () => { + const open = await dispatch(state, 'agentmark_desktop_open', {}) + const { desktop_id } = JSON.parse(open.text) + await dispatch(state, 'agentmark_desktop_snapshot', { desktop_id }) + + const result = await dispatch(state, 'agentmark_desktop_fingerprint', { desktop_id }) + expect(result.isError).toBe(true) + expect(result.text).toMatch(/action_id.*element_id/) + }) + + it('agentmark_desktop_find_by_fingerprint resolves a saved fingerprint back to the current element', async () => { + const open = await dispatch(state, 'agentmark_desktop_open', {}) + const { desktop_id } = JSON.parse(open.text) + + await dispatch(state, 'agentmark_desktop_snapshot', { + desktop_id, + target: { window_id: 'nowcerts_customer' }, + }) + + const fpResult = await dispatch(state, 'agentmark_desktop_fingerprint', { + desktop_id, + action_id: 'act_in_company', + }) + const { fingerprint } = JSON.parse(fpResult.text) + + const found = await dispatch(state, 'agentmark_desktop_find_by_fingerprint', { + desktop_id, + fingerprint, + }) + expect(found.isError).toBeFalsy() + const body = JSON.parse(found.text) + expect(body.found).toBe(true) + expect(body.element_id).toBe('in_company') + expect(body.score).toBeGreaterThanOrEqual(60) + }) + + it('agentmark_desktop_find_by_fingerprint returns isError + found=false when nothing matches', async () => { + const open = await dispatch(state, 'agentmark_desktop_open', {}) + const { desktop_id } = JSON.parse(open.text) + await dispatch(state, 'agentmark_desktop_snapshot', { + desktop_id, + target: { window_id: 'nowcerts_customer' }, + }) + + const found = await dispatch(state, 'agentmark_desktop_find_by_fingerprint', { + desktop_id, + fingerprint: { role: 'text_input', name: 'A field that does not exist anywhere', depth: 5 }, + }) + expect(found.isError).toBe(true) + const body = JSON.parse(found.text) + expect(body.found).toBe(false) + }) + it('agentmark_desktop_close removes the session', async () => { const open = await dispatch(state, 'agentmark_desktop_open', {}) const { desktop_id } = JSON.parse(open.text)