diff --git a/src/desktop/fixture-backend.ts b/src/desktop/fixture-backend.ts index b27d794..9b6c585 100644 --- a/src/desktop/fixture-backend.ts +++ b/src/desktop/fixture-backend.ts @@ -18,6 +18,7 @@ import type { DesktopCaptureBackend, CaptureDesktopOptions, DesktopElement, + DesktopTargetSummary, ExecuteDesktopOptions, ExecuteDesktopResult, } from './types' @@ -51,6 +52,22 @@ export class FixtureBackend implements DesktopCaptureBackend { this.latencyMs = opts.latencyMs ?? 0 } + async listTargets(): Promise { + if (this.latencyMs) await delay(this.latencyMs) + + // One summary per registered preset. Use the preset key as + // window_id so a subsequent capture({ target: { window_id: key }}) + // resolves back to the same preset. + return Object.entries(this.presets).map(([key, cap]) => ({ + window_id: key, + process_name: cap.process_name, + process_id: cap.process_id, + window_title: cap.window_title, + window_class: cap.window_class, + has_focus: key === this.defaultPreset, + })) + } + async capture(opts: CaptureDesktopOptions = {}): Promise { if (this.latencyMs) await delay(this.latencyMs) diff --git a/src/desktop/types.ts b/src/desktop/types.ts index 54e897c..7692a07 100644 --- a/src/desktop/types.ts +++ b/src/desktop/types.ts @@ -23,6 +23,11 @@ export interface DesktopCaptureBackend { * or `fixture` for tests. */ readonly name: string + /** Enumerate top-level windows the backend can see. Returns lightweight + * summaries (no element tree) so an agent can pick a target before + * paying for the full capture. */ + listTargets(): Promise + /** Capture the current state of a target window/application. */ capture(opts: CaptureDesktopOptions): Promise @@ -34,6 +39,23 @@ export interface DesktopCaptureBackend { close?(): Promise } +/** Lightweight summary of one open window — what `listTargets()` returns. */ +export interface DesktopTargetSummary { + /** Opaque handle the caller passes back as `target.window_id` to + * capture this specific window. Format is backend-defined + * (Windows: `hwnd:0x...`; macOS: `axapi::`; fixture: + * the preset key). */ + window_id: string + process_name?: string + process_id?: number + window_title: string + /** Toolkit / class hint — Windows class name (e.g. `XLMAIN`), + * macOS AX role (e.g. `AXWindow`), useful for UI inspectors. */ + window_class?: string + /** True when this window currently has keyboard focus. */ + has_focus: boolean +} + /** Identifies a target window. Backends accept any combination they can * resolve; if nothing is provided the currently focused window is used. */ export interface DesktopTarget { diff --git a/src/desktop/windows-uia-backend.ts b/src/desktop/windows-uia-backend.ts index 6a0b88d..28874b3 100644 --- a/src/desktop/windows-uia-backend.ts +++ b/src/desktop/windows-uia-backend.ts @@ -45,6 +45,7 @@ import type { CaptureDesktopOptions, DesktopCapture, DesktopCaptureBackend, + DesktopTargetSummary, ExecuteDesktopAction, ExecuteDesktopOptions, ExecuteDesktopResult, @@ -118,6 +119,20 @@ export class WindowsUiaBackend implements DesktopCaptureBackend { // ── DesktopCaptureBackend implementation ───────────────────────── + async listTargets(): Promise { + await this.ensureStarted() + const result = (await this.call('list_windows', {})) as { windows?: RawWindowSummary[] } + const raw = result?.windows ?? [] + return raw.map((w) => ({ + window_id: w.windowId, + process_name: w.processName ?? undefined, + process_id: w.processId ?? undefined, + window_title: w.windowTitle, + window_class: w.windowClass ?? undefined, + has_focus: !!w.hasFocus, + })) + } + async capture(opts: CaptureDesktopOptions = {}): Promise { await this.ensureStarted() const params = { @@ -416,6 +431,15 @@ function resolveBridgePath(): string { // Wire format — raw shapes returned by the bridge // ────────────────────────────────────────────────────────────────────── +interface RawWindowSummary { + windowId: string + processName?: string | null + processId?: number | null + windowTitle: string + windowClass?: string | null + hasFocus: boolean +} + interface RawDesktopCapture { platform: 'windows' | 'macos' | 'linux' processName?: string | null diff --git a/src/mcp/dispatcher.ts b/src/mcp/dispatcher.ts index 7cf9df5..6efeb09 100644 --- a/src/mcp/dispatcher.ts +++ b/src/mcp/dispatcher.ts @@ -98,6 +98,8 @@ export async function dispatch( return await openDesktop(state, args) case 'agentmark_desktop_close': return await closeDesktop(state, args) + case 'agentmark_desktop_list_targets': + return await desktopListTargets(state, args) case 'agentmark_desktop_snapshot': return await desktopSnapshot(state, args) case 'agentmark_desktop_execute': @@ -395,6 +397,15 @@ async function closeDesktop(state: DispatcherState, args: Record): Promise { + const id = requireString(args, 'desktop_id') + const session = requireDesktop(state, id) + const windows = await session.backend.listTargets() + return { + text: JSON.stringify({ windows }, null, 2), + } +} + async function desktopSnapshot(state: DispatcherState, args: Record): Promise { const id = requireString(args, 'desktop_id') const session = requireDesktop(state, id) diff --git a/src/mcp/tool-defs.ts b/src/mcp/tool-defs.ts index 463b6f2..70aa8a0 100644 --- a/src/mcp/tool-defs.ts +++ b/src/mcp/tool-defs.ts @@ -322,6 +322,23 @@ const DESKTOP_TOOLS: McpToolDef[] = [ required: ['desktop_id'], }, }, + { + name: 'agentmark_desktop_list_targets', + description: + 'Enumerate top-level windows the backend can see. Returns one ' + + 'lightweight summary per window (process_name, process_id, ' + + 'window_title, window_class, window_id, has_focus) without ' + + 'walking the full element tree. Use this to let the user (or ' + + 'agent) pick a window before calling agentmark_desktop_snapshot ' + + 'with that specific window_id.', + inputSchema: { + type: 'object', + properties: { + desktop_id: { type: 'string' }, + }, + required: ['desktop_id'], + }, + }, { name: 'agentmark_desktop_snapshot', description: diff --git a/test/desktop/windows-uia-backend.test.ts b/test/desktop/windows-uia-backend.test.ts index b731bf5..e15e692 100644 --- a/test/desktop/windows-uia-backend.test.ts +++ b/test/desktop/windows-uia-backend.test.ts @@ -42,6 +42,19 @@ describe('WindowsUiaBackend', () => { expect(() => new WindowsUiaBackend({ bridgePath: FAKE_BRIDGE })).toThrow(/requires Windows/) }) + it('listTargets maps the bridge `windows` array to snake_case DesktopTargetSummary', async () => { + backend = makeBackend() + const targets = await backend.listTargets() + expect(targets.length).toBeGreaterThan(0) + const w = targets[0] + // The fake bridge ships one synthetic window. + expect(w.window_id).toMatch(/^hwnd:/) + expect(w.window_title).toBe('Fake Window 1') + expect(w.process_name).toBe('FakeApp.exe') + expect(w.process_id).toBe(42) + expect(w.has_focus).toBe(true) + }) + it('captures via the bridge and maps the response to DesktopCapture', async () => { backend = makeBackend() const cap = await backend.capture({}) diff --git a/test/mcp/desktop-dispatcher.test.ts b/test/mcp/desktop-dispatcher.test.ts index d6866a9..a423058 100644 --- a/test/mcp/desktop-dispatcher.test.ts +++ b/test/mcp/desktop-dispatcher.test.ts @@ -27,14 +27,45 @@ afterEach(async () => { }) describe('MCP — desktop tools', () => { - it('registers all four desktop tools in ALL_TOOLS', () => { + it('registers all five desktop tools in ALL_TOOLS', () => { const names = ALL_TOOLS.map(t => t.name) expect(names).toContain('agentmark_desktop_open') expect(names).toContain('agentmark_desktop_close') + expect(names).toContain('agentmark_desktop_list_targets') expect(names).toContain('agentmark_desktop_snapshot') expect(names).toContain('agentmark_desktop_execute') }) + it('agentmark_desktop_list_targets returns the fixture preset windows', async () => { + const open = await dispatch(state, 'agentmark_desktop_open', {}) + const { desktop_id } = JSON.parse(open.text) + + const result = await dispatch(state, 'agentmark_desktop_list_targets', { desktop_id }) + expect(result.isError).toBeFalsy() + + const body = JSON.parse(result.text) + expect(Array.isArray(body.windows)).toBe(true) + // Fixture ships two preset shapes: excel_blank and nowcerts_customer. + // Each is exposed under two keys (excel + excel_blank; nowcerts + nowcerts_customer) + // so length is 4. Don't over-constrain on the exact number; just + // require both preset families are represented. + const titles = body.windows.map((w: { window_title: string }) => w.window_title) + expect(titles.some((t: string) => t.includes('Excel'))).toBe(true) + expect(titles.some((t: string) => t.includes('NowCerts'))).toBe(true) + // Every entry has the right shape. + for (const w of body.windows) { + expect(w).toHaveProperty('window_id') + expect(w).toHaveProperty('window_title') + expect(typeof w.has_focus).toBe('boolean') + } + }) + + it('agentmark_desktop_list_targets errors on unknown desktop_id', async () => { + const result = await dispatch(state, 'agentmark_desktop_list_targets', { desktop_id: 'dt_missing' }) + expect(result.isError).toBe(true) + expect(result.text).toContain('Unknown desktop_id') + }) + it('agentmark_desktop_open with no args defaults to the fixture backend', async () => { const result = await dispatch(state, 'agentmark_desktop_open', {}) expect(result.isError).toBeFalsy()