diff --git a/src/mcp/index.ts b/src/mcp/index.ts index a3fdf7f..e23f627 100644 --- a/src/mcp/index.ts +++ b/src/mcp/index.ts @@ -63,6 +63,18 @@ export type { QueuedMessage, } from '../plugins/network' +// Vision Pack — Layer 2 fallback (screenshot tool). Opt-in. +export { + createVisionPlugin, + captureScreenshot, + VISION_TOOLS, +} from '../plugins/vision' +export type { + VisionPluginConfig, + ScreenshotOptions, + ScreenshotResult, +} from '../plugins/vision' + // Microsoft Workflows Pack (Graph-only v0) — opt-in; not part of the // default plugin set. Pass it explicitly via `createMcpServer({ plugins })`. export { diff --git a/src/plugins/vision/index.ts b/src/plugins/vision/index.ts new file mode 100644 index 0000000..f768564 --- /dev/null +++ b/src/plugins/vision/index.ts @@ -0,0 +1,41 @@ +/** + * Vision Pack — Layer 2 fallback for accessibility-tree gaps. + * + * Right now ships one tool: full-screen screenshot. The multimodal AI + * does its own parsing on the returned image. As we expand: + * + * - per-window capture via bridges + * - OmniParser-style structured GUI parsing inside the plugin + * - "tree was sparse, auto-fallback to vision" hook on desktop_snapshot + */ +import { captureScreenshot } from './screenshot' +import { VISION_TOOLS } from './tool-defs' +import type { AgentMarkPlugin, DispatchResult, ToolHandler } from '../../mcp/plugin' + +export interface VisionPluginConfig { + /** Reserved for future config (preferred OCR backend, model hints, etc.). */ + _reserved?: never +} + +export function createVisionPlugin(_config: VisionPluginConfig = {}): AgentMarkPlugin { + const handlers: Record = { + agentmark_screenshot: async (args): Promise => { + const result = await captureScreenshot({ + outputPath: typeof args.output_path === 'string' ? args.output_path : undefined, + displayIndex: typeof args.display_index === 'number' ? args.display_index : undefined, + }) + return { text: JSON.stringify(result, null, 2) } + }, + } + + return { + name: 'vision', + version: '0.1.0', + tools: VISION_TOOLS, + handlers, + } +} + +export { captureScreenshot } from './screenshot' +export { VISION_TOOLS } from './tool-defs' +export type { ScreenshotOptions, ScreenshotResult } from './screenshot' diff --git a/src/plugins/vision/screenshot.ts b/src/plugins/vision/screenshot.ts new file mode 100644 index 0000000..88fb193 --- /dev/null +++ b/src/plugins/vision/screenshot.ts @@ -0,0 +1,148 @@ +/** + * Full-screen screenshot capture for the Vision Pack. + * + * Shells out to OS-native CLI utilities — no native deps, no npm + * packages. Each platform writes to a temp file, the temp is read, and + * the file is removed. + * + * macOS: `screencapture -x ` (always installed) + * Linux: `gnome-screenshot -f ` or `scrot ` + * Windows: PowerShell + .NET System.Drawing (built into .NET 8) + * + * Per-window captures aren't in v0 — they need bridge work (UIA / AXAPI + * window-handle → PNG buffer). Full-screen is enough to unblock the + * "AI vision fallback when accessibility is broken" use case: the + * multimodal model can crop / target on its own from the full image. + */ +import { spawn } from 'node:child_process' +import { readFile, unlink, mkdir } from 'node:fs/promises' +import * as os from 'node:os' +import * as path from 'node:path' + +export interface ScreenshotOptions { + /** + * When set, the captured PNG is written here instead of being + * returned in-band. Returns just { path, bytes }. Useful for large + * captures where the agent doesn't need the raw image data and a + * follow-up vision call can re-read from disk. + */ + outputPath?: string + /** Which display index to capture (default: primary). macOS-only — other + * platforms ignore this and capture the active screen. */ + displayIndex?: number +} + +export interface ScreenshotResult { + /** Where the PNG was written, if `outputPath` was supplied. */ + path?: string + /** Base64-encoded PNG payload. Omitted when `outputPath` is set. */ + image_base64?: string + /** Size in bytes. */ + bytes: number + /** The CLI utility that produced the capture. */ + captured_by: 'screencapture' | 'gnome-screenshot' | 'scrot' | 'powershell' + platform: NodeJS.Platform +} + +export async function captureScreenshot(opts: ScreenshotOptions = {}): Promise { + const tmpDir = path.join(os.tmpdir(), 'agentmark-screenshots') + await mkdir(tmpDir, { recursive: true }) + const tmpPath = opts.outputPath ?? path.join(tmpDir, `screen-${process.pid}-${Date.now()}.png`) + + let capturedBy: ScreenshotResult['captured_by'] + try { + if (process.platform === 'darwin') { + capturedBy = 'screencapture' + const args = ['-x'] + if (typeof opts.displayIndex === 'number') { + args.push('-D', String(opts.displayIndex + 1)) // screencapture is 1-indexed + } + args.push(tmpPath) + await runCommand('screencapture', args) + } else if (process.platform === 'win32') { + capturedBy = 'powershell' + await runCommand('powershell', [ + '-NoProfile', + '-Command', + buildWindowsScreenshotScript(tmpPath), + ]) + } else { + const linuxResult = await captureLinux(tmpPath) + capturedBy = linuxResult + } + } catch (err) { + // Clean up any partial output before bubbling up. + await unlink(tmpPath).catch(() => {}) + throw err + } + + const bytes = await readFile(tmpPath) + const sizeBytes = bytes.length + + if (opts.outputPath) { + return { + path: tmpPath, + bytes: sizeBytes, + captured_by: capturedBy, + platform: process.platform, + } + } + + // In-band return — delete the temp file once read. + await unlink(tmpPath).catch(() => {}) + return { + image_base64: bytes.toString('base64'), + bytes: sizeBytes, + captured_by: capturedBy, + platform: process.platform, + } +} + +async function captureLinux(targetPath: string): Promise<'gnome-screenshot' | 'scrot'> { + try { + await runCommand('gnome-screenshot', ['-f', targetPath]) + return 'gnome-screenshot' + } catch { + // fall through + } + try { + await runCommand('scrot', [targetPath]) + return 'scrot' + } catch { + throw new Error( + 'Linux screenshot requires gnome-screenshot or scrot. ' + + 'Install one: `apt install gnome-screenshot` or `apt install scrot`.', + ) + } +} + +function runCommand(command: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] }) + let stderr = '' + child.stderr.on('data', (b: Buffer) => { stderr += b.toString('utf8') }) + child.on('error', reject) + child.on('close', (code) => { + if (code === 0) resolve() + else reject(new Error(`${command} exited ${code}: ${stderr.trim()}`)) + }) + }) +} + +function buildWindowsScreenshotScript(targetPath: string): string { + // Native .NET 8 PowerShell snippet — no external module install needed. + // System.Windows.Forms.Screen.PrimaryScreen.Bounds gives the full + // primary monitor; CopyFromScreen draws it onto a Bitmap. + const escaped = targetPath.replace(/'/g, "''") + return [ + "Add-Type -AssemblyName System.Windows.Forms", + "Add-Type -AssemblyName System.Drawing", + "$bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds", + "$bmp = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height", + "$gfx = [System.Drawing.Graphics]::FromImage($bmp)", + "$gfx.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size)", + `$bmp.Save('${escaped}', [System.Drawing.Imaging.ImageFormat]::Png)`, + "$gfx.Dispose()", + "$bmp.Dispose()", + ].join('; ') +} diff --git a/src/plugins/vision/tool-defs.ts b/src/plugins/vision/tool-defs.ts new file mode 100644 index 0000000..6262b74 --- /dev/null +++ b/src/plugins/vision/tool-defs.ts @@ -0,0 +1,42 @@ +/** + * Vision Pack — tool definitions. + * + * v0 ships full-screen capture only. Per-window screenshots come in a + * follow-up that adds bridge-level support (UIA / AXAPI window handle + * → PNG buffer). + */ +import type { McpToolDef } from '../../mcp/tool-defs' + +export const VISION_TOOLS: McpToolDef[] = [ + { + name: 'agentmark_screenshot', + description: + 'Capture a full-screen PNG screenshot. Returns the image as ' + + 'base64 in the response by default, or writes it to `output_path` ' + + 'and returns just the path + byte count. Use when the ' + + 'accessibility tree is too sparse to drive an app reliably ' + + '(custom-drawn UIs, Electron apps with weak a11y, games) — the ' + + 'multimodal AI can vision-parse the image directly.\n' + + '\nUtilities used per platform:\n' + + ' - macOS: screencapture (always installed)\n' + + ' - Windows: PowerShell + .NET System.Drawing\n' + + ' - Linux: gnome-screenshot OR scrot (install one)', + inputSchema: { + type: 'object', + properties: { + output_path: { + type: 'string', + description: + 'If set, write the PNG to this path and return just ' + + '{ path, bytes }. Otherwise return base64 inline.', + }, + display_index: { + type: 'number', + description: + 'Zero-based display index (macOS only — other platforms ' + + 'capture the active screen). Default: primary.', + }, + }, + }, + }, +] diff --git a/test/vision/vision-plugin.test.ts b/test/vision/vision-plugin.test.ts new file mode 100644 index 0000000..c9fe674 --- /dev/null +++ b/test/vision/vision-plugin.test.ts @@ -0,0 +1,33 @@ +/** + * Tests for the Vision Pack. + * + * The screenshot itself shells out to OS-native tools that need a real + * display session, so we don't run the real capture in unit tests. We + * verify plugin registration, tool surface, and that the handler is + * wired to the screenshot function (which is tested via the live demo + * paths on real Mac + Win VMs). + */ +import { describe, it, expect } from 'vitest' +import { + createVisionPlugin, + VISION_TOOLS, +} from '../../src/plugins/vision' +import { Dispatcher } from '../../src/mcp/plugin' + +describe('Vision plugin — registration', () => { + it('registers every tool with a matching handler', () => { + const plugin = createVisionPlugin() + const dispatcher = new Dispatcher([plugin]) + expect(dispatcher.toolNames.sort()).toEqual(VISION_TOOLS.map((t) => t.name).sort()) + }) + + it('exposes the v0 tool set', () => { + expect(VISION_TOOLS.map((t) => t.name)).toEqual(['agentmark_screenshot']) + }) + + it('plugin.name + version are set', () => { + const plugin = createVisionPlugin() + expect(plugin.name).toBe('vision') + expect(plugin.version).toBe('0.1.0') + }) +})