feat(plugins): Vision Pack v0 — full-screen screenshot for AI fallback

src/mcp/index.ts

@@ -63,6 +63,18 @@ export type {
     QueuedMessage,
 } from '../plugins/network'
 
+// Vision Pack — Layer 2 fallback (screenshot tool). Opt-in.
+export {
+    createVisionPlugin,
+    captureScreenshot,
+    VISION_TOOLS,
+} from '../plugins/vision'
+export type {
+    VisionPluginConfig,
+    ScreenshotOptions,
+    ScreenshotResult,
+} from '../plugins/vision'
+
 // Microsoft Workflows Pack (Graph-only v0) — opt-in; not part of the
 // default plugin set. Pass it explicitly via `createMcpServer({ plugins })`.
 export {

src/plugins/vision/index.ts

@@ -0,0 +1,41 @@
+/**
+ * Vision Pack — Layer 2 fallback for accessibility-tree gaps.
+ *
+ * Right now ships one tool: full-screen screenshot. The multimodal AI
+ * does its own parsing on the returned image. As we expand:
+ *
+ *   - per-window capture via bridges
+ *   - OmniParser-style structured GUI parsing inside the plugin
+ *   - "tree was sparse, auto-fallback to vision" hook on desktop_snapshot
+ */
+import { captureScreenshot } from './screenshot'
+import { VISION_TOOLS } from './tool-defs'
+import type { AgentMarkPlugin, DispatchResult, ToolHandler } from '../../mcp/plugin'
+
+export interface VisionPluginConfig {
+    /** Reserved for future config (preferred OCR backend, model hints, etc.). */
+    _reserved?: never
+}
+
+export function createVisionPlugin(_config: VisionPluginConfig = {}): AgentMarkPlugin {
+    const handlers: Record<string, ToolHandler> = {
+        agentmark_screenshot: async (args): Promise<DispatchResult> => {
+            const result = await captureScreenshot({
+                outputPath: typeof args.output_path === 'string' ? args.output_path : undefined,
+                displayIndex: typeof args.display_index === 'number' ? args.display_index : undefined,
+            })
+            return { text: JSON.stringify(result, null, 2) }
+        },
+    }
+
+    return {
+        name: 'vision',
+        version: '0.1.0',
+        tools: VISION_TOOLS,
+        handlers,
+    }
+}
+
+export { captureScreenshot } from './screenshot'
+export { VISION_TOOLS } from './tool-defs'
+export type { ScreenshotOptions, ScreenshotResult } from './screenshot'

src/plugins/vision/screenshot.ts

@@ -0,0 +1,148 @@
+/**
+ * Full-screen screenshot capture for the Vision Pack.
+ *
+ * Shells out to OS-native CLI utilities — no native deps, no npm
+ * packages. Each platform writes to a temp file, the temp is read, and
+ * the file is removed.
+ *
+ *   macOS:   `screencapture -x <path>`                  (always installed)
+ *   Linux:   `gnome-screenshot -f <path>` or `scrot <path>`
+ *   Windows: PowerShell + .NET System.Drawing (built into .NET 8)
+ *
+ * Per-window captures aren't in v0 — they need bridge work (UIA / AXAPI
+ * window-handle → PNG buffer). Full-screen is enough to unblock the
+ * "AI vision fallback when accessibility is broken" use case: the
+ * multimodal model can crop / target on its own from the full image.
+ */
+import { spawn } from 'node:child_process'
+import { readFile, unlink, mkdir } from 'node:fs/promises'
+import * as os from 'node:os'
+import * as path from 'node:path'
+
+export interface ScreenshotOptions {
+    /**
+     * When set, the captured PNG is written here instead of being
+     * returned in-band. Returns just { path, bytes }. Useful for large
+     * captures where the agent doesn't need the raw image data and a
+     * follow-up vision call can re-read from disk.
+     */
+    outputPath?: string
+    /** Which display index to capture (default: primary). macOS-only — other
+     *  platforms ignore this and capture the active screen. */
+    displayIndex?: number
+}
+
+export interface ScreenshotResult {
+    /** Where the PNG was written, if `outputPath` was supplied. */
+    path?: string
+    /** Base64-encoded PNG payload. Omitted when `outputPath` is set. */
+    image_base64?: string
+    /** Size in bytes. */
+    bytes: number
+    /** The CLI utility that produced the capture. */
+    captured_by: 'screencapture' | 'gnome-screenshot' | 'scrot' | 'powershell'
+    platform: NodeJS.Platform
+}
+
+export async function captureScreenshot(opts: ScreenshotOptions = {}): Promise<ScreenshotResult> {
+    const tmpDir = path.join(os.tmpdir(), 'agentmark-screenshots')
+    await mkdir(tmpDir, { recursive: true })
+    const tmpPath = opts.outputPath ?? path.join(tmpDir, `screen-${process.pid}-${Date.now()}.png`)
+
+    let capturedBy: ScreenshotResult['captured_by']
+    try {
+        if (process.platform === 'darwin') {
+            capturedBy = 'screencapture'
+            const args = ['-x']
+            if (typeof opts.displayIndex === 'number') {
+                args.push('-D', String(opts.displayIndex + 1)) // screencapture is 1-indexed
+            }
+            args.push(tmpPath)
+            await runCommand('screencapture', args)
+        } else if (process.platform === 'win32') {
+            capturedBy = 'powershell'
+            await runCommand('powershell', [
+                '-NoProfile',
+                '-Command',
+                buildWindowsScreenshotScript(tmpPath),
+            ])
+        } else {
+            const linuxResult = await captureLinux(tmpPath)
+            capturedBy = linuxResult
+        }
+    } catch (err) {
+        // Clean up any partial output before bubbling up.
+        await unlink(tmpPath).catch(() => {})
+        throw err
+    }
+
+    const bytes = await readFile(tmpPath)
+    const sizeBytes = bytes.length
+
+    if (opts.outputPath) {
+        return {
+            path: tmpPath,
+            bytes: sizeBytes,
+            captured_by: capturedBy,
+            platform: process.platform,
+        }
+    }
+
+    // In-band return — delete the temp file once read.
+    await unlink(tmpPath).catch(() => {})
+    return {
+        image_base64: bytes.toString('base64'),
+        bytes: sizeBytes,
+        captured_by: capturedBy,
+        platform: process.platform,
+    }
+}
+
+async function captureLinux(targetPath: string): Promise<'gnome-screenshot' | 'scrot'> {
+    try {
+        await runCommand('gnome-screenshot', ['-f', targetPath])
+        return 'gnome-screenshot'
+    } catch {
+        // fall through
+    }
+    try {
+        await runCommand('scrot', [targetPath])
+        return 'scrot'
+    } catch {
+        throw new Error(
+            'Linux screenshot requires gnome-screenshot or scrot. '
+            + 'Install one: `apt install gnome-screenshot` or `apt install scrot`.',
+        )
+    }
+}
+
+function runCommand(command: string, args: string[]): Promise<void> {
+    return new Promise((resolve, reject) => {
+        const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] })
+        let stderr = ''
+        child.stderr.on('data', (b: Buffer) => { stderr += b.toString('utf8') })
+        child.on('error', reject)
+        child.on('close', (code) => {
+            if (code === 0) resolve()
+            else reject(new Error(`${command} exited ${code}: ${stderr.trim()}`))
+        })
+    })
+}
+
+function buildWindowsScreenshotScript(targetPath: string): string {
+    // Native .NET 8 PowerShell snippet — no external module install needed.
+    // System.Windows.Forms.Screen.PrimaryScreen.Bounds gives the full
+    // primary monitor; CopyFromScreen draws it onto a Bitmap.
+    const escaped = targetPath.replace(/'/g, "''")
+    return [
+        "Add-Type -AssemblyName System.Windows.Forms",
+        "Add-Type -AssemblyName System.Drawing",
+        "$bounds = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds",
+        "$bmp = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height",
+        "$gfx = [System.Drawing.Graphics]::FromImage($bmp)",
+        "$gfx.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size)",
+        `$bmp.Save('${escaped}', [System.Drawing.Imaging.ImageFormat]::Png)`,
+        "$gfx.Dispose()",
+        "$bmp.Dispose()",
+    ].join('; ')
+}

src/plugins/vision/tool-defs.ts

@@ -0,0 +1,42 @@
+/**
+ * Vision Pack — tool definitions.
+ *
+ * v0 ships full-screen capture only. Per-window screenshots come in a
+ * follow-up that adds bridge-level support (UIA / AXAPI window handle
+ * → PNG buffer).
+ */
+import type { McpToolDef } from '../../mcp/tool-defs'
+
+export const VISION_TOOLS: McpToolDef[] = [
+    {
+        name: 'agentmark_screenshot',
+        description:
+            'Capture a full-screen PNG screenshot. Returns the image as '
+            + 'base64 in the response by default, or writes it to `output_path` '
+            + 'and returns just the path + byte count. Use when the '
+            + 'accessibility tree is too sparse to drive an app reliably '
+            + '(custom-drawn UIs, Electron apps with weak a11y, games) — the '
+            + 'multimodal AI can vision-parse the image directly.\n'
+            + '\nUtilities used per platform:\n'
+            + '  - macOS:   screencapture (always installed)\n'
+            + '  - Windows: PowerShell + .NET System.Drawing\n'
+            + '  - Linux:   gnome-screenshot OR scrot (install one)',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                output_path: {
+                    type: 'string',
+                    description:
+                        'If set, write the PNG to this path and return just '
+                        + '{ path, bytes }. Otherwise return base64 inline.',
+                },
+                display_index: {
+                    type: 'number',
+                    description:
+                        'Zero-based display index (macOS only — other platforms '
+                        + 'capture the active screen). Default: primary.',
+                },
+            },
+        },
+    },
+]

test/vision/vision-plugin.test.ts

@@ -0,0 +1,33 @@
+/**
+ * Tests for the Vision Pack.
+ *
+ * The screenshot itself shells out to OS-native tools that need a real
+ * display session, so we don't run the real capture in unit tests. We
+ * verify plugin registration, tool surface, and that the handler is
+ * wired to the screenshot function (which is tested via the live demo
+ * paths on real Mac + Win VMs).
+ */
+import { describe, it, expect } from 'vitest'
+import {
+    createVisionPlugin,
+    VISION_TOOLS,
+} from '../../src/plugins/vision'
+import { Dispatcher } from '../../src/mcp/plugin'
+
+describe('Vision plugin — registration', () => {
+    it('registers every tool with a matching handler', () => {
+        const plugin = createVisionPlugin()
+        const dispatcher = new Dispatcher([plugin])
+        expect(dispatcher.toolNames.sort()).toEqual(VISION_TOOLS.map((t) => t.name).sort())
+    })
+
+    it('exposes the v0 tool set', () => {
+        expect(VISION_TOOLS.map((t) => t.name)).toEqual(['agentmark_screenshot'])
+    })
+
+    it('plugin.name + version are set', () => {
+        const plugin = createVisionPlugin()
+        expect(plugin.name).toBe('vision')
+        expect(plugin.version).toBe('0.1.0')
+    })
+})