diff --git a/registry/curated/system/browser-automation/package.json b/registry/curated/system/browser-automation/package.json index 5573e3d..5d3faa5 100644 --- a/registry/curated/system/browser-automation/package.json +++ b/registry/curated/system/browser-automation/package.json @@ -49,5 +49,6 @@ "homepage": "https://github.com/framerslab/agentos-extensions/tree/master/registry/curated/system/browser-automation#readme", "bugs": { "url": "https://github.com/framerslab/agentos-extensions/issues" - } + }, + "type": "module" } diff --git a/registry/curated/system/browser-automation/src/attach/AttachController.ts b/registry/curated/system/browser-automation/src/attach/AttachController.ts index 8b9a677..8970763 100644 --- a/registry/curated/system/browser-automation/src/attach/AttachController.ts +++ b/registry/curated/system/browser-automation/src/attach/AttachController.ts @@ -51,6 +51,13 @@ export interface AttachBackend { * `UNSUPPORTED_OP` when missing. Never exposed on the agent tool surface. */ evalInTab?(tab: string, expression: string, timeoutMs?: number): Promise; + /** + * OPTIONAL: capture a PNG screenshot of the agent tab, returned as base64. + * CDP-only (`Page.captureScreenshot`); the JXA transport has no pixel access, + * so callers surface `UNSUPPORTED_OP` when this is absent. Read-only — capture + * never navigates, focuses, or mutates the page. + */ + screenshotTab?(tab: string, fullPage?: boolean): Promise; } /** @@ -67,6 +74,8 @@ export interface AttachSurface { pause(): void; resume(): void; setDryRun(on: boolean): void; + /** OPTIONAL (CDP transport only): write a PNG of the agent tab to disk. */ + screenshot?(filePath: string, fullPage?: boolean): Promise<{ path: string; bytes: number }>; } /** @@ -266,6 +275,40 @@ export class AttachController { } } + /** + * Capture a PNG screenshot of the agent tab and write it to `filePath`. + * + * CDP-only: the JXA transport has no pixel access, so a jxa-backed session + * refuses with `UNSUPPORTED_OP` rather than pretending. Read-only — capture + * never navigates, focuses, or mutates the page. + * + * @returns The written path plus byte size, for evidence logging. + */ + async screenshot(filePath: string, fullPage?: boolean): Promise<{ path: string; bytes: number }> { + this.requireReady(); + requireLease(this.opts.leaseFile, this.lease!.nonce); + const fn = this.backend.screenshotTab?.bind(this.backend); + if (!fn) { + throw new AttachError('UNSUPPORTED_OP', `${this.backend.kind} backend has no screenshot capability (CDP transport required)`); + } + if (this.dryRun) return { path: filePath, bytes: 0 }; + this.machine.to('reading'); + try { + const b64 = await withDeadline(fn(this.tab!, fullPage), this.deadline, 'screenshot'); + const { writeFileSync, mkdirSync } = await import('node:fs'); + const { dirname } = await import('node:path'); + mkdirSync(dirname(filePath), { recursive: true }); + const buf = Buffer.from(b64, 'base64'); + writeFileSync(filePath, buf); + this.machine.to('ready'); + return { path: filePath, bytes: buf.length }; + } catch (err) { + if (this.machine.state === 'reading') this.machine.to('failed'); + this.lastError = toStructuredError(err); + throw err; + } + } + /** Fixed, parameterized extraction expression: the selector map is data, never code. */ private static extractExpression(fields?: Record): string { const spec = JSON.stringify(fields ?? {}); diff --git a/registry/curated/system/browser-automation/src/attach/__tests__/pack-registration.test.ts b/registry/curated/system/browser-automation/src/attach/__tests__/pack-registration.test.ts index 7a4479f..333b714 100644 --- a/registry/curated/system/browser-automation/src/attach/__tests__/pack-registration.test.ts +++ b/registry/curated/system/browser-automation/src/attach/__tests__/pack-registration.test.ts @@ -18,6 +18,7 @@ describe('browser-automation attach registration', () => { 'browser_attach_goto', 'browser_attach_read', 'browser_attach_release', + 'browser_attach_screenshot', 'browser_attach_status', ]); for (const d of attach) expect(d.enableByDefault).toBe(false); diff --git a/registry/curated/system/browser-automation/src/attach/__tests__/tools.test.ts b/registry/curated/system/browser-automation/src/attach/__tests__/tools.test.ts index 9fbb178..7532c72 100644 --- a/registry/curated/system/browser-automation/src/attach/__tests__/tools.test.ts +++ b/registry/curated/system/browser-automation/src/attach/__tests__/tools.test.ts @@ -41,17 +41,18 @@ beforeEach(() => { }); describe('attach tools', () => { - it('exposes exactly the six ids and no eval/js tool', () => { + it('exposes exactly the seven ids and no eval/js tool', () => { expect(ATTACH_TOOL_IDS).toEqual([ 'browser_attach_status', 'browser_attach_claim', 'browser_attach_goto', 'browser_attach_read', + 'browser_attach_screenshot', 'browser_attach_release', 'browser_attach_control', ]); expect(ATTACH_TOOL_IDS).not.toContain('browser_attach_eval'); - expect(createAttachTools(controller)).toHaveLength(6); + expect(createAttachTools(controller)).toHaveLength(7); }); it('control tool pauses, resumes, and toggles dry-run at runtime', async () => { @@ -90,7 +91,7 @@ describe('attach tools', () => { }); it('status never throws even before a claim', async () => { - const res = await new AttachStatusTool(controller).execute(); + const res = (await new AttachStatusTool(controller).execute()) as { success: boolean; data: any }; expect(res.success).toBe(true); expect(res.data.leaseHeld).toBe(false); }); @@ -108,7 +109,12 @@ describe('tools over the daemon surface', () => { try { const surface = new DaemonAttachSurface({ ipcDir: join(dir, 'ipc') }); const tools = createAttachTools(surface); - const byId = Object.fromEntries(tools.map((t) => [t.id, t])); + // Tool args differ per tool; index them loosely so the union of input + // schemas doesn't collapse into an impossible intersection. + const byId = Object.fromEntries(tools.map((t) => [t.id, t])) as Record< + string, + { id: string; hasSideEffects: boolean; execute: (args?: any) => Promise } + >; expect(Object.keys(byId).sort()).toEqual([...ATTACH_TOOL_IDS].sort()); // ids unchanged expect(byId['browser_attach_read'].hasSideEffects).toBe(false); expect(byId['browser_attach_status'].hasSideEffects).toBe(false); @@ -120,7 +126,7 @@ describe('tools over the daemon surface', () => { const read = await byId['browser_attach_read'].execute({}); expect(read.success).toBe(true); expect(read.data.untrusted).toBe(true); - expect((await byId['browser_attach_release'].execute()).success).toBe(true); + expect((await byId['browser_attach_release'].execute({})).success).toBe(true); await new AttachDaemonClient({ ipcDir: join(dir, 'ipc'), client: 'test-quit', pollMs: 25 }).quit(); await run; diff --git a/registry/curated/system/browser-automation/src/attach/backends/jxa.ts b/registry/curated/system/browser-automation/src/attach/backends/jxa.ts index 29f57d4..9605a99 100644 --- a/registry/curated/system/browser-automation/src/attach/backends/jxa.ts +++ b/registry/curated/system/browser-automation/src/attach/backends/jxa.ts @@ -79,6 +79,16 @@ function run(argv) { delay(1); return t.url(); } + if (cmd === 'raise') { + // Bring the agent window forward AND return its bounds, so a display + // capture can be cropped to this window only — never the whole screen + // (which would leak the user's unrelated windows into evidence shots). + w.index = 1; + try { w.activeTabIndex = w.tabs().findIndex((x) => String(x.id()) === String(tid)) + 1; } catch (e) {} + delay(0.6); + const b = w.bounds(); + return JSON.stringify({ x: b.x, y: b.y, width: b.width, height: b.height }); + } if (cmd === 'url') return t.url() + '\\n' + t.title(); if (cmd === 'read') { const sel = argv[4] || 'body'; @@ -149,6 +159,11 @@ export class JxaBackend implements AttachBackend { } async claimAgentTab(): Promise { + // IDEMPOTENT: probeIdentity() claims a tab before the controller's own + // claimAgentTab() call, so a second `claim` would look for another + // about:blank tab that no longer exists and fail NO_BLANK_TAB. Reuse the + // tab this backend already holds instead of consuming a second one. + if (this.claimed) return `${this.claimed.wid} ${this.claimed.tid}`; const out = await this.exec('claim', []); const [wid, tid] = out.split(/\s+/); if (!wid || !tid) throw new AttachError('TAB_CLOSED', `claim returned malformed ids: "${out}"`); @@ -173,4 +188,63 @@ export class JxaBackend implements AttachBackend { const [wid, tid] = tab.split(/\s+/); return this.exec('read', [wid, tid, selector ?? 'body', String(maxChars ?? 6000)]); } + + /** + * Screenshot via macOS `screencapture` (base64 PNG). + * + * The JXA transport has no CDP pixel pipe, so this raises the agent tab's + * window to the front and captures the display. Two consequences the caller + * must accept: it FOCUSES the agent window (the one exception to the + * no-focus rule — a screenshot is meaningless if the window is behind + * others), and the capture is the whole screen, so `fullPage` is ignored. + * Requires the host to hold macOS Screen Recording permission; without it + * `screencapture` writes nothing and this surfaces a structured failure. + */ + async screenshotTab(tab: string, _fullPage?: boolean): Promise { + // OFF BY DEFAULT. `screencapture -R` works in display POINTS while Chrome's + // window bounds and Retina backing scale disagree, so the cropped region + // drifted onto NEIGHBOURING windows in live testing (2026-07-24) — which + // would silently put the user's unrelated, private windows into a research + // evidence file. Correct page-pixel capture is CDP `Page.captureScreenshot` + // (see RawCdpBackend); this lane stays refused unless explicitly opted in + // via WUNDERLAND_ATTACH_JXA_SHOTS=1 for a single-display debug session. + if (process.env.WUNDERLAND_ATTACH_JXA_SHOTS !== '1') { + throw new AttachError( + 'UNSUPPORTED_OP', + 'screenshots require the CDP transport (WUNDERLAND_ATTACH_TRANSPORT=cdp); the JXA display-capture lane is disabled because its crop can include other windows', + ); + } + const [wid, tid] = tab.split(/\s+/); + const raised = await this.exec('raise', [wid, tid]); + const { mkdtempSync, readFileSync, existsSync, rmSync } = await import('node:fs'); + const { tmpdir } = await import('node:os'); + const file = join(mkdtempSync(join(tmpdir(), 'attach-shot-')), 'shot.png'); + // Crop to the agent WINDOW's bounds. A full-display capture would include + // every other window the user has open — private content must never land + // in a research evidence shot. + let region: string | undefined; + try { + const b = JSON.parse(raised) as { x: number; y: number; width: number; height: number }; + if ([b.x, b.y, b.width, b.height].every((n) => typeof n === 'number' && Number.isFinite(n)) && b.width > 0 && b.height > 0) { + region = `${Math.max(0, Math.round(b.x))},${Math.max(0, Math.round(b.y))},${Math.round(b.width)},${Math.round(b.height)}`; + } + } catch { + /* older driver without bounds → refuse rather than capture the screen */ + } + if (!region) { + throw new AttachError('UNSUPPORTED_OP', 'could not resolve agent-window bounds; refusing a full-display capture'); + } + try { + await pExecFile('screencapture', ['-x', '-o', '-t', 'png', '-R', region, file], { timeout: 20_000 }); + if (!existsSync(file)) { + throw new AttachError('UNSUPPORTED_OP', 'screencapture produced no file (Screen Recording permission?)'); + } + return readFileSync(file).toString('base64'); + } catch (err) { + if (err instanceof AttachError) throw err; + throw new AttachError('UNSUPPORTED_OP', `screencapture failed: ${err instanceof Error ? err.message : String(err)}`); + } finally { + rmSync(file, { force: true }); + } + } } diff --git a/registry/curated/system/browser-automation/src/attach/backends/raw-cdp.ts b/registry/curated/system/browser-automation/src/attach/backends/raw-cdp.ts index 5f7aee1..28dfcaa 100644 --- a/registry/curated/system/browser-automation/src/attach/backends/raw-cdp.ts +++ b/registry/curated/system/browser-automation/src/attach/backends/raw-cdp.ts @@ -206,6 +206,26 @@ export class RawCdpBackend implements AttachBackend { return value; } + /** + * `Page.captureScreenshot` on the agent tab → base64 PNG (optional backend + * capability). Read-only: no navigation, no focus change, no page mutation. + * `fullPage` uses `captureBeyondViewport` so a long results page comes back + * whole instead of clipped to the viewport. + */ + async screenshotTab(tab: string, fullPage?: boolean): Promise { + this.requireTab(tab); + const res = (await this.send( + 'Page.captureScreenshot', + { format: 'png', captureBeyondViewport: !!fullPage }, + this.sessionId!, + 45_000, + )) as { data?: string }; + if (!res.data) { + throw new AttachError('UNKNOWN', 'Page.captureScreenshot returned no data'); + } + return res.data; + } + private requireTab(tab: string): void { if (!this.sessionId || tab !== this.createdTargetId) { throw new AttachError('TAB_CLOSED', 'agent tab is not claimed on this transport'); diff --git a/registry/curated/system/browser-automation/src/attach/daemon/client.ts b/registry/curated/system/browser-automation/src/attach/daemon/client.ts index 3a44844..dc971e6 100644 --- a/registry/curated/system/browser-automation/src/attach/daemon/client.ts +++ b/registry/curated/system/browser-automation/src/attach/daemon/client.ts @@ -112,6 +112,14 @@ export class AttachDaemonClient { } /** Give up driving rights (daemon parks the tab; its session persists). */ + /** Capture a PNG of the agent tab to `path` (CDP transport only). */ + async screenshot(path: string, fullPage?: boolean): Promise<{ path: string; bytes: number }> { + return (await this.request('screenshot', { path, fullPage: !!fullPage }, 60_000)) as { + path: string; + bytes: number; + }; + } + async release(): Promise { await this.request('release'); } diff --git a/registry/curated/system/browser-automation/src/attach/daemon/daemon.ts b/registry/curated/system/browser-automation/src/attach/daemon/daemon.ts index 48abdf2..62c2ec4 100644 --- a/registry/curated/system/browser-automation/src/attach/daemon/daemon.ts +++ b/registry/curated/system/browser-automation/src/attach/daemon/daemon.ts @@ -253,6 +253,14 @@ export class AttachDaemon { const data = await this.controller.extract(a.fields as Record | undefined); return { ...base, ok: true, data: { untrusted: true, data } }; } + case 'screenshot': { + this.requireClaimant(cmd); + if (typeof a.path !== 'string' || !a.path) { + return { ...base, ok: false, error: { code: 'UNKNOWN', message: 'screenshot requires a path' } }; + } + const shot = await this.controller.screenshot(a.path, a.fullPage === true); + return { ...base, ok: true, data: shot }; + } case 'eval': { this.requireClaimant(cmd); const value = await this.controller.evaluate( diff --git a/registry/curated/system/browser-automation/src/attach/daemon/protocol.ts b/registry/curated/system/browser-automation/src/attach/daemon/protocol.ts index 33673c3..392fafa 100644 --- a/registry/curated/system/browser-automation/src/attach/daemon/protocol.ts +++ b/registry/curated/system/browser-automation/src/attach/daemon/protocol.ts @@ -27,6 +27,7 @@ export type AttachOp = | 'read' | 'extract' | 'eval' + | 'screenshot' | 'release' | 'control' | 'quit'; diff --git a/registry/curated/system/browser-automation/src/attach/daemon/surface.ts b/registry/curated/system/browser-automation/src/attach/daemon/surface.ts index f7fef66..af0e02b 100644 --- a/registry/curated/system/browser-automation/src/attach/daemon/surface.ts +++ b/registry/curated/system/browser-automation/src/attach/daemon/surface.ts @@ -62,6 +62,11 @@ export class DaemonAttachSurface implements AttachSurface { return (await this.client.read({ selector, maxChars })).text; } + /** Capture a PNG of the agent tab to `path` (CDP transport behind the daemon). */ + async screenshot(path: string, fullPage?: boolean): Promise<{ path: string; bytes: number }> { + return this.client.screenshot(path, fullPage); + } + /** Give up driving rights; the daemon keeps its held session. */ async detach(): Promise { await this.client.release(); diff --git a/registry/curated/system/browser-automation/src/attach/tools.ts b/registry/curated/system/browser-automation/src/attach/tools.ts index f4c3d70..2aa440c 100644 --- a/registry/curated/system/browser-automation/src/attach/tools.ts +++ b/registry/curated/system/browser-automation/src/attach/tools.ts @@ -18,7 +18,7 @@ * @module browser-automation/attach/tools */ import type { AttachSurface } from './AttachController.js'; -import { toStructuredError } from './errors.js'; +import { AttachError, toStructuredError } from './errors.js'; /** Wrap an operation into the standard tool result envelope. */ async function envelope(fn) { @@ -110,6 +110,36 @@ export class AttachReadTool { } } +/** `browser_attach_screenshot` — PNG evidence capture of the agent tab (CDP only). */ +export class AttachScreenshotTool { + readonly id = 'browser_attach_screenshot'; + readonly name = 'browser_attach_screenshot'; + readonly displayName = 'Attach: screenshot'; + readonly description = + 'Capture a PNG screenshot of the agent tab to a local file path — visual evidence for a research report. Requires the CDP transport (the AppleScript/JXA transport has no pixel access and refuses with UNSUPPORTED_OP). Read-only: never navigates or mutates the page.'; + readonly category = 'browser'; + readonly version = '0.1.0'; + readonly hasSideEffects = false; + readonly inputSchema = { + type: 'object' as const, + properties: { + path: { type: 'string', description: 'Absolute file path to write the PNG to' }, + fullPage: { type: 'boolean', description: 'Capture the full scrollable page instead of just the viewport' }, + }, + required: ['path'], + }; + constructor(private controller: AttachSurface) {} + async execute(args: { path: string; fullPage?: boolean }) { + return envelope(async () => { + const fn = this.controller.screenshot?.bind(this.controller); + if (!fn) { + throw new AttachError('UNSUPPORTED_OP', 'this attach transport has no screenshot capability (CDP required)'); + } + return fn(args.path, args.fullPage); + }); + } +} + /** `browser_attach_release` — park + release the lease (no browser teardown). */ export class AttachReleaseTool { readonly id = 'browser_attach_release'; @@ -188,6 +218,7 @@ export function createAttachTools(controller: AttachSurface) { new AttachClaimTool(controller), new AttachGotoTool(controller), new AttachReadTool(controller), + new AttachScreenshotTool(controller), new AttachReleaseTool(controller), new AttachControlTool(controller), ]; @@ -199,6 +230,7 @@ export const ATTACH_TOOL_IDS = [ 'browser_attach_claim', 'browser_attach_goto', 'browser_attach_read', + 'browser_attach_screenshot', 'browser_attach_release', 'browser_attach_control', ] as const; diff --git a/registry/curated/system/browser-automation/tsconfig.json b/registry/curated/system/browser-automation/tsconfig.json index 9df0ad0..6500e2a 100644 --- a/registry/curated/system/browser-automation/tsconfig.json +++ b/registry/curated/system/browser-automation/tsconfig.json @@ -1,16 +1,26 @@ { "compilerOptions": { - "target": "ES2020", - "module": "commonjs", - "lib": ["ES2020"], + "target": "ES2022", + "module": "Node16", + "lib": [ + "ES2022" + ], "outDir": "./dist", "rootDir": "./src", "strict": true, "esModuleInterop": true, "declaration": true, "declarationMap": true, - "sourceMap": true + "sourceMap": true, + "moduleResolution": "Node16" }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist", "test", "**/*.spec.ts"] + "include": [ + "src/**/*" + ], + "exclude": [ + "node_modules", + "dist", + "test", + "**/*.spec.ts" + ] }