-
Notifications
You must be signed in to change notification settings - Fork 0
Attach screenshots (CDP) + ESM build fix + idempotent claim #37
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -79,6 +79,16 @@ function run(argv) { | |
| delay(1); | ||
| return t.url(); | ||
| } | ||
| if (cmd === 'raise') { | ||
| // Bring the agent window forward AND return its bounds, so a display | ||
| // capture can be cropped to this window only — never the whole screen | ||
| // (which would leak the user's unrelated windows into evidence shots). | ||
| w.index = 1; | ||
| try { w.activeTabIndex = w.tabs().findIndex((x) => String(x.id()) === String(tid)) + 1; } catch (e) {} | ||
| delay(0.6); | ||
| const b = w.bounds(); | ||
| return JSON.stringify({ x: b.x, y: b.y, width: b.width, height: b.height }); | ||
| } | ||
| if (cmd === 'url') return t.url() + '\\n' + t.title(); | ||
| if (cmd === 'read') { | ||
| const sel = argv[4] || 'body'; | ||
|
|
@@ -149,6 +159,11 @@ export class JxaBackend implements AttachBackend { | |
| } | ||
|
|
||
| async claimAgentTab(): Promise<string> { | ||
| // IDEMPOTENT: probeIdentity() claims a tab before the controller's own | ||
| // claimAgentTab() call, so a second `claim` would look for another | ||
| // about:blank tab that no longer exists and fail NO_BLANK_TAB. Reuse the | ||
| // tab this backend already holds instead of consuming a second one. | ||
| if (this.claimed) return `${this.claimed.wid} ${this.claimed.tid}`; | ||
| const out = await this.exec('claim', []); | ||
| const [wid, tid] = out.split(/\s+/); | ||
| if (!wid || !tid) throw new AttachError('TAB_CLOSED', `claim returned malformed ids: "${out}"`); | ||
|
|
@@ -173,4 +188,63 @@ export class JxaBackend implements AttachBackend { | |
| const [wid, tid] = tab.split(/\s+/); | ||
| return this.exec('read', [wid, tid, selector ?? 'body', String(maxChars ?? 6000)]); | ||
| } | ||
|
|
||
| /** | ||
| * Screenshot via macOS `screencapture` (base64 PNG). | ||
| * | ||
| * The JXA transport has no CDP pixel pipe, so this raises the agent tab's | ||
| * window to the front and captures the display. Two consequences the caller | ||
| * must accept: it FOCUSES the agent window (the one exception to the | ||
| * no-focus rule — a screenshot is meaningless if the window is behind | ||
| * others), and the capture is the whole screen, so `fullPage` is ignored. | ||
| * Requires the host to hold macOS Screen Recording permission; without it | ||
| * `screencapture` writes nothing and this surfaces a structured failure. | ||
| */ | ||
| async screenshotTab(tab: string, _fullPage?: boolean): Promise<string> { | ||
| // OFF BY DEFAULT. `screencapture -R` works in display POINTS while Chrome's | ||
| // window bounds and Retina backing scale disagree, so the cropped region | ||
| // drifted onto NEIGHBOURING windows in live testing (2026-07-24) — which | ||
| // would silently put the user's unrelated, private windows into a research | ||
| // evidence file. Correct page-pixel capture is CDP `Page.captureScreenshot` | ||
| // (see RawCdpBackend); this lane stays refused unless explicitly opted in | ||
| // via WUNDERLAND_ATTACH_JXA_SHOTS=1 for a single-display debug session. | ||
| if (process.env.WUNDERLAND_ATTACH_JXA_SHOTS !== '1') { | ||
| throw new AttachError( | ||
| 'UNSUPPORTED_OP', | ||
| 'screenshots require the CDP transport (WUNDERLAND_ATTACH_TRANSPORT=cdp); the JXA display-capture lane is disabled because its crop can include other windows', | ||
| ); | ||
| } | ||
| const [wid, tid] = tab.split(/\s+/); | ||
| const raised = await this.exec('raise', [wid, tid]); | ||
| const { mkdtempSync, readFileSync, existsSync, rmSync } = await import('node:fs'); | ||
| const { tmpdir } = await import('node:os'); | ||
| const file = join(mkdtempSync(join(tmpdir(), 'attach-shot-')), 'shot.png'); | ||
| // Crop to the agent WINDOW's bounds. A full-display capture would include | ||
| // every other window the user has open — private content must never land | ||
| // in a research evidence shot. | ||
| let region: string | undefined; | ||
| try { | ||
| const b = JSON.parse(raised) as { x: number; y: number; width: number; height: number }; | ||
| if ([b.x, b.y, b.width, b.height].every((n) => typeof n === 'number' && Number.isFinite(n)) && b.width > 0 && b.height > 0) { | ||
| region = `${Math.max(0, Math.round(b.x))},${Math.max(0, Math.round(b.y))},${Math.round(b.width)},${Math.round(b.height)}`; | ||
| } | ||
|
Comment on lines
+221
to
+230
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. suggestion (performance): Temporary directories created for screenshots are never removed, which can accumulate over time. Each screenshot call creates a new temp directory via Suggested implementation: const { mkdtempSync, readFileSync, existsSync, rmSync } = await import('node:fs');
const { tmpdir } = await import('node:os');
const dir = mkdtempSync(join(tmpdir(), 'attach-shot-'));
const file = join(dir, 'shot.png');You’ll also need to update the surrounding error-handling/cleanup logic:
|
||
| } catch { | ||
| /* older driver without bounds → refuse rather than capture the screen */ | ||
| } | ||
| if (!region) { | ||
| throw new AttachError('UNSUPPORTED_OP', 'could not resolve agent-window bounds; refusing a full-display capture'); | ||
| } | ||
| try { | ||
| await pExecFile('screencapture', ['-x', '-o', '-t', 'png', '-R', region, file], { timeout: 20_000 }); | ||
| if (!existsSync(file)) { | ||
| throw new AttachError('UNSUPPORTED_OP', 'screencapture produced no file (Screen Recording permission?)'); | ||
| } | ||
| return readFileSync(file).toString('base64'); | ||
| } catch (err) { | ||
| if (err instanceof AttachError) throw err; | ||
| throw new AttachError('UNSUPPORTED_OP', `screencapture failed: ${err instanceof Error ? err.message : String(err)}`); | ||
|
Comment on lines
+203
to
+245
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🔒 Security & Privacy | 🟠 Major | 🏗️ Heavy lift Do not expose the known-unsafe JXA capture fallback. The implementation acknowledges that its point/pixel mismatch can capture neighboring private windows, then enables that same path via an environment variable. Keep 🤖 Prompt for AI Agents |
||
| } finally { | ||
| rmSync(file, { force: true }); | ||
| } | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -206,6 +206,26 @@ export class RawCdpBackend implements AttachBackend { | |
| return value; | ||
| } | ||
|
|
||
| /** | ||
| * `Page.captureScreenshot` on the agent tab → base64 PNG (optional backend | ||
| * capability). Read-only: no navigation, no focus change, no page mutation. | ||
| * `fullPage` uses `captureBeyondViewport` so a long results page comes back | ||
| * whole instead of clipped to the viewport. | ||
| */ | ||
| async screenshotTab(tab: string, fullPage?: boolean): Promise<string> { | ||
| this.requireTab(tab); | ||
| const res = (await this.send( | ||
| 'Page.captureScreenshot', | ||
| { format: 'png', captureBeyondViewport: !!fullPage }, | ||
| this.sessionId!, | ||
|
Comment on lines
+218
to
+220
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When Useful? React with 👍 / 👎. |
||
| 45_000, | ||
|
Comment on lines
+215
to
+221
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🩺 Stability & Availability | 🟠 Major | 🏗️ Heavy lift 🧩 Analysis chain🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '\n== raw-cdp.ts around screenshotTab ==\n'
sed -n '180,280p' registry/curated/system/browser-automation/src/attach/backends/raw-cdp.ts
printf '\n== search for screenshot-related guards and metrics ==\n'
rg -n "captureBeyondViewport|Page\.getLayoutMetrics|layoutMetrics|screenshotTab|captureScreenshot|clip|pixel budget|fullPage" registry/curated/system/browser-automation/src -SRepository: framerslab/agentos-extensions Length of output: 8311 🏁 Script executed: #!/bin/bash
set -euo pipefail
sed -n '1,260p' registry/curated/system/browser-automation/src/attach/backends/raw-cdp.tsRepository: framerslab/agentos-extensions Length of output: 10155 🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '\n== raw-cdp.ts around screenshotTab ==\n'
sed -n '180,280p' registry/curated/system/browser-automation/src/attach/backends/raw-cdp.ts
printf '\n== search for screenshot-related guards and metrics ==\n'
rg -n "captureBeyondViewport|Page\.getLayoutMetrics|layoutMetrics|screenshotTab|captureScreenshot|clip|pixel budget|fullPage" registry/curated/system/browser-automation/src -SRepository: framerslab/agentos-extensions Length of output: 8311 🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '\n== AttachController screenshot flow ==\n'
sed -n '260,340p' registry/curated/system/browser-automation/src/attach/AttachController.ts
printf '\n== daemon/client screenshot path ==\n'
sed -n '100,150p' registry/curated/system/browser-automation/src/attach/daemon/client.ts
sed -n '240,290p' registry/curated/system/browser-automation/src/attach/daemon/daemon.ts
printf '\n== search for screenshot limits / byte caps ==\n'
rg -n "screenshot.*(limit|max|cap|budget|bytes|size)|base64|Buffer\\.from\\(.*base64|captureBeyondViewport|fullPage" registry/curated/system/browser-automation/src -SRepository: framerslab/agentos-extensions Length of output: 13686 Cap full-page screenshot output.
🤖 Prompt for AI Agents |
||
| )) as { data?: string }; | ||
| if (!res.data) { | ||
| throw new AttachError('UNKNOWN', 'Page.captureScreenshot returned no data'); | ||
| } | ||
| return res.data; | ||
| } | ||
|
|
||
| private requireTab(tab: string): void { | ||
| if (!this.sessionId || tab !== this.createdTargetId) { | ||
| throw new AttachError('TAB_CLOSED', 'agent tab is not claimed on this transport'); | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -18,7 +18,7 @@ | |||||||||||||||||||||||||||||||||||||
| * @module browser-automation/attach/tools | ||||||||||||||||||||||||||||||||||||||
| */ | ||||||||||||||||||||||||||||||||||||||
| import type { AttachSurface } from './AttachController.js'; | ||||||||||||||||||||||||||||||||||||||
| import { toStructuredError } from './errors.js'; | ||||||||||||||||||||||||||||||||||||||
| import { AttachError, toStructuredError } from './errors.js'; | ||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||
| /** Wrap an operation into the standard tool result envelope. */ | ||||||||||||||||||||||||||||||||||||||
| async function envelope(fn) { | ||||||||||||||||||||||||||||||||||||||
|
|
@@ -110,6 +110,36 @@ export class AttachReadTool { | |||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||
| /** `browser_attach_screenshot` — PNG evidence capture of the agent tab (CDP only). */ | ||||||||||||||||||||||||||||||||||||||
| export class AttachScreenshotTool { | ||||||||||||||||||||||||||||||||||||||
| readonly id = 'browser_attach_screenshot'; | ||||||||||||||||||||||||||||||||||||||
| readonly name = 'browser_attach_screenshot'; | ||||||||||||||||||||||||||||||||||||||
| readonly displayName = 'Attach: screenshot'; | ||||||||||||||||||||||||||||||||||||||
| readonly description = | ||||||||||||||||||||||||||||||||||||||
| 'Capture a PNG screenshot of the agent tab to a local file path — visual evidence for a research report. Requires the CDP transport (the AppleScript/JXA transport has no pixel access and refuses with UNSUPPORTED_OP). Read-only: never navigates or mutates the page.'; | ||||||||||||||||||||||||||||||||||||||
| readonly category = 'browser'; | ||||||||||||||||||||||||||||||||||||||
| readonly version = '0.1.0'; | ||||||||||||||||||||||||||||||||||||||
| readonly hasSideEffects = false; | ||||||||||||||||||||||||||||||||||||||
|
Comment on lines
+118
to
+122
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. suggestion (bug_risk): The screenshot tool writes a file to disk, which is a side effect despite being read-only from the page’s perspective. Although the operation is read-only for page state,
Suggested change
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When a mission uses the documented Useful? React with 👍 / 👎.
Comment on lines
+119
to
+122
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🔒 Security & Privacy | 🟠 Major | ⚡ Quick win Mark the screenshot tool as side-effecting. This tool creates directories and writes a PNG, but Line 122 declares Proposed fix- readonly hasSideEffects = false;
+ readonly hasSideEffects = true;📝 Committable suggestion
Suggested change
🤖 Prompt for AI Agents |
||||||||||||||||||||||||||||||||||||||
| readonly inputSchema = { | ||||||||||||||||||||||||||||||||||||||
| type: 'object' as const, | ||||||||||||||||||||||||||||||||||||||
| properties: { | ||||||||||||||||||||||||||||||||||||||
| path: { type: 'string', description: 'Absolute file path to write the PNG to' }, | ||||||||||||||||||||||||||||||||||||||
| fullPage: { type: 'boolean', description: 'Capture the full scrollable page instead of just the viewport' }, | ||||||||||||||||||||||||||||||||||||||
| }, | ||||||||||||||||||||||||||||||||||||||
| required: ['path'], | ||||||||||||||||||||||||||||||||||||||
| }; | ||||||||||||||||||||||||||||||||||||||
| constructor(private controller: AttachSurface) {} | ||||||||||||||||||||||||||||||||||||||
| async execute(args: { path: string; fullPage?: boolean }) { | ||||||||||||||||||||||||||||||||||||||
| return envelope(async () => { | ||||||||||||||||||||||||||||||||||||||
| const fn = this.controller.screenshot?.bind(this.controller); | ||||||||||||||||||||||||||||||||||||||
| if (!fn) { | ||||||||||||||||||||||||||||||||||||||
| throw new AttachError('UNSUPPORTED_OP', 'this attach transport has no screenshot capability (CDP required)'); | ||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||
| return fn(args.path, args.fullPage); | ||||||||||||||||||||||||||||||||||||||
| }); | ||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||
| /** `browser_attach_release` — park + release the lease (no browser teardown). */ | ||||||||||||||||||||||||||||||||||||||
| export class AttachReleaseTool { | ||||||||||||||||||||||||||||||||||||||
| readonly id = 'browser_attach_release'; | ||||||||||||||||||||||||||||||||||||||
|
|
@@ -188,6 +218,7 @@ export function createAttachTools(controller: AttachSurface) { | |||||||||||||||||||||||||||||||||||||
| new AttachClaimTool(controller), | ||||||||||||||||||||||||||||||||||||||
| new AttachGotoTool(controller), | ||||||||||||||||||||||||||||||||||||||
| new AttachReadTool(controller), | ||||||||||||||||||||||||||||||||||||||
| new AttachScreenshotTool(controller), | ||||||||||||||||||||||||||||||||||||||
| new AttachReleaseTool(controller), | ||||||||||||||||||||||||||||||||||||||
| new AttachControlTool(controller), | ||||||||||||||||||||||||||||||||||||||
| ]; | ||||||||||||||||||||||||||||||||||||||
|
|
@@ -199,6 +230,7 @@ export const ATTACH_TOOL_IDS = [ | |||||||||||||||||||||||||||||||||||||
| 'browser_attach_claim', | ||||||||||||||||||||||||||||||||||||||
| 'browser_attach_goto', | ||||||||||||||||||||||||||||||||||||||
| 'browser_attach_read', | ||||||||||||||||||||||||||||||||||||||
| 'browser_attach_screenshot', | ||||||||||||||||||||||||||||||||||||||
| 'browser_attach_release', | ||||||||||||||||||||||||||||||||||||||
| 'browser_attach_control', | ||||||||||||||||||||||||||||||||||||||
| ] as const; | ||||||||||||||||||||||||||||||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,16 +1,26 @@ | ||
| { | ||
| "compilerOptions": { | ||
| "target": "ES2020", | ||
| "module": "commonjs", | ||
| "lib": ["ES2020"], | ||
| "target": "ES2022", | ||
| "module": "Node16", | ||
| "lib": [ | ||
| "ES2022" | ||
| ], | ||
| "outDir": "./dist", | ||
| "rootDir": "./src", | ||
| "strict": true, | ||
| "esModuleInterop": true, | ||
| "declaration": true, | ||
| "declarationMap": true, | ||
| "sourceMap": true | ||
| "sourceMap": true, | ||
| "moduleResolution": "Node16" | ||
| }, | ||
| "include": ["src/**/*"], | ||
| "exclude": ["node_modules", "dist", "test", "**/*.spec.ts"] | ||
| "include": [ | ||
| "src/**/*" | ||
| ], | ||
| "exclude": [ | ||
| "node_modules", | ||
| "dist", | ||
| "test", | ||
| "**/*.spec.ts" | ||
| ] | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🔒 Security & Privacy | 🟠 Major | 🏗️ Heavy lift
Constrain screenshot destinations before writing.
Line 300 writes a claimant-supplied path as the daemon user. Since daemon validation only checks that
pathis non-empty, a caller can overwrite arbitrary files or target special files. Restrict output to a configured evidence directory and reject traversal, symlinks, and non-regular targets; use exclusive/atomic creation.🤖 Prompt for AI Agents