fix(desktop): hand the panel occlusion frame only to a user-driven renderer

capturePanelSnapshot sent a JPEG of the agent browser's current page to the app
renderer, which is content that renderer's own JS cannot otherwise read — the
view is a separate process composited over the window. A compromised renderer
can drive set-panel-bounds, panel-action navigate and set-panel-occluded itself,
so it could aim the shared agent browser at a site with a persisted session and
collect its pixels by script alone, bypassing the tool-call binding that guards
browser_screenshot for exactly this reason.

The frame is now withheld unless the main process has seen recent real input in
that renderer, which is what an overlay opening actually represents. The
flicker-prevention behaviour is untouched: an empty capture already returns
before the send while `finally` still runs the occlusion, so "occlude without a
placeholder" is a path the state machine already handles rather than a new one.

Author: waleedlatif1

apps/desktop/src/main/browser-agent/panel.test.ts

@@ -2,8 +2,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
 
 vi.mock('electron', () => import('@/test/electron-mock'))
 
+import type { WebContents } from 'electron'
 import { BrowserWindow, WebContentsView } from 'electron'
 import * as panelModule from '@/main/browser-agent/panel'
+import { trackInputActivity } from '@/main/input-activity'
 
 type PanelModule = typeof import('@/main/browser-agent/panel')
 
@@ -27,6 +29,29 @@ function freshPanel(): PanelModule {
 
 const PANEL_RECT = { x: 400, y: 64, width: 600, height: 800 }
 
+type InputListener = (event: unknown, input: { type: string }) => void
+
+/**
+ * Registers a window's renderer with the main-process input tracker and returns
+ * a way to give it a real click. The occlusion snapshot is only handed over to a
+ * renderer the user has recently driven, so a fixture has to say so explicitly.
+ */
+function trackWindowInput(win: BrowserWindow) {
+  const listeners: InputListener[] = []
+  const contents = win.webContents as unknown as {
+    on: (channel: string, listener: InputListener) => void
+    isDestroyed: () => boolean
+  }
+  contents.on = (channel, listener) => {
+    if (channel === 'input-event') listeners.push(listener)
+  }
+  contents.isDestroyed = () => false
+  trackInputActivity(win.webContents as WebContents)
+  return () => {
+    for (const listener of listeners) listener({}, { type: 'mouseDown' })
+  }
+}
+
 /** A panel showing one tab, which is the state occlusion applies to. */
 function showPanel(panel: PanelModule) {
   const win = new BrowserWindow()
@@ -38,13 +63,15 @@ function showPanel(panel: PanelModule) {
     ensureInitialTab: () => {},
     onViewDetached: () => {},
   })
+  const press = trackWindowInput(win)
+  press()
   panel.setPanelBounds(PANEL_RECT, win)
   /** Swaps in another tab's view, as switching tabs does. */
   const switchTab = (next: WebContentsView) => {
     active = { id: 'tab-2', view: next, pinned: false }
     panel.layout()
   }
-  return { win, view, switchTab }
+  return { win, view, switchTab, press }
 }
 
 /** When the view was hidden, in the global mock invocation order. */
@@ -91,6 +118,38 @@ describe('panel occlusion', () => {
     expect(sent).toBeLessThan(hiddenAt(view) as number)
   })
 
+  it('withholds the frame from a renderer the user has not driven', async () => {
+    const panel = freshPanel()
+    const { win, view } = showPanel(panel)
+    // A script-only caller: no real OS input has reached this renderer inside
+    // the recency window, so the pixels of the agent page are not handed over.
+    vi.useFakeTimers()
+    try {
+      vi.advanceTimersByTime(5_000)
+      panel.setPanelOccluded(true, win)
+      await vi.waitFor(() => expect(hiddenAt(view)).toBeDefined())
+    } finally {
+      vi.useRealTimers()
+    }
+
+    expect(snapshotSentAt(win)).toBeUndefined()
+  })
+
+  it('still occludes when the frame is withheld, so nothing is left half-applied', async () => {
+    const panel = freshPanel()
+    const { win, view } = showPanel(panel)
+    vi.useFakeTimers()
+    try {
+      vi.advanceTimersByTime(5_000)
+      panel.setPanelOccluded(true, win)
+      // Hiding must not depend on the snapshot: an empty capture already
+      // reaches the same path, so this is an existing outcome, not a new one.
+      await vi.waitFor(() => expect(hiddenAt(view)).toBeDefined())
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
   it('stays visible when the overlay closes while the frame is being taken', async () => {
     const { win, view } = showPanel(panel)
 

apps/desktop/src/main/browser-agent/panel.ts

@@ -20,6 +20,7 @@ import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
 import type { BrowserWindow, WebContentsView } from 'electron'
 import type { AgentTab } from '@/main/browser-agent/session'
+import { hasRecentDeliberateInput } from '@/main/input-activity'
 
 const logger = createLogger('BrowserAgentPanel')
 
@@ -333,6 +334,22 @@ function capturePanelSnapshot(onSettled?: () => void): void {
       // picture of the page, so it goes to the window still showing the
       // browser or nowhere at all.
       if (panelWindow() !== win || win.isDestroyed()) return
+      // A frame of the agent browser is content the renderer's own JS cannot
+      // otherwise read — the view is a separate process composited over the
+      // window. It is handed over so an overlay can show a placeholder instead
+      // of a blank gap, and an overlay opens because the user did something. A
+      // compromised renderer can drive set-panel-bounds, panel-action navigate
+      // and set-panel-occluded on its own, so without this the pixels of an
+      // authenticated page could be collected by script alone, bypassing the
+      // tool-call binding that guards browser_screenshot for exactly this.
+      //
+      // Skipping the send is an already-supported outcome: an empty capture
+      // returns here too, and `finally` still runs the occlusion, so no new
+      // state is introduced — at worst a placeholder is missing for one frame.
+      if (!hasRecentDeliberateInput(win.webContents)) {
+        logger.warn('Withheld browser panel snapshot with no recent user input')
+        return
+      }
       // Downscale and JPEG-encode before crossing IPC. capturePage returns a
       // device-pixel PNG — on a retina half-window that is millions of pixels,
       // and toDataURL's PNG encode is synchronous on the main process, so a

apps/desktop/src/main/browser-agent/session.test.ts

@@ -4,9 +4,11 @@ vi.mock('electron', () => import('@/test/electron-mock'))
 
 import { MAX_BROWSER_TABS } from '@sim/browser-protocol'
 import { sleep } from '@sim/utils/helpers'
+import type { WebContents } from 'electron'
 import { BrowserWindow, session as electronSession } from 'electron'
 import * as panel from '@/main/browser-agent/panel'
 import * as sessionModule from '@/main/browser-agent/session'
+import { trackInputActivity } from '@/main/input-activity'
 
 type SessionModule = typeof import('@/main/browser-agent/session')
 
@@ -34,6 +36,8 @@ interface MockView {
   setVisible: ReturnType<typeof vi.fn>
 }
 
+type InputListener = (event: unknown, input: { type: string }) => void
+
 function mainWindowMock() {
   const win = new BrowserWindow() as unknown as {
     contentView: {
@@ -43,6 +47,20 @@ function mainWindowMock() {
     webContents: { getZoomFactor?: ReturnType<typeof vi.fn> }
   }
   win.webContents.getZoomFactor = vi.fn(() => 1)
+  // The occlusion snapshot is only handed to a renderer the user has recently
+  // driven, so these fixtures — which stand in for a user with the panel open —
+  // register with the input tracker and report one real click.
+  const listeners: InputListener[] = []
+  const contents = win.webContents as unknown as {
+    on: (channel: string, listener: InputListener) => void
+    isDestroyed: () => boolean
+  }
+  contents.on = (channel, listener) => {
+    if (channel === 'input-event') listeners.push(listener)
+  }
+  contents.isDestroyed = () => false
+  trackInputActivity(win.webContents as unknown as WebContents)
+  for (const listener of listeners) listener({}, { type: 'mouseDown' })
   return win as unknown as BrowserWindow
 }