diff --git a/CHANGELOG.md b/CHANGELOG.md index 9469b97f3..36f382dc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,31 @@ All notable changes to WebBrain are documented in this file. This changelog was generated from the repository Git history and release tags. Versions without a Git tag are inferred from version-bump commits and the current `package.json` / browser manifest versions. +## [26.0.0] - 2026-07-26 + +### Added +- Added non-blocking local browser runs: regular Chrome and Firefox tasks stay pinned to their original tab without activating it or stealing window focus, while `/foreground [prompt]` provides a one-run compatibility escape hatch for sites that must render visually in the foreground. +- Added offline converters for exported traces in Agent Trajectory Interchange Format (ATIF) and OpenTelemetry Protocol (OTLP), with documentation and regression coverage. +- Added an accessible in-panel **New conversation** confirmation that works when Vivaldi Web Panels suppress native browser dialogs. + +### Changed +- Rebuilt CAPTCHA handling around frame-aware challenge detection and targeted response injection for reCAPTCHA, hCaptcha, and Turnstile, including branded-dialog and ancestor-loader recognition, cross-frame verification, manual-completion recovery, explicit gate abandonment, and fail-closed handling for ambiguous or conflicting candidates. +- Isolated selected-text context from independent, scheduled, cloud, and saved-workflow runs while preserving grounded follow-ups, attachments, compaction, and shortcuts within the originating selection conversation. +- Extended the rapid duplicate-submit guard to Firefox and made acknowledged resubmits re-arm the protection window in both browsers. +- Refreshed and reorganized the English, French, and Chinese documentation, landing-page demos, language ordering, and agent/skills/slash-command references. +- Polished the floating chat-navigation control, linked Settings title behavior, full-screen recording status, and slash-command keyboard actions. +- Extracted shared loop-detection, image-budget, and text tool-call parsing helpers without changing their browser parity. + +### Fixed +- Rejected blank Chrome full-page captures, improved background screenshot compatibility, and verified inactive Firefox tabs before declaring a run complete. +- Fixed CapSolver key migration so valid saved `CAP-` keys enable solving consistently while malformed keys remain disabled. +- Fixed selection-scoped history, workflow, attachment, and navigation paths that could leak context into unrelated runs. +- Fixed CAPTCHA frame visibility, challenge correlation, response-field targeting, token fallback, and recovery after redirects or manual completion. +- Fixed the full-screen recording message so it matches whether the recording indicator is actually visible. + +### Tests +- Added mirrored Chrome/Firefox regression coverage for background runs, inactive-tab capture, CAPTCHA gating and injection, selection-scope isolation, duplicate-submit protection, trace conversion, and the updated UI flows. + ## [25.9.0] - 2026-07-25 ### Added diff --git a/README.md b/README.md index 8427a90bc..512e5f5d5 100644 --- a/README.md +++ b/README.md @@ -212,10 +212,11 @@ they diverge. Chrome DevTools Protocol via `chrome.debugger`, so the Firefox build has no shadow-DOM piercing, no real trusted mouse events (some React/Vue handlers won't fire), no closed-shadow-root traversal, no `resolveSelector` retry budget, no -SPA-navigation-aware retry, and no CDP screenshots (it falls back to -`tabs.captureVisibleTab`, active tabs only). Site adapters, vision detection, -loop detection, the auto-screenshot loop, and the Compact prompt/tool set *are* -mirrored to Firefox. Some single-page apps may also fail to trigger +SPA-navigation-aware retry, and no CDP screenshots. It uses `tabs.captureTab` +for viewport screenshots, including inactive run tabs, but still cannot provide +Chrome's pixel-perfect or full-page CDP capture. Site adapters, vision +detection, loop detection, the auto-screenshot loop, and the Compact prompt/tool +set *are* mirrored to Firefox. Some single-page apps may also fail to trigger content-script re-injection after client-side navigation. ## Contributing diff --git a/docs/architecture.md b/docs/architecture.md index 03ab3710d..d36e87d50 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -719,7 +719,7 @@ Firefox uses `browser.storage.session`. |---|---|---| | Background | Service worker (ephemeral) | Background page (persistent) | | Events | CDP-trusted (`isTrusted=true`) | Synthetic (`isTrusted=false`) | -| Screenshots | CDP `Page.captureScreenshot` | `browser.tabs.captureVisibleTab()` | +| Screenshots | CDP `Page.captureScreenshot` with run-scoped focus emulation for background tabs | `browser.tabs.captureTab()` for direct inactive-tab capture | | Conversation/UI persistence | `chrome.storage.session` | `browser.storage.session` | | Offscreen document | Yes (fetch proxy + recorder) | Not available | | Trace recorder | IndexedDB (opt-in) | IndexedDB (opt-in) — same `trace/recorder.js` | diff --git a/docs/fr/architecture.md b/docs/fr/architecture.md index 733073130..cf3148626 100644 --- a/docs/fr/architecture.md +++ b/docs/fr/architecture.md @@ -402,7 +402,7 @@ rouvert reconstruise correctement le Markdown en cours. Chrome utilise |---|---|---| | Arrière-plan | Service worker (éphémère) | Page d'arrière-plan (persistante) | | Événements | CDP de confiance (`isTrusted=true`) | Synthétiques (`isTrusted=false`) | -| Captures d'écran | CDP `Page.captureScreenshot` | `browser.tabs.captureVisibleTab()` | +| Captures d'écran | CDP `Page.captureScreenshot` avec émulation de focus limitée à l'exécution en arrière-plan | `browser.tabs.captureTab()` pour capturer directement un onglet inactif | | Persistance conversation/interface | `chrome.storage.session` | `browser.storage.session` | | Document hors-écran | Oui (proxy fetch + enregistreur) | Non disponible | | Enregistreur de trace | IndexedDB (optionnel) | IndexedDB (optionnel) — même `trace/recorder.js` | diff --git a/docs/fr/slash-commands.md b/docs/fr/slash-commands.md index 28750fa10..d3c16a038 100644 --- a/docs/fr/slash-commands.md +++ b/docs/fr/slash-commands.md @@ -31,6 +31,7 @@ d'une espace ouvre l'autocomplétion de ses options disponibles. | `/workflow --export ` | Télécharger un fichier JSON portable `webbrain-workflow/1` assaini | | `/workflow --import --file` | Importer un fichier de workflow portable comme nouveau workflow local | | `/allow-api` | **Dérogation de mutation API par conversation.** Voir [plus bas](#allow-api). | +| `/foreground [invite]` | Exécuter une tâche locale au premier plan pour assurer la compatibilité visuelle | | `/dangerously-skip-permissions` | **Contournement global des demandes d'autorisation.** Désactive `Ask before consequential actions` sans ouvrir les Paramètres. WebBrain agira sans demandes par site jusqu'à ce que vous réactiviez le réglage. | | `/compact` | Force le compactage du contexte pour la conversation actuelle | | `/verbose` | Bascule l'affichage verbeux/compact des outils | @@ -64,6 +65,22 @@ quitter la page initiale ne la ramène pas vers l'URL surveillée, et l'onglet auxiliaire est fermé à la fin du watch. Les échecs d'interrogation transitoires sont tolérés ; trois échecs consécutifs arrêtent le watch. +## `/foreground` + +Les exécutions locales ordinaires restent liées à leur onglet d'origine et +fonctionnent sans activer cet onglet ni donner le focus à sa fenêtre. Chrome +effectue les captures via CDP avec une émulation du focus limitée à l'exécution ; +Firefox capture directement l'onglet cible avec `tabs.captureTab`. Si Chrome +renvoie plusieurs fois une image vide en arrière-plan, WebBrain l'écarte et +continue à partir du DOM et des données d'accessibilité. + +Utilisez `/foreground ` comme solution de compatibilité pour une seule +exécution lorsqu'un site ne restitue pas correctement son état visuel en +arrière-plan. Cette commande rétablit l'activation de l'onglet et le focus de la +fenêtre pour cette exécution uniquement. Elle n'est pas persistante, et les +exécutions Cloud gérées conservent leur comportement actuel au premier plan car +leur navigateur est dédié à la tâche. + ## `/allow-api` `/allow-api` lève la restriction UI-d'abord pour la conversation en cours, afin @@ -92,10 +109,11 @@ zone visible juste avant et après l'exécution (Chrome et Firefox). Par exemple `checkout-before.png` et `checkout-after.png` ; sans `--save-as`, WebBrain utilise des noms horodatés. -Si l'exécution ouvre un autre onglet, WebBrain réactive l'onglet d'origine avant -d'enregistrer la capture « après ». Si l'enregistrement ou la capture initiale ne -peut pas être démarré et sauvegardé, l'exécution n'est pas envoyée. Les -`/record` et `/screenshot` autonomes conservent leur comportement existant. +Pour ce suffixe de diagnostic, Chrome peut réactiver l'onglet d'origine avant +d'enregistrer la capture « après ». Firefox capture directement cet onglet sans +l'activer. Si l'enregistrement ou la capture initiale ne peut pas être démarré +et sauvegardé, l'exécution n'est pas envoyée. Les `/record` et `/screenshot` +autonomes conservent leur comportement existant. ## Exports, instantanés et workflows diff --git a/docs/slash-commands.md b/docs/slash-commands.md index 738ac0905..9d8b242d6 100644 --- a/docs/slash-commands.md +++ b/docs/slash-commands.md @@ -31,6 +31,7 @@ for its available flags. | `/workflow --export ` | Download a sanitized portable `webbrain-workflow/1` JSON file | | `/workflow --import --file` | Import a portable workflow file as a new local saved workflow | | `/allow-api` | **Per-conversation API mutation override.** See [below](#allow-api). | +| `/foreground [prompt]` | Run one local task in the foreground for visual compatibility | | `/dangerously-skip-permissions` | **Global permission-prompt bypass.** Turns off `Ask before consequential actions` without opening Settings. WebBrain will act without per-site prompts until you re-enable the setting. | | `/compact` | Force context compaction for the current conversation | | `/verbose` | Toggle verbose/compact tool display | @@ -63,6 +64,20 @@ navigate it back to the watched URL, and the helper tab is closed when the watch ends. Transient poll failures are tolerated; three consecutive failures stop the watch. +## `/foreground` + +Regular local runs stay pinned to their original tab and operate without +activating that tab or focusing its window. Chrome captures through CDP with +focus emulation scoped to the run; Firefox captures the target tab directly +with `tabs.captureTab`. If Chrome repeatedly returns a blank background frame, +WebBrain discards it and continues from DOM and accessibility data. + +Use `/foreground ` as a one-run compatibility escape hatch for a site +whose visual state does not render correctly in the background. It restores tab +activation and window focus for that run. The setting is not persistent, and +managed cloud runs retain their existing foreground behavior because their +browser is dedicated to the task. + ## `/allow-api` `/allow-api` lifts the UI-first restriction for the current conversation so the @@ -89,10 +104,11 @@ immediately before and after the run (Chrome and Firefox). For example, `checkout-before.png` and `checkout-after.png`; without `--save-as`, WebBrain uses timestamped filenames. -If the run opens another tab, WebBrain reactivates the originating run tab -before saving the after screenshot. If the recording or initial screenshot -cannot be started and saved, the run is not sent. Standalone `/record` and -`/screenshot` keep their existing behavior. +For the Chrome diagnostic suffix, WebBrain may reactivate the originating run +tab before saving the after screenshot. Firefox captures that tab directly +without activating it. If the recording or initial screenshot cannot be +started and saved, the run is not sent. Standalone `/record` and `/screenshot` +keep their existing behavior. ## Exports, snapshots, and workflows diff --git a/docs/zh-CN/architecture.md b/docs/zh-CN/architecture.md index ca2c19a7b..d86846672 100644 --- a/docs/zh-CN/architecture.md +++ b/docs/zh-CN/architecture.md @@ -278,7 +278,7 @@ while (steps < maxSteps) { |---|---|---| | 后台 | Service Worker（临时） | 后台页面（持久化） | | 事件 | CDP 受信任（`isTrusted=true`） | 合成事件（`isTrusted=false`） | -| 截图 | CDP `Page.captureScreenshot` | `browser.tabs.captureVisibleTab()` | +| 截图 | CDP `Page.captureScreenshot`，后台运行时使用仅限该次运行的焦点模拟 | `browser.tabs.captureTab()`，可直接捕获非活动标签页 | | 对话/UI 持久化 | `chrome.storage.session` | `browser.storage.session` | | 离屏文档 | 有（fetch 代理 + 录制器） | 不可用 | | 轨迹记录器 | IndexedDB（可选） | IndexedDB（可选）— 相同的 `trace/recorder.js` | diff --git a/docs/zh-CN/slash-commands.md b/docs/zh-CN/slash-commands.md index c5fe89ece..e2cdfe5da 100644 --- a/docs/zh-CN/slash-commands.md +++ b/docs/zh-CN/slash-commands.md @@ -29,6 +29,7 @@ WebBrain 接受作为输入框某行开头的斜杠命令。在面板内输入 ` | `/workflow --export ` | 下载经过清洗的可移植 `webbrain-workflow/1` JSON 文件 | | `/workflow --import --file` | 将可移植工作流文件导入为新的本地工作流 | | `/allow-api` | **按对话的 API 变更覆盖。** 参见[下文](#allow-api)。 | +| `/foreground [提示词]` | 为视觉兼容性在前台运行一次本地任务 | | `/dangerously-skip-permissions` | **全局权限提示绕过。** 无需打开设置即可关闭「操作前询问」。在你重新启用该设置之前，WebBrain 将不再按站点弹出提示。 | | `/compact` | 强制压缩当前对话上下文 | | `/verbose` | 切换详细/压缩工具显示 | @@ -55,6 +56,17 @@ WebBrain 接受作为输入框某行开头的斜杠命令。在面板内输入 ` 轮询在专用的非活动标签页中进行，因此离开发起页面不会把它导航回被监视的 URL；watch 结束时会关闭该辅助标签页。短暂的轮询失败可以容忍；连续三次失败会停止 watch。 +## `/foreground` + +常规本地运行会固定在最初的标签页上执行，不会激活该标签页或聚焦其窗口。Chrome 通过 +CDP 截图，并仅在该次运行期间模拟焦点；Firefox 使用 `tabs.captureTab` 直接捕获目标 +标签页。如果 Chrome 多次返回空白的后台画面，WebBrain 会丢弃该画面，继续使用 DOM +和无障碍数据。 + +若某个网站无法在后台正确渲染视觉状态，可使用 `/foreground <提示词>` 作为仅对本次 +运行生效的兼容模式。它会为该次运行恢复标签页激活和窗口聚焦，不会成为持久设置。 +托管 Cloud 运行仍保留现有前台行为，因为其浏览器专用于该任务。 + ## `/allow-api` `/allow-api` 会为当前对话解除 UI 优先限制，使智能体在 UI 失败时可通过 `fetch_url` @@ -77,9 +89,9 @@ WebBrain 接受作为输入框某行开头的斜杠命令。在面板内输入 ` `测试结账流程 /screenshot --save-as checkout.png` 会保存 `checkout-before.png` 和 `checkout-after.png`；若不带 `--save-as`，WebBrain 使用带时间戳的文件名。 -如果运行打开了另一个标签页，WebBrain 会在保存「之后」截图前重新激活发起运行的标签页。 -如果录制或初始截图无法启动并保存，该运行不会被发送。独立的 `/record` 与 -`/screenshot` 保持原有行为。 +对于这个诊断后缀，Chrome 可能会在保存「之后」截图前重新激活发起运行的标签页； +Firefox 会直接捕获该标签页而不激活它。如果录制或初始截图无法启动并保存，该运行不会 +被发送。独立的 `/record` 与 `/screenshot` 保持原有行为。 ## 导出、快照与工作流 diff --git a/package-lock.json b/package-lock.json index 690b682dd..22fd7de8b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "webbrain", - "version": "25.9.7", + "version": "26.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "webbrain", - "version": "25.9.7", + "version": "26.0.0", "license": "MIT", "devDependencies": { "playwright": "^1.48.0" diff --git a/package.json b/package.json index dae405ba3..9e3532f5b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "webbrain", - "version": "25.9.7", + "version": "26.0.0", "description": "Open-source AI browser agent — chat with pages, automate tasks, multi-provider LLM support.", "private": true, "type": "module", diff --git a/src/chrome/ARCHITECTURE.md b/src/chrome/ARCHITECTURE.md index 99ed5bccb..322d7d325 100644 --- a/src/chrome/ARCHITECTURE.md +++ b/src/chrome/ARCHITECTURE.md @@ -1,6 +1,6 @@ # WebBrain Chrome/Edge Extension — Architecture -> Version 25.9.7 · Manifest V3 · Service Worker background +> Version 26.0.0 · Manifest V3 · Service Worker background ## High-Level Overview diff --git a/src/chrome/manifest.json b/src/chrome/manifest.json index a372ad8f7..30a5595e3 100644 --- a/src/chrome/manifest.json +++ b/src/chrome/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "WebBrain", - "version": "25.9.7", + "version": "26.0.0", "description": "Open-source AI browser agent — chat with pages, automate tasks, multi-provider LLM support.", "permissions": [ "sidePanel", diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index 801058d9c..8cfe4e74a 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -365,6 +365,13 @@ export class Agent extends LoopDetector { this._recentSubmitClicks = new Map(); // tabId -> recent submit click timestamps this._formValidationBlocks = new Map(); // tabId -> validation state that must change before another submit this._runningTabs = new Set(); // tabIds with an active processMessage/Stream in flight + // Ordinary runs capture and act in their original tab without activating + // it. Managed cloud runs and the explicit /foreground compatibility + // override retain the old Page.bringToFront behavior for their lifetime. + this._foregroundCaptureTabs = new Set(); + // Focus emulation is enabled lazily for background screenshot paths. CDP + // sessions outlive individual runs, so every run cleanup must disable it. + this._focusEmulatedTabs = new Set(); this.completionInvariants = new Map(); // tabId -> run-scoped post-action verification state this._completionRunCounter = 0; this.scheduler = null; @@ -773,19 +780,48 @@ export class Agent extends LoopDetector { } } + async _captureFullPageWithBlankRetry(tabId, capturePolicy) { + const probe = await this._captureViewportProbe(tabId); + const captureOnce = async () => { + const capture = await this._withIndicatorsHidden(tabId, () => + cdpClient.captureFullPageScreenshot(tabId, capturePolicy) + ); + const imageData = typeof capture === 'string' ? capture : capture?.data; + if (!imageData) return null; + return { + dataUrl: `data:image/png;base64,${imageData}`, + capture, + }; + }; + return this._retryBlankScreenshotCapture( + await captureOnce(), + captureOnce, + { + probe, + // A full-page image is expected to reflect the whole scroll extent. + // Document length alone therefore cannot distinguish a legitimate + // uniform page from a blank background compositor frame. + includeDocumentLengthSignal: false, + }, + ); + } + async captureFullPageScreenshotForUser(tabId) { if (!tabId) return { ok: false, error: 'No tab ID' }; try { const capturePolicy = await this._getFullPageCapturePolicy(tabId); await cdpClient.attach(tabId); - await this._bringToFrontForCapture(tabId); - const capture = await this._withIndicatorsHidden(tabId, () => - cdpClient.captureFullPageScreenshot(tabId, capturePolicy) - ); - const imageData = typeof capture === 'string' ? capture : capture?.data; + await this._preparePageForCapture(tabId); + const captured = await this._captureFullPageWithBlankRetry(tabId, capturePolicy); + const capture = captured?.capture; const warning = typeof capture === 'object' ? capture?.warning || null : null; - if (!imageData) return { ok: false, error: 'Full-page screenshot returned no image data' }; - return { ok: true, dataUrl: `data:image/png;base64,${imageData}`, warning }; + if (!captured?.dataUrl) { + return { ok: false, error: 'Full-page screenshot returned no image data' }; + } + if (captured?.blankFrameRetry?.finalBlank) { + return { ok: false, error: 'Background full-page screenshot remained blank after retries' }; + } + return { ok: true, dataUrl: captured.dataUrl, warning }; } catch (e) { return { ok: false, error: e?.message || String(e) }; } @@ -2831,6 +2867,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d frameId: Number.isInteger(entry?.frameId) ? entry.frameId : 0, payload: entry?.result, })); + const hiddenChallenges = entries.flatMap(entry => { + const label = String(entry.payload?.hiddenChallenge?.label || ''); + if (!label) return []; + const normalized = detectChallengeDialog( + `dialog ${JSON.stringify(label.slice(0, 200))}`, + { allowGenericFailure: options?.allowGenericFailure === true }, + ); + if (!normalized?.normalizedLabel) return []; + return [{ + frameId: entry.frameId, + frameUrl: String(entry.payload?.frameContext?.frameUrl || ''), + label: normalized.label, + normalizedLabel: normalized.normalizedLabel, + }]; + }); const inspectedFrameIds = new Set(entries .filter(entry => entry.payload && typeof entry.payload === 'object') .map(entry => entry.frameId)); @@ -2840,9 +2891,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d return includeStatus ? { challenge, + challengeHidden: hiddenChallenges.length > 0, + hiddenChallenges, + // With no expected frame, "complete" still requires at least one + // frame to have actually been inspected. inspectionComplete: expectedFrameId === null - || inspectedFrameIds.has(expectedFrameId) - || (navigationInspectionComplete && !expectedFrameStillExists), + ? inspectedFrameIds.size > 0 + : (inspectedFrameIds.has(expectedFrameId) + || (navigationInspectionComplete && !expectedFrameStillExists)), } : challenge; }; @@ -2855,7 +2911,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d return inspected(results); } catch { return includeStatus - ? { challenge: null, inspectionComplete: false } + ? { + challenge: null, + challengeHidden: false, + hiddenChallenges: [], + inspectionComplete: false, + } : null; } } @@ -2883,6 +2944,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ? challenge.frameId : null, captchaChallengeFrameUrl: challenge.frameUrl || '', + captchaChallengeVisibilityConfirmed: true, }, {}, ); @@ -2901,6 +2963,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } const activeGate = this._captchaGateStates.get(tabId); + const treeFilter = String(toolArgs?.filter || 'all').toLowerCase(); + let observedChallengeFrameId = Number.isInteger(toolResult.captchaChallengeFrameId) + ? toolResult.captchaChallengeFrameId + : null; + let observedChallengeFrameUrl = String(toolResult.captchaChallengeFrameUrl || ''); let challenge = detectChallengeDialog(toolResult.pageContent, { allowGenericFailure: !!activeGate, }); @@ -2910,11 +2977,36 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d { allowGenericFailure: !!activeGate }, ); } + if ( + challenge + && treeFilter !== 'visible' + && toolResult.captchaChallengeVisibilityConfirmed !== true + ) { + const recheck = await this._detectChallengeDialogBeforeMutation(tabId, { + includeStatus: true, + expectedFrameId: observedChallengeFrameId, + allowGenericFailure: !!activeGate, + }); + if (recheck.challenge?.label) { + challenge = detectChallengeDialog( + `dialog ${JSON.stringify(String(recheck.challenge.label).slice(0, 200))}`, + { allowGenericFailure: !!activeGate }, + ); + observedChallengeFrameId = Number.isInteger(recheck.challenge.frameId) + ? recheck.challenge.frameId + : null; + observedChallengeFrameUrl = String(recheck.challenge.frameUrl || ''); + } + // A DOM scan cannot prove that the tree-observed dialog is the same + // element as a hidden match: a page may retain a hidden template while + // rendering the active challenge in a closed shadow root. Hidden, + // missing, and inconclusive scan results therefore remain diagnostic + // only and keep the gate fail-closed. + } let pageUrl = String(toolResult.currentUrl || toolResult.pageUrl || ''); if (!pageUrl) { try { pageUrl = await this._currentUrl(tabId); } catch {} } - const treeFilter = String(toolArgs?.filter || 'all').toLowerCase(); const requestedPage = toolArgs?.page; const requestedMaxDepth = toolArgs?.maxDepth; const parsedMaxDepth = Number(requestedMaxDepth); @@ -3112,10 +3204,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d key, status: publicGate.status, publicGate, - ...(Number.isInteger(toolResult.captchaChallengeFrameId) + ...(Number.isInteger(observedChallengeFrameId) ? { - challengeFrameId: toolResult.captchaChallengeFrameId, - challengeFrameUrl: String(toolResult.captchaChallengeFrameUrl || ''), + challengeFrameId: observedChallengeFrameId, + challengeFrameUrl: observedChallengeFrameUrl, } : {}), }); @@ -4996,7 +5088,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } } - _pageSignalsContentBehindBlank(probe) { + _pageSignalsContentBehindBlank(probe, { includeDocumentLengthSignal = true } = {}) { if (!probe) return true; const textChars = Number(probe.documentTextChars || 0); const visibleTextChars = Number(probe.visibleTextChars || 0); @@ -5010,15 +5102,26 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d visibleTextChars > 20 || imageCount > 0 || domNodes > 150 || - (innerHeight > 0 && scrollHeight > innerHeight + 200) + ( + includeDocumentLengthSignal + && innerHeight > 0 + && scrollHeight > innerHeight + 200 + ) ); } - async _retryBlankScreenshotCapture(firstShot, captureOnce, { probe = null } = {}) { + async _retryBlankScreenshotCapture( + firstShot, + captureOnce, + { probe = null, includeDocumentLengthSignal = true } = {}, + ) { if (!firstShot?.dataUrl || typeof captureOnce !== 'function') return firstShot; let shot = firstShot; let blankness = await this._analyzeScreenshotBlankness(shot.dataUrl); - if (!blankness?.blank || !this._pageSignalsContentBehindBlank(probe)) return shot; + if ( + !blankness?.blank + || !this._pageSignalsContentBehindBlank(probe, { includeDocumentLengthSignal }) + ) return shot; const meta = { detected: true, @@ -5113,7 +5216,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d try { await cdpClient.attach(tabId); await cdpClient.sendCommand(tabId, 'Page.enable'); - await this._bringToFrontForCapture(tabId); + await this._preparePageForCapture(tabId); // Probe the CSS viewport first so we can either (a) clip exactly // to it for pixel-accurate captures, or (b) compute a budget-aware @@ -5158,7 +5261,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d return { dataUrl: redacted, width: cssW, height: cssH, coordAligned: true }; }; const first = await captureOnce(); - return await this._retryBlankScreenshotCapture(first, captureOnce, { probe }); + const captured = await this._retryBlankScreenshotCapture(first, captureOnce, { probe }); + // A persistently blank background frame is worse than no image: the + // normal page-reading/AX context remains available to the planner, + // while a blank image can falsely imply that the page itself is empty. + return captured?.blankFrameRetry?.finalBlank ? null : captured; } // Non-coord-aligned mode: pre-compute target dims via the budget @@ -5193,7 +5300,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d }; }; const first = await captureOnce(); - return await this._retryBlankScreenshotCapture(first, captureOnce, { probe }); + const captured = await this._retryBlankScreenshotCapture(first, captureOnce, { probe }); + return captured?.blankFrameRetry?.finalBlank ? null : captured; } catch (e) { return null; } @@ -5446,39 +5554,47 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d try { await cdpClient.attach(tabId); await cdpClient.sendCommand(tabId, 'Page.enable'); - await this._bringToFrontForCapture(tabId); - const vp = await cdpClient.evaluate(tabId, '({w: window.innerWidth, h: window.innerHeight})'); - const cssW = Math.max(1, Math.round(vp?.result?.value?.w || 1024)); - const cssH = Math.max(1, Math.round(vp?.result?.value?.h || 768)); - const shot = await this._withIndicatorsHidden(tabId, () => - cdpClient.sendCommand(tabId, 'Page.captureScreenshot', { - format: 'png', - fromSurface: true, - clip: { x: 0, y: 0, width: cssW, height: cssH, scale: 1 }, - }) - ); - if (!shot?.data) return null; - const cropDataUrl = `data:image/png;base64,${shot.data}`; - // Model-facing path honors maxImageDimension (issue #311). Keep the raw - // `cropDataUrl` at full CSS resolution for the local download crop; - // `_locateVisibleMediaWithVision` maps vision boxes back to that space. - const shrunk = await this._shrinkImageForBudget(cropDataUrl, cssW, cssH, this._budgetForCapture()); - let dataUrl = shrunk.dataUrl; - // Local screenshot redaction (issue #312): the model-facing `dataUrl` - // is sent to vision for media localization, so pixelate PII on it. - // The raw `cropDataUrl` is kept untouched for the local download. - if (this.screenshotRedaction) { - dataUrl = await this._redactScreenshotDataUrl(tabId, dataUrl, { coordinateSpace: 'viewport' }); - } - return { - dataUrl, - cropDataUrl, - width: cssW, - height: cssH, - visionWidth: shrunk.width, - visionHeight: shrunk.height, - coordAligned: true, + await this._preparePageForCapture(tabId); + const probe = await this._captureViewportProbe(tabId); + const cssW = Math.max(1, Math.round(probe?.innerWidth || 1024)); + const cssH = Math.max(1, Math.round(probe?.innerHeight || 768)); + const captureOnce = async () => { + const shot = await this._withIndicatorsHidden(tabId, () => + cdpClient.sendCommand(tabId, 'Page.captureScreenshot', { + format: 'png', + fromSurface: true, + clip: { x: 0, y: 0, width: cssW, height: cssH, scale: 1 }, + }) + ); + if (!shot?.data) return null; + const cropDataUrl = `data:image/png;base64,${shot.data}`; + // Model-facing path honors maxImageDimension (issue #311). Keep the raw + // `cropDataUrl` at full CSS resolution for the local download crop; + // `_locateVisibleMediaWithVision` maps vision boxes back to that space. + const shrunk = await this._shrinkImageForBudget(cropDataUrl, cssW, cssH, this._budgetForCapture()); + let dataUrl = shrunk.dataUrl; + // Local screenshot redaction (issue #312): the model-facing `dataUrl` + // is sent to vision for media localization, so pixelate PII on it. + // The raw `cropDataUrl` is kept untouched for the local download. + if (this.screenshotRedaction) { + dataUrl = await this._redactScreenshotDataUrl(tabId, dataUrl, { coordinateSpace: 'viewport' }); + } + return { + dataUrl, + cropDataUrl, + width: cssW, + height: cssH, + visionWidth: shrunk.width, + visionHeight: shrunk.height, + coordAligned: true, + }; }; + const captured = await this._retryBlankScreenshotCapture( + await captureOnce(), + captureOnce, + { probe }, + ); + return captured?.blankFrameRetry?.finalBlank ? null : captured; } catch (_) { const fallback = await this._captureAutoScreenshot(tabId, { coordAligned: true }); return fallback ? { ...fallback, cropDataUrl: fallback.dataUrl } : null; @@ -5721,15 +5837,58 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } /** - * Try to bring a tab to the front via CDP before capturing. The surface - * screenshot path (`fromSurface: true`) can produce stale/blank frames if - * the page is occluded or backgrounded in some compositor states. This is - * best-effort — ignore failures and let the capture proceed. + * Prepare a page for capture without changing the user's selected tab. + * + * Focus emulation makes background pages report themselves as focused and + * active without activating their tab. It is scoped to the current run and + * disabled in the run's finally block because debugger sessions persist. + * Cloud runs and /foreground deliberately retain the old activation path. */ - async _bringToFrontForCapture(tabId) { + async _preparePageForCapture(tabId) { + if (this._foregroundCaptureTabs.has(tabId)) { + try { + await cdpClient.sendCommand(tabId, 'Page.bringToFront'); + return { foreground: true, focusEmulated: false }; + } catch { + return { foreground: false, focusEmulated: false }; + } + } + if (!this._runningTabs.has(tabId)) { + return { foreground: false, focusEmulated: false }; + } try { - await cdpClient.sendCommand(tabId, 'Page.bringToFront'); - } catch { /* ignore */ } + await cdpClient.sendCommand(tabId, 'Emulation.setFocusEmulationEnabled', { enabled: true }); + this._focusEmulatedTabs.add(tabId); + return { foreground: false, focusEmulated: true }; + } catch { + // Older Chromium builds may not expose the experimental command. + // Background capture can still succeed, so do not activate the tab. + return { foreground: false, focusEmulated: false }; + } + } + + async _clearBackgroundFocusEmulation(tabId) { + if (!this._focusEmulatedTabs.has(tabId)) return; + this._focusEmulatedTabs.delete(tabId); + try { + await cdpClient.sendCommand(tabId, 'Emulation.setFocusEmulationEnabled', { enabled: false }); + } catch { /* detached/closed tabs need no cleanup */ } + } + + _configureCapturePolicyForRun(tabId, runOptions = {}) { + const previousForeground = this._foregroundCaptureTabs.has(tabId); + if (runOptions?.cloudRun === true || runOptions?.foreground === true) { + this._foregroundCaptureTabs.add(tabId); + } else { + this._foregroundCaptureTabs.delete(tabId); + } + return previousForeground; + } + + async _restoreCapturePolicyAfterRun(tabId, previousForeground) { + await this._clearBackgroundFocusEmulation(tabId); + if (previousForeground) this._foregroundCaptureTabs.add(tabId); + else this._foregroundCaptureTabs.delete(tabId); } // ───────────── Image-budget (token-conscious screenshots) ───────────── @@ -9444,6 +9603,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d this.lastAutoScreenshotTs.delete(tabId); this.autoScreenshotCount.delete(tabId); this.screenshotClickScale.delete(tabId); + void this._clearBackgroundFocusEmulation(tabId); + this._foregroundCaptureTabs.delete(tabId); this.lastSeenAdapter.delete(tabId); this.activeSkillIds.delete(tabId); this._runModeOverrides.delete(tabId); @@ -13392,6 +13553,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d this.conversationModes.set(tabId, 'act'); const completionRunToken = this._beginCompletionInvariant(tabId); this._runningTabs.add(tabId); + const previousForegroundCapture = this._configureCapturePolicyForRun(tabId, runOptions); const startUrl = await this._currentUrl(tabId); const conversationId = await this.ensureConversationId(tabId, 'act'); const traceRunId = await trace.startRun({ @@ -13582,6 +13744,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d this.currentCostState.delete(tabId); this._planExecutionGuards.delete(tabId); this._resetActiveSkillsForRun(tabId); + await this._restoreCapturePolicyAfterRun(tabId, previousForegroundCapture); this._runningTabs.delete(tabId); this._clearRunLoopState(tabId); this._clickAxCdpFallbacks?.delete(tabId); @@ -14144,7 +14307,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d await cdpClient.attach(tabId); await cdpClient.sendCommand(tabId, 'Page.enable'); probe = await this._captureViewportProbe(tabId); - await this._bringToFrontForCapture(tabId); + await this._preparePageForCapture(tabId); coordAligned = !!(args && args.coord_aligned); const cssW = Math.max(1, Math.round(probe?.innerWidth || 1024)); const cssH = Math.max(1, Math.round(probe?.innerHeight || 768)); @@ -14239,12 +14402,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d description = captured.description || ''; coordDownscaled = !!captured.coordDownscaled; blankFrameRetry = captured.blankFrameRetry || null; + if (captured?.blankFrameRetry?.finalBlank) { + throw new Error('Background screenshot remained blank after retries'); + } } catch { const tab = await chrome.tabs.get(tabId); if (!tab?.active) { return { success: false, - error: 'Cannot capture screenshot: this tab is not the active tab in its window. Switch to the tab before using /screenshot, or use a page-reading tool.', + error: 'Background screenshot capture was unavailable. Continue with get_accessibility_tree, get_interactive_elements, or read_page, or rerun the task with /foreground for visual compatibility.', }; } // Tabs API fallback: no clip/scale available. Capture full, then @@ -14303,6 +14469,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d coordDownscaled = !!captured?.coordDownscaled; blankFrameRetry = captured?.blankFrameRetry || null; } + if (blankFrameRetry?.finalBlank) { + return { + success: false, + error: 'Screenshot remained blank after retries. Continue with page-reading tools or rerun the task with /foreground for visual compatibility.', + }; + } if (!dataUrl) { return { success: false, @@ -14449,29 +14621,41 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d let imageDataUrl = null; let annotatedRect = null; if (plannerCanSeeImages) { - await this._bringToFrontForCapture(tabId); - const shot = await this._withIndicatorsHidden(tabId, () => - cdpClient.sendCommand(tabId, 'Page.captureScreenshot', { - format: 'png', quality: 80, fromSurface: true, - }) - ); - imageDataUrl = `data:image/png;base64,${shot.data}`; - // Local screenshot redaction (issue #312): pixelate form fields + - // email/phone text BEFORE this verification screenshot reaches the - // planner model. Runs before the optional interaction-rect outline. - if (this.screenshotRedaction) { - imageDataUrl = await this._redactScreenshotDataUrl(tabId, imageDataUrl, { coordinateSpace: 'viewport' }); - } - // If we remember the rect of the last ax interaction on this tab, - // outline it on the screenshot so the model can anchor its review - // to the element it actually touched. - const last = this._lastInteractionRect.get(tabId); - if (last) { - const cssViewport = probe - ? { width: probe.innerWidth, height: probe.innerHeight } + await this._preparePageForCapture(tabId); + const captureOnce = async () => { + const shot = await this._withIndicatorsHidden(tabId, () => + cdpClient.sendCommand(tabId, 'Page.captureScreenshot', { + format: 'png', quality: 80, fromSurface: true, + }) + ); + return shot?.data + ? { dataUrl: `data:image/png;base64,${shot.data}` } : null; - imageDataUrl = await this._annotateScreenshot(imageDataUrl, last, cssViewport); - annotatedRect = { x: last.x, y: last.y, w: last.w, h: last.h }; + }; + const captured = await this._retryBlankScreenshotCapture( + await captureOnce(), + captureOnce, + { probe }, + ); + if (captured?.dataUrl && !captured?.blankFrameRetry?.finalBlank) { + imageDataUrl = captured.dataUrl; + // Local screenshot redaction (issue #312): pixelate form fields + + // email/phone text BEFORE this verification screenshot reaches the + // planner model. Runs before the optional interaction-rect outline. + if (this.screenshotRedaction) { + imageDataUrl = await this._redactScreenshotDataUrl(tabId, imageDataUrl, { coordinateSpace: 'viewport' }); + } + // If we remember the rect of the last ax interaction on this tab, + // outline it on the screenshot so the model can anchor its review + // to the element it actually touched. + const last = this._lastInteractionRect.get(tabId); + if (last) { + const cssViewport = probe + ? { width: probe.innerWidth, height: probe.innerHeight } + : null; + imageDataUrl = await this._annotateScreenshot(imageDataUrl, last, cssViewport); + annotatedRect = { x: last.x, y: last.y, w: last.w, h: last.h }; + } } } @@ -14931,15 +15115,16 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d try { const capturePolicy = await this._getFullPageCapturePolicy(tabId); await cdpClient.attach(tabId); - await this._bringToFrontForCapture(tabId); - const capture = await this._withIndicatorsHidden(tabId, () => - cdpClient.captureFullPageScreenshot(tabId, capturePolicy) - ); - const imageData = typeof capture === 'string' ? capture : capture?.data; + await this._preparePageForCapture(tabId); + const captured = await this._captureFullPageWithBlankRetry(tabId, capturePolicy); + const capture = captured?.capture; const captureWarning = typeof capture === 'object' ? capture?.warning || null : null; const captureBounds = typeof capture === 'object' ? capture?.captureBounds || null : null; - if (!imageData) throw new Error('Full-page screenshot returned no image data'); - const rawUrl = `data:image/png;base64,${imageData}`; + if (!captured?.dataUrl) throw new Error('Full-page screenshot returned no image data'); + if (captured?.blankFrameRetry?.finalBlank) { + throw new Error('Background full-page screenshot remained blank after retries'); + } + const rawUrl = captured.dataUrl; const warningNote = captureWarning ? `\nWarning: ${captureWarning}` : ''; // If the caller asked to save, do it with the RAW (uncompressed, @@ -15110,18 +15295,32 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // 2. Capture screenshot try { await cdpClient.sendCommand(tabId, 'Page.enable'); - await this._bringToFrontForCapture(tabId); - const shot = await this._withIndicatorsHidden(tabId, () => - cdpClient.sendCommand(tabId, 'Page.captureScreenshot', { - format: 'png', quality: 100, fromSurface: true, - }) + await this._preparePageForCapture(tabId); + const probe = await this._captureViewportProbe(tabId); + const captureOnce = async () => { + const shot = await this._withIndicatorsHidden(tabId, () => + cdpClient.sendCommand(tabId, 'Page.captureScreenshot', { + format: 'png', quality: 100, fromSurface: true, + }) + ); + return shot?.data + ? { dataUrl: `data:image/png;base64,${shot.data}` } + : null; + }; + const captured = await this._retryBlankScreenshotCapture( + await captureOnce(), + captureOnce, + { probe }, ); + if (!captured?.dataUrl || captured?.blankFrameRetry?.finalBlank) { + throw new Error('Screenshot was blank or unavailable'); + } // Route the screenshot through `_attachImage` (like the `screenshot` // tool) so the batch loop strips it and re-attaches it as an // image_url block. Left inline as `result.image`, the base64 blob // blows past the tool-result char cap and gets truncated to garbage // that the vision model can never read. - let verifyShotUrl = `data:image/png;base64,${shot.data}`; + let verifyShotUrl = captured.dataUrl; // Apply the user's image budget (issue #311) before the image // reaches the model, then run local redaction (issue #312) on the // budget-fit copy so PII is pixelated at the size the model sees. @@ -18678,6 +18877,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d this._clickAxCdpFallbacks.delete(tabId); const completionRunToken = this._beginCompletionInvariant(tabId); this._runningTabs.add(tabId); + const previousForegroundCapture = this._configureCapturePolicyForRun(tabId, runOptions); this._runModeOverrides.set(tabId, mode); const previousCloudContext = this.cloudRunContexts.get(tabId); if (runOptions.cloudRun) { @@ -18696,6 +18896,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (previousCloudContext) this.cloudRunContexts.set(tabId, previousCloudContext); else this.cloudRunContexts.delete(tabId); } + await this._restoreCapturePolicyAfterRun(tabId, previousForegroundCapture); this._runningTabs.delete(tabId); this._clearRunLoopState(tabId); this._clickAxCdpFallbacks.delete(tabId); @@ -19510,6 +19711,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d this._clickAxCdpFallbacks.delete(tabId); const completionRunToken = this._beginCompletionInvariant(tabId); this._runningTabs.add(tabId); + const previousForegroundCapture = this._configureCapturePolicyForRun(tabId, runOptions); this._runModeOverrides.set(tabId, mode); const previousCloudContext = this.cloudRunContexts.get(tabId); if (runOptions.cloudRun) { @@ -19528,6 +19730,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (previousCloudContext) this.cloudRunContexts.set(tabId, previousCloudContext); else this.cloudRunContexts.delete(tabId); } + await this._restoreCapturePolicyAfterRun(tabId, previousForegroundCapture); this._runningTabs.delete(tabId); this._clearRunLoopState(tabId); this._clickAxCdpFallbacks.delete(tabId); diff --git a/src/chrome/src/agent/captcha-frame-runtime.js b/src/chrome/src/agent/captcha-frame-runtime.js index 63f0fa6ac..d73743a85 100644 --- a/src/chrome/src/agent/captcha-frame-runtime.js +++ b/src/chrome/src/agent/captcha-frame-runtime.js @@ -630,11 +630,22 @@ export function detectCaptchaCandidatesInPage(scope = null) { const visibleElement = (element) => { if (!element) return false; try { - const style = typeof pageWindow?.getComputedStyle === 'function' + const elementStyle = typeof pageWindow?.getComputedStyle === 'function' ? pageWindow.getComputedStyle(element) : null; - if (style && (style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0)) return false; - if (element.hidden || element.getAttribute?.('aria-hidden') === 'true') return false; + if ( + elementStyle + && (elementStyle.visibility === 'hidden' || elementStyle.visibility === 'collapse') + ) return false; + let current = element; + for (let depth = 0; current && depth < 30; depth += 1) { + const style = typeof pageWindow?.getComputedStyle === 'function' + ? pageWindow.getComputedStyle(current) + : null; + if (style && (style.display === 'none' || Number(style.opacity) === 0)) return false; + if (current.hidden || current.getAttribute?.('aria-hidden') === 'true') return false; + current = current.parentElement || null; + } if (typeof element.getBoundingClientRect === 'function') { const rect = element.getBoundingClientRect(); if (!rect || rect.width <= 0 || rect.height <= 0) return false; @@ -660,13 +671,21 @@ export function detectCaptchaCandidatesInPage(scope = null) { .map(id => pageDocument.getElementById?.(id)?.textContent || '') .join(' '); } catch (_) {} + let renderedHeadings = ''; + try { + renderedHeadings = Array.from( + element.querySelectorAll?.('h1, h2, h3, [role="heading"]') || [] + ) + .filter(visibleElement) + .map(heading => heading.textContent || '') + .join(' '); + } catch (_) {} return [ element.getAttribute?.('aria-label'), labelledBy, - element.querySelector?.('h1, h2, h3, [role="heading"]')?.textContent, + renderedHeadings, element.getAttribute?.('title'), element.innerText, - element.textContent, ].some(value => challengeDialogRe.test(String(value || ''))); }); const elementInChallengeDialog = (element) => { diff --git a/src/chrome/src/agent/captcha-gate.js b/src/chrome/src/agent/captcha-gate.js index 4cffd9110..d2e2cfa3f 100644 --- a/src/chrome/src/agent/captcha-gate.js +++ b/src/chrome/src/agent/captcha-gate.js @@ -107,9 +107,17 @@ export function detectChallengeDialogInPage(options = null) { }; const visible = (element) => { try { - const style = getComputedStyle(element); - if (style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0) return false; - if (element.hidden || element.getAttribute?.('aria-hidden') === 'true') return false; + const elementStyle = getComputedStyle(element); + if (elementStyle.visibility === 'hidden' || elementStyle.visibility === 'collapse') return false; + let current = element; + for (let depth = 0; current && depth < 30; depth += 1) { + const style = getComputedStyle(current); + if (style.display === 'none' || Number(style.opacity) === 0) return false; + if (current.hidden || current.getAttribute?.('aria-hidden') === 'true') return false; + current = current.parentElement + || (typeof current.getRootNode === 'function' ? current.getRootNode()?.host : null) + || null; + } const rect = element.getBoundingClientRect(); if (rect.width <= 0 || rect.height <= 0) return false; const viewportWidth = typeof window !== 'undefined' && typeof window.innerWidth === 'number' @@ -139,9 +147,16 @@ export function detectChallengeDialogInPage(options = null) { visible: visible(element), }; }); + // A challenge dialog that exists in the DOM but is hidden or off-viewport + // is positive evidence the challenge is inactive. Report it separately so + // callers can distinguish "confirmed hidden" from "not found" — a dialog + // this scan cannot see at all (closed shadow root, unreachable frame) must + // not be treated as disproved. + let hiddenChallenge = null; const finish = challenge => includeFrameContext ? { challenge, + ...(hiddenChallenge ? { hiddenChallenge } : {}), frameContext: { frameUrl, frameName, @@ -149,23 +164,80 @@ export function detectChallengeDialogInPage(options = null) { }, } : challenge; - for (const element of document.querySelectorAll('dialog, [role="dialog"], [role="alertdialog"]')) { - if (!visible(element)) continue; + // The accessibility tree pierces open shadow roots on some sites, where + // frameworks (e.g. LinkedIn's interop shell) render whole dialogs that + // document.querySelectorAll cannot see. Scan open shadow roots too so this + // preflight can confirm the same dialogs the tree reports. + const dialogSelector = 'dialog, [role="dialog"], [role="alertdialog"]'; + const dialogCandidates = []; + const collectDialogs = (root, depth) => { + try { + for (const element of root.querySelectorAll(dialogSelector)) { + if (dialogCandidates.length >= 200) break; + dialogCandidates.push(element); + } + } catch {} + if (depth >= 4) return; + let descendants; + try { descendants = root.querySelectorAll('*'); } catch { return; } + for (const host of descendants) { + if (host.shadowRoot) collectDialogs(host.shadowRoot, depth + 1); + } + }; + collectDialogs(document, 0); + const recordHiddenChallenge = (element) => { + if (hiddenChallenge) return; + try { + const values = [ + element.getAttribute?.('aria-label'), + element.getAttribute?.('title'), + element.innerText, + element.textContent, + ...Array.from(element.querySelectorAll?.('h1, h2, h3, [role="heading"]') || []) + .map(heading => heading.textContent || ''), + ]; + for (const value of values) { + const text = String(value || ''); + if (!matchesChallenge(text)) continue; + const line = text.split(/\r?\n/).find(entry => matchesChallenge(entry)) || text; + const label = line.replace(/\s+/g, ' ').trim().slice(0, 200); + if (label) { + hiddenChallenge = { label }; + return; + } + } + } catch {} + }; + for (const element of dialogCandidates) { + if (!visible(element)) { + recordHiddenChallenge(element); + continue; + } let labelledBy = ''; try { + const idRoot = (typeof element.getRootNode === 'function' && element.getRootNode()) || document; labelledBy = String(element.getAttribute('aria-labelledby') || '') .split(/\s+/) .filter(Boolean) - .map(id => document.getElementById(id)?.textContent || '') + .map(id => (typeof idRoot.getElementById === 'function' ? idRoot : document) + .getElementById(id)?.textContent || '') + .join(' '); + } catch {} + let renderedHeadings = ''; + try { + renderedHeadings = Array.from( + element.querySelectorAll?.('h1, h2, h3, [role="heading"]') || [] + ) + .filter(visible) + .map(heading => heading.textContent || '') .join(' '); } catch {} const values = [ element.getAttribute?.('aria-label'), labelledBy, - element.querySelector?.('h1, h2, h3, [role="heading"]')?.textContent, + renderedHeadings, element.getAttribute?.('title'), element.innerText, - element.textContent, ]; for (const value of values) { const text = String(value || ''); diff --git a/src/chrome/src/background.js b/src/chrome/src/background.js index 50e02effc..425591c1d 100644 --- a/src/chrome/src/background.js +++ b/src/chrome/src/background.js @@ -1445,6 +1445,14 @@ function assertNoActiveTabRun(tabId) { } } +async function activateForegroundCompatibilityTab(tabId) { + const tab = await chrome.tabs.get(tabId); + await chrome.tabs.update(tabId, { active: true }); + if (tab?.windowId != null) { + await chrome.windows.update(tab.windowId, { focused: true }); + } +} + const detachedRunStarts = new Map(); const detachedRunFailures = new Map(); const RUN_KEEPALIVE_INTERVAL_MS = 20_000; @@ -2146,7 +2154,11 @@ async function handleMessage(msg, sender) { assertRunCanStart(tabId, msg); const isWorkflowRun = !!msg.workflowId; const mode = isWorkflowRun ? 'act' : (msg.mode || 'ask'); - const runUi = await beginContinuationRunUiSnapshot(tabId, msg.requestId, { mode, kind: 'chat' }); + const runUi = await beginContinuationRunUiSnapshot(tabId, msg.requestId, { + mode, + kind: 'chat', + foreground: msg.foreground === true, + }); const releaseRunKeepalive = acquireRunKeepalive(); // /allow-api flag is per-conversation. The sidebar tracks it locally @@ -2165,6 +2177,8 @@ async function handleMessage(msg, sender) { let result = ''; let runError = null; try { + if (msg.foreground) await activateForegroundCompatibilityTab(tabId); + // Capture belongs to the background run lifecycle so it survives the // side panel closing or reloading while the agent is still working. runCaptureState = await runCaptureController.start(msg.runCapture, tabId); @@ -2185,6 +2199,7 @@ async function handleMessage(msg, sender) { const runOptions = { ...(isWorkflowRun ? { independentRun: true } : {}), ...(msg.recommendedAction ? { recommendedAction: msg.recommendedAction } : {}), + ...(msg.foreground ? { foreground: true } : {}), ...(msg.sourceGrounding === SELECTION_ONLY_SOURCE_GROUNDING ? { sourceGrounding: SELECTION_ONLY_SOURCE_GROUNDING } : {}), @@ -2280,7 +2295,11 @@ async function handleMessage(msg, sender) { if (!tabId) throw new Error('No tab ID'); assertNoActiveTabRun(tabId); const mode = msg.mode || 'ask'; - const runUi = beginRunUiSnapshot(tabId, msg.requestId); + const runUi = beginRunUiSnapshot(tabId, msg.requestId, { + mode, + kind: 'chat', + foreground: msg.foreground === true, + }); const releaseRunKeepalive = acquireRunKeepalive(); if (msg.apiMutationsAllowed) agent.setApiMutationsAllowed(tabId, true); @@ -2291,8 +2310,11 @@ async function handleMessage(msg, sender) { let result = ''; let runError = null; try { + if (msg.foreground) await activateForegroundCompatibilityTab(tabId); + const runOptions = { ...(msg.recommendedAction ? { recommendedAction: msg.recommendedAction } : {}), + ...(msg.foreground ? { foreground: true } : {}), ...(msg.sourceGrounding === SELECTION_ONLY_SOURCE_GROUNDING ? { sourceGrounding: SELECTION_ONLY_SOURCE_GROUNDING } : {}), @@ -2336,7 +2358,11 @@ async function handleMessage(msg, sender) { if (!tabId) throw new Error('No tab ID'); assertRunCanStart(tabId, msg); const mode = msg.mode || 'ask'; - const runUi = await beginContinuationRunUiSnapshot(tabId, msg.requestId, { mode, kind: 'continue' }); + const runUi = await beginContinuationRunUiSnapshot(tabId, msg.requestId, { + mode, + kind: 'continue', + foreground: msg.foreground === true, + }); const releaseRunKeepalive = acquireRunKeepalive(); sendIndicatorMessage(tabId, 'WB_SHOW_AGENT_INDICATORS'); @@ -2345,10 +2371,13 @@ async function handleMessage(msg, sender) { let result = ''; let runError = null; try { + if (msg.foreground) await activateForegroundCompatibilityTab(tabId); + result = await agent.continueProcessing(tabId, (type, data) => { updates.push({ type, data }); sendAgentUpdate(tabId, runUi.requestId, type, data); }, mode, { + ...(msg.foreground ? { foreground: true } : {}), detachedRequestId: runUi.requestId, isDetachedStartCancelled: () => isDetachedRunStartCancelled(tabId, msg), beforeConsequentialTool: () => flushRunUiSnapshot(tabId, runUi.requestId), diff --git a/src/chrome/src/run-ui-journal.js b/src/chrome/src/run-ui-journal.js index 2be6e5b90..57a0430fa 100644 --- a/src/chrome/src/run-ui-journal.js +++ b/src/chrome/src/run-ui-journal.js @@ -116,6 +116,7 @@ export class RunUiJournal { requestId: createRunRequestId(tabId, requestId), mode: String(metadata?.mode || ''), kind: metadata?.kind === 'continue' ? 'continue' : 'chat', + foreground: metadata?.foreground === true, runId: null, status: 'running', seq: 0, @@ -144,6 +145,7 @@ export class RunUiJournal { const snapshot = this.snapshots.get(tabId); if (!snapshot || String(snapshot.requestId) !== String(requestId)) return null; if (metadata?.mode) snapshot.mode = String(metadata.mode); + if (typeof metadata?.foreground === 'boolean') snapshot.foreground = metadata.foreground; if (!snapshot.kind && metadata?.kind) { snapshot.kind = metadata.kind === 'continue' ? 'continue' : 'chat'; } @@ -273,6 +275,7 @@ export class RunUiJournal { } if (typeof snapshot.mode !== 'string') snapshot.mode = ''; if (snapshot.kind !== 'continue' && snapshot.kind !== 'chat') snapshot.kind = 'chat'; + if (snapshot.foreground !== true) snapshot.foreground = false; if (typeof snapshot.streamedText !== 'string') snapshot.streamedText = ''; if (snapshot.streamedText.length > RUN_UI_STREAM_TEXT_LIMIT) { snapshot.streamedText = ''; diff --git a/src/chrome/src/ui/locales/ar.js b/src/chrome/src/ui/locales/ar.js index 334e8bc5f..a6d526d8e 100644 --- a/src/chrome/src/ui/locales/ar.js +++ b/src/chrome/src/ui/locales/ar.js @@ -576,6 +576,7 @@ export default { 'sp.slash.show_scratchpad': 'عرض لوحة المسودة الحالية', 'sp.slash.clear_scratchpad': 'مسح منطقة المسودة الحالية', 'sp.slash.allow_api': 'السماح بتعديلات API لهذه المحادثة', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'تعطيل مطالبات الأذونات عالميًا', 'sp.slash.compact': 'ضغط سياق هذه المحادثة', 'sp.slash.verbose': 'تبديل عرض الأدوات بين مفصل ومضغوط', diff --git a/src/chrome/src/ui/locales/bn.js b/src/chrome/src/ui/locales/bn.js index 8149ba666..a3e922eb5 100644 --- a/src/chrome/src/ui/locales/bn.js +++ b/src/chrome/src/ui/locales/bn.js @@ -129,6 +129,7 @@ export default { 'sp.slash.save_workflow': "সর্বশেষ সফল রেকর্ড করা রান সংরক্ষণ করুন", 'sp.slash.delete_workflow': "আইডি দ্বারা একটি সংরক্ষিত ওয়ার্কফ্লো মুছুন", 'sp.slash.allow_api': "এই কথোপকথনের জন্য API মিউটেশনের অনুমতি দিন", + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': "বিশ্বব্যাপী অনুমতি প্রম্পট অক্ষম করুন", 'sp.slash.compact': "এই কথোপকথন প্রসঙ্গে কম্প্যাক্ট", 'sp.slash.verbose': "ভার্বোজ বা কমপ্যাক্ট টুল ডিসপ্লে টগল করুন", diff --git a/src/chrome/src/ui/locales/de.js b/src/chrome/src/ui/locales/de.js index bb16a9e02..05863b5a9 100644 --- a/src/chrome/src/ui/locales/de.js +++ b/src/chrome/src/ui/locales/de.js @@ -122,6 +122,7 @@ export default { 'sp.slash.save_workflow': 'Letzten erfolgreichen aufgezeichneten Durchlauf speichern', 'sp.slash.delete_workflow': 'Gespeicherten Workflow nach ID löschen', 'sp.slash.allow_api': 'API-Mutationen für dieses Gespräch erlauben', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Berechtigungsabfragen global deaktivieren', 'sp.slash.compact': 'Diesen Gesprächskontext kompaktieren', 'sp.slash.verbose': 'Ausführliche oder kompakte Tool-Anzeige umschalten', diff --git a/src/chrome/src/ui/locales/en.js b/src/chrome/src/ui/locales/en.js index ad50400d7..af972a960 100644 --- a/src/chrome/src/ui/locales/en.js +++ b/src/chrome/src/ui/locales/en.js @@ -129,6 +129,7 @@ export default { 'sp.slash.save_workflow': 'Save the latest successful recorded run', 'sp.slash.delete_workflow': 'Delete a saved workflow by ID', 'sp.slash.allow_api': 'Allow API mutations for this conversation', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Disable permission prompts globally', 'sp.slash.compact': 'Compact this conversation context', 'sp.slash.verbose': 'Toggle verbose or compact tool display', diff --git a/src/chrome/src/ui/locales/es.js b/src/chrome/src/ui/locales/es.js index 79e727017..492c8ff29 100644 --- a/src/chrome/src/ui/locales/es.js +++ b/src/chrome/src/ui/locales/es.js @@ -576,6 +576,7 @@ export default { 'sp.slash.show_scratchpad': 'Mostrar el bloc de notas actual', 'sp.slash.clear_scratchpad': 'Borrar el bloc de notas actual', 'sp.slash.allow_api': 'Permitir mutaciones de API en esta conversación', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Desactivar las solicitudes de permiso globalmente', 'sp.slash.compact': 'Compactar el contexto de esta conversación', 'sp.slash.verbose': 'Alternar entre vista de herramientas detallada o compacta', diff --git a/src/chrome/src/ui/locales/fa.js b/src/chrome/src/ui/locales/fa.js index 7dfb6c111..598e48220 100644 --- a/src/chrome/src/ui/locales/fa.js +++ b/src/chrome/src/ui/locales/fa.js @@ -129,6 +129,7 @@ export default { 'sp.slash.save_workflow': "آخرین اجرای موفق ثبت شده را ذخیره کنید", 'sp.slash.delete_workflow': "یک گردش کار ذخیره شده توسط ID را حذف کنید", 'sp.slash.allow_api': "جهش های API را برای این مکالمه مجاز کنید", + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': "درخواست های مجوز را در سطح جهانی غیرفعال کنید", 'sp.slash.compact': "این زمینه مکالمه را فشرده کنید", 'sp.slash.verbose': "نمایش ابزار فشرده یا پرمخاطب را تغییر دهید", diff --git a/src/chrome/src/ui/locales/fr.js b/src/chrome/src/ui/locales/fr.js index b138e1613..060ce2561 100644 --- a/src/chrome/src/ui/locales/fr.js +++ b/src/chrome/src/ui/locales/fr.js @@ -576,6 +576,7 @@ export default { 'sp.slash.show_scratchpad': 'Afficher le bloc-notes actuel', 'sp.slash.clear_scratchpad': 'Effacer le bloc-notes actuel', 'sp.slash.allow_api': 'Autoriser les mutations API pour cette conversation', + 'sp.slash.foreground': 'Exécuter cette tâche au premier plan pour assurer la compatibilité visuelle', 'sp.slash.dangerously_skip_permissions': 'Désactiver les demandes d\'autorisation globalement', 'sp.slash.compact': 'Compacter le contexte de cette conversation', 'sp.slash.verbose': "Basculer l'affichage des outils entre détaillé et compact", diff --git a/src/chrome/src/ui/locales/he.js b/src/chrome/src/ui/locales/he.js index c58917f28..3ede94fba 100644 --- a/src/chrome/src/ui/locales/he.js +++ b/src/chrome/src/ui/locales/he.js @@ -117,6 +117,7 @@ export default { "sp.slash.show_memory": "הצג זיכרון משתמש שמור", "sp.slash.forget_memory": "שכח זיכרון שמור לפי מזהה", "sp.slash.allow_api": "אפשר שינויי API בשיחה הזו", + "sp.slash.foreground": "Run this task in the foreground for visual compatibility", "sp.slash.dangerously_skip_permissions": "השבת בקשות הרשאה באופן גלובלי", "sp.slash.compact": "דחוס את הקשר השיחה הזה", "sp.slash.verbose": "החלף את תצוגת הכלים המילולית או הקומפקטית", diff --git a/src/chrome/src/ui/locales/hi.js b/src/chrome/src/ui/locales/hi.js index bd07b162f..18e5ed7af 100644 --- a/src/chrome/src/ui/locales/hi.js +++ b/src/chrome/src/ui/locales/hi.js @@ -129,6 +129,7 @@ export default { 'sp.slash.save_workflow': "नवीनतम सफल रिकॉर्ड किए गए रन को सहेजें", 'sp.slash.delete_workflow': "आईडी के आधार पर सहेजे गए वर्कफ़्लो को हटाएँ", 'sp.slash.allow_api': "इस वार्तालाप के लिए एपीआई म्यूटेशन की अनुमति दें", + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': "विश्व स्तर पर अनुमति संकेतों को अक्षम करें", 'sp.slash.compact': "इस वार्तालाप प्रसंग को संक्षिप्त करें", 'sp.slash.verbose': "वर्बोज़ या कॉम्पैक्ट टूल डिस्प्ले को टॉगल करें", diff --git a/src/chrome/src/ui/locales/id.js b/src/chrome/src/ui/locales/id.js index 9aacfff99..71779e8e9 100644 --- a/src/chrome/src/ui/locales/id.js +++ b/src/chrome/src/ui/locales/id.js @@ -576,6 +576,7 @@ export default { 'sp.slash.show_scratchpad': 'Tampilkan catatan saat ini', 'sp.slash.clear_scratchpad': 'Kosongkan bak pasir saat ini', 'sp.slash.allow_api': 'Izinkan mutasi API untuk percakapan ini', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Nonaktifkan permintaan izin secara global', 'sp.slash.compact': 'Kompak konteks percakapan ini', 'sp.slash.verbose': 'Beralih tampilan alat antara verbose dan kompak', diff --git a/src/chrome/src/ui/locales/ja.js b/src/chrome/src/ui/locales/ja.js index b7f647010..ac7ef5502 100644 --- a/src/chrome/src/ui/locales/ja.js +++ b/src/chrome/src/ui/locales/ja.js @@ -576,6 +576,7 @@ export default { 'sp.slash.show_scratchpad': '現在のスクラッチパッドを表示', 'sp.slash.clear_scratchpad': '現在のスクラッチパッドをクリア', 'sp.slash.allow_api': 'この会話でAPIの変更を許可', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': '権限の確認をグローバルに無効化', 'sp.slash.compact': 'この会話のコンテキストをコンパクト化', 'sp.slash.verbose': 'ツール表示を詳細とコンパクトで切り替え', diff --git a/src/chrome/src/ui/locales/ko.js b/src/chrome/src/ui/locales/ko.js index 79a7c0bb1..70dce358e 100644 --- a/src/chrome/src/ui/locales/ko.js +++ b/src/chrome/src/ui/locales/ko.js @@ -576,6 +576,7 @@ export default { 'sp.slash.show_scratchpad': '현재 스크래치패드 표시', 'sp.slash.clear_scratchpad': '현재 스크래치패드 지우기', 'sp.slash.allow_api': '이 대화에서 API 변경 허용', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': '권한 확인을 전역적으로 비활성화', 'sp.slash.compact': '이 대화 컨텍스트 압축', 'sp.slash.verbose': '도구 표시를 상세 또는 압축으로 전환', diff --git a/src/chrome/src/ui/locales/ms.js b/src/chrome/src/ui/locales/ms.js index 9be3a0f46..dfba09ed2 100644 --- a/src/chrome/src/ui/locales/ms.js +++ b/src/chrome/src/ui/locales/ms.js @@ -576,6 +576,7 @@ export default { 'sp.slash.show_scratchpad': 'Tunjukkan pad nota semasa', 'sp.slash.clear_scratchpad': 'Kosongkan pad nota semasa', 'sp.slash.allow_api': 'Benarkan mutasi API untuk perbualan ini', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Nyahdaya gesaan kebenaran secara global', 'sp.slash.compact': 'Padatkan konteks perbualan ini', 'sp.slash.verbose': 'Togol paparan alat antara terperinci dan padat', diff --git a/src/chrome/src/ui/locales/nl.js b/src/chrome/src/ui/locales/nl.js index 356323c7a..fa2306f57 100644 --- a/src/chrome/src/ui/locales/nl.js +++ b/src/chrome/src/ui/locales/nl.js @@ -125,6 +125,7 @@ export default { 'sp.slash.save_workflow': 'Sla de laatste succesvolle opgenomen uitvoering op', 'sp.slash.delete_workflow': 'Verwijder een opgeslagen workflow op ID', 'sp.slash.allow_api': 'Sta API-wijzigingen toe voor dit gesprek', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Schakel machtigingsvragen globaal uit', 'sp.slash.compact': 'Compact deze gesprekscontext', 'sp.slash.verbose': 'Schakel uitvoerige of compacte toolweergave', diff --git a/src/chrome/src/ui/locales/pl.js b/src/chrome/src/ui/locales/pl.js index ae26c26ee..15e2445ff 100644 --- a/src/chrome/src/ui/locales/pl.js +++ b/src/chrome/src/ui/locales/pl.js @@ -87,6 +87,7 @@ export default { 'sp.slash.show_scratchpad': 'Pokaż bieżący notatnik', 'sp.slash.clear_scratchpad': 'Wyczyść bieżący notatnik', 'sp.slash.allow_api': 'Zezwól na modyfikacje API w tej rozmowie', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Wyłącz pytania o uprawnienia globalnie', 'sp.slash.compact': 'Skompaktuj kontekst tej rozmowy', 'sp.slash.verbose': 'Przełącz szczegółowy lub kompaktowy widok narzędzi', diff --git a/src/chrome/src/ui/locales/pt.js b/src/chrome/src/ui/locales/pt.js index 1c442c94c..73575db6d 100644 --- a/src/chrome/src/ui/locales/pt.js +++ b/src/chrome/src/ui/locales/pt.js @@ -129,6 +129,7 @@ export default { 'sp.slash.save_workflow': "Salve a última execução registrada com sucesso", 'sp.slash.delete_workflow': "Excluir um fluxo de trabalho salvo por ID", 'sp.slash.allow_api': "Permitir mutações de API para esta conversa", + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': "Desabilitar prompts de permissão globalmente", 'sp.slash.compact': "Compacte este contexto de conversa", 'sp.slash.verbose': "Alternar exibição de ferramenta detalhada ou compacta", diff --git a/src/chrome/src/ui/locales/ru.js b/src/chrome/src/ui/locales/ru.js index 58ec22350..2d3b29b9c 100644 --- a/src/chrome/src/ui/locales/ru.js +++ b/src/chrome/src/ui/locales/ru.js @@ -576,6 +576,7 @@ export default { 'sp.slash.show_scratchpad': 'Показать текущий блокнот', 'sp.slash.clear_scratchpad': 'Очистить текущий блокнот', 'sp.slash.allow_api': 'Разрешить изменения API для этого разговора', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Отключить запросы разрешений глобально', 'sp.slash.compact': 'Сжать контекст этого разговора', 'sp.slash.verbose': 'Переключить подробный или компактный вид инструментов', diff --git a/src/chrome/src/ui/locales/th.js b/src/chrome/src/ui/locales/th.js index 8f8188853..a174a1df7 100644 --- a/src/chrome/src/ui/locales/th.js +++ b/src/chrome/src/ui/locales/th.js @@ -576,6 +576,7 @@ export default { 'sp.slash.show_scratchpad': 'แสดงสมุดบันทึกปัจจุบัน', 'sp.slash.clear_scratchpad': 'ล้างกระดานร่างปัจจุบัน', 'sp.slash.allow_api': 'อนุญาตการเปลี่ยนแปลง API สำหรับการสนทนานี้', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'ปิดการถามสิทธิ์ทั่วทั้งระบบ', 'sp.slash.compact': 'บีบอัดบริบทการสนทนานี้', 'sp.slash.verbose': 'สลับการแสดงเครื่องมือระหว่างละเอียดและกระชับ', diff --git a/src/chrome/src/ui/locales/tl.js b/src/chrome/src/ui/locales/tl.js index 1fb9ec53c..646476da0 100644 --- a/src/chrome/src/ui/locales/tl.js +++ b/src/chrome/src/ui/locales/tl.js @@ -576,6 +576,7 @@ export default { 'sp.slash.show_scratchpad': 'Ipakita ang kasalukuyang scratchpad', 'sp.slash.clear_scratchpad': 'I-clear ang kasalukuyang scratchpad', 'sp.slash.allow_api': 'Payagan ang mga API mutation para sa pag-uusap na ito', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Patayin ang mga prompt ng pahintulot sa buong sistema', 'sp.slash.compact': 'I-compact ang konteksto ng pag-uusap na ito', 'sp.slash.verbose': 'I-toggle ang pagpapakita ng tool sa pagitan ng detalyado at compact', diff --git a/src/chrome/src/ui/locales/tr.js b/src/chrome/src/ui/locales/tr.js index b76bc4d18..28a85c4a3 100644 --- a/src/chrome/src/ui/locales/tr.js +++ b/src/chrome/src/ui/locales/tr.js @@ -615,6 +615,7 @@ export default { 'sp.slash.show_scratchpad': 'Mevcut not defterini göster', 'sp.slash.clear_scratchpad': 'Geçerli not defterini temizle', 'sp.slash.allow_api': 'Bu konuşma için API değişikliklerine izin ver', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'İzin istemlerini küresel olarak devre dışı bırak', 'sp.slash.compact': 'Bu konuşmanın bağlamını sıkıştır', 'sp.slash.verbose': 'Araç görünümünü ayrıntılı ve kompakt arasında geçiş yap', diff --git a/src/chrome/src/ui/locales/uk.js b/src/chrome/src/ui/locales/uk.js index b446716b4..69765f4f4 100644 --- a/src/chrome/src/ui/locales/uk.js +++ b/src/chrome/src/ui/locales/uk.js @@ -576,6 +576,7 @@ export default { 'sp.slash.show_scratchpad': 'Показати поточний блокнот', 'sp.slash.clear_scratchpad': 'Очистити поточний чернетник', 'sp.slash.allow_api': 'Дозволити зміни API для цієї розмови', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Вимкнути запити дозволів глобально', 'sp.slash.compact': 'Стиснути контекст цієї розмови', 'sp.slash.verbose': 'Перемкнути докладний або компактний вигляд інструментів', diff --git a/src/chrome/src/ui/locales/vi.js b/src/chrome/src/ui/locales/vi.js index 780b9daeb..40b4e004f 100644 --- a/src/chrome/src/ui/locales/vi.js +++ b/src/chrome/src/ui/locales/vi.js @@ -129,6 +129,7 @@ export default { 'sp.slash.save_workflow': "Lưu lần chạy được ghi thành công mới nhất", 'sp.slash.delete_workflow': "Xóa quy trình làm việc đã lưu theo ID", 'sp.slash.allow_api': "Cho phép đột biến API cho cuộc trò chuyện này", + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': "Tắt lời nhắc cấp phép trên toàn cầu", 'sp.slash.compact': "Thu gọn bối cảnh hội thoại này", 'sp.slash.verbose': "Chuyển đổi hiển thị công cụ dài dòng hoặc nhỏ gọn", diff --git a/src/chrome/src/ui/locales/zh.js b/src/chrome/src/ui/locales/zh.js index d526b12dc..880cf824c 100644 --- a/src/chrome/src/ui/locales/zh.js +++ b/src/chrome/src/ui/locales/zh.js @@ -576,6 +576,7 @@ export default { 'sp.slash.show_scratchpad': '显示当前草稿板', 'sp.slash.clear_scratchpad': '清除当前草稿板', 'sp.slash.allow_api': '允许此对话进行 API 修改', + 'sp.slash.foreground': '在前台运行此任务以确保视觉兼容性', 'sp.slash.dangerously_skip_permissions': '全局关闭权限提示', 'sp.slash.compact': '压缩此对话上下文', 'sp.slash.verbose': '切换工具显示的详细或紧凑模式', diff --git a/src/chrome/src/ui/settings.js b/src/chrome/src/ui/settings.js index da1869d44..b679d2366 100644 --- a/src/chrome/src/ui/settings.js +++ b/src/chrome/src/ui/settings.js @@ -52,7 +52,7 @@ import { ADDITIONAL_PROVIDER_UI } from '../providers/provider-catalog.js'; // Version shown in the subtitle. Kept here so it only needs one update per // release; the subtitle string itself is translated. -const EXT_VERSION = '25.9.7'; +const EXT_VERSION = '26.0.0'; const providersContainer = document.getElementById('providers'); const displaySettings = document.getElementById('display-settings'); diff --git a/src/chrome/src/ui/sidepanel.js b/src/chrome/src/ui/sidepanel.js index 62fde4be3..199368263 100644 --- a/src/chrome/src/ui/sidepanel.js +++ b/src/chrome/src/ui/sidepanel.js @@ -625,6 +625,7 @@ const SLASH_COMMANDS = [ ], }, { value: '/allow-api', usage: '/allow-api [prompt]', descriptionKey: 'sp.slash.allow_api', action: 'enable', acceptsPayload: true }, + { value: '/foreground', usage: '/foreground [prompt]', descriptionKey: 'sp.slash.foreground', action: 'enable', acceptsPayload: true }, { value: '/dangerously-skip-permissions', usage: '/dangerously-skip-permissions [prompt]', descriptionKey: 'sp.slash.dangerously_skip_permissions', action: 'disable', acceptsPayload: true, outOfBand: true }, { value: '/compact', usage: '/compact [prompt]', descriptionKey: 'sp.slash.compact', action: 'compact', acceptsPayload: true }, { value: '/verbose', usage: '/verbose', descriptionKey: 'sp.slash.verbose', action: 'toggle', outOfBand: true }, @@ -3732,6 +3733,7 @@ async function adoptRestoredRunState(tabId, state) { tabId, requestId, mode, + foreground: runUi.foreground === true, }, { probeFirst: true, requireDurableSubmittedTurn: runUi.kind !== 'continue', @@ -3777,6 +3779,7 @@ async function applyActiveRunState(numericTabId, state) { || addMessage('assistant', ''); currentAssistantEl = runAssistantEl; runAssistantEl.dataset.runRequestId = String(runUi.requestId); + runAssistantEl.dataset.retryForeground = runUi.foreground === true ? 'true' : 'false'; if (runUi.runId) runAssistantEl.dataset.runId = String(runUi.runId); const lastRenderedSeq = Number(runAssistantEl.dataset.lastRenderedSeq || 0); const replayEvents = Array.isArray(runUi.events) ? runUi.events : []; @@ -4094,7 +4097,9 @@ function rebindContinueButtons() { document.querySelectorAll('.continue-btn').forEach(btn => { if (btn.dataset.bound) return; btn.dataset.bound = 'true'; - btn.addEventListener('click', continueAgent); + btn.addEventListener('click', () => continueAgent({ + foreground: btn.dataset.foreground === 'true', + })); }); } @@ -5074,7 +5079,10 @@ function resumeAfterSubscription(btn) { ? btn.dataset.resumeMode : agentMode; setMode(mode); - void continueAgent({ mode }); + void continueAgent({ + mode, + foreground: btn?.dataset?.resumeForeground === 'true', + }); } function rebindSubscribeButtons() { @@ -5113,6 +5121,7 @@ function retryPayloadFromButton(btn) { text, mode, apiMutationsAllowed: btn.dataset.retryApiMutationsAllowed === 'true', + foreground: btn.dataset.retryForeground === 'true', ...(btn.dataset.retrySourceGrounding === SELECTION_ONLY_SOURCE_GROUNDING ? { sourceGrounding: SELECTION_ONLY_SOURCE_GROUNDING } : {}), @@ -5147,6 +5156,7 @@ function bindErrorRetryButton(btn) { __retry: { mode: payload.mode, apiMutationsAllowed: payload.apiMutationsAllowed, + foreground: payload.foreground, ...(payload.sourceGrounding ? { sourceGrounding: payload.sourceGrounding } : {}), attachments: payload.attachments, }, @@ -5185,6 +5195,7 @@ function retryPayloadForRunAssistant(assistantEl) { ? assistantEl.dataset.runMode : agentMode, apiMutationsAllowed: assistantEl?.dataset.retryApiMutationsAllowed === 'true', + foreground: assistantEl?.dataset.retryForeground === 'true', ...(assistantEl?.dataset.retrySourceGrounding === SELECTION_ONLY_SOURCE_GROUNDING ? { sourceGrounding: SELECTION_ONLY_SOURCE_GROUNDING } : {}), @@ -6416,6 +6427,10 @@ async function parseSlashCommands(text, tabId = currentTabId, options = {}) { return payload; } + if (command.value === '/foreground') { + return payload; + } + if (command.value === '/dangerously-skip-permissions') { await chrome.storage.local.set({ [PERMISSION_GATE_KEY]: false }).catch(() => {}); askBeforeConsequential = false; @@ -6845,6 +6860,9 @@ async function sendMessage(extraChatParams = {}) { const apiMutationsAllowedForSend = retryOptions ? !!retryOptions.apiMutationsAllowed : isApiMutationsAllowedForTab(tabId) || /^\/allow-api\b/i.test(text); + const foregroundForSend = retryOptions + ? !!retryOptions.foreground + : /^\/foreground\b/i.test(text); resetComposerHistoryNavigation(tabId); saveInputDraftForTab(tabId, ''); hideSlashCommandAutocomplete(); @@ -6908,6 +6926,7 @@ async function sendMessage(extraChatParams = {}) { text, mode: modeForSend, apiMutationsAllowed: apiMutationsAllowedForSend, + foreground: foregroundForSend, ...(sourceGrounding ? { sourceGrounding } : {}), attachments: attachmentsForSend, }; @@ -6927,6 +6946,7 @@ async function sendMessage(extraChatParams = {}) { assistantEl.dataset.runRequestId = requestId; assistantEl.dataset.runMode = modeForSend; assistantEl.dataset.retryApiMutationsAllowed = apiMutationsAllowedForSend ? 'true' : 'false'; + assistantEl.dataset.retryForeground = foregroundForSend ? 'true' : 'false'; assistantEl.dataset.retrySourceGrounding = sourceGrounding || ''; assistantEl.dataset.retryAttachmentCount = String(attachmentsForSend.length); assistantEl.dataset.lastRenderedSeq = '0'; @@ -6952,6 +6972,7 @@ async function sendMessage(extraChatParams = {}) { locale: getLocale(), intentFailureMessage: t('sp.plan.intent_unavailable'), apiMutationsAllowed: apiMutationsAllowedForSend, + foreground: foregroundForSend, ...(runCaptureDirective ? { runCapture: { kind: runCaptureDirective.kind, @@ -7509,7 +7530,11 @@ function handleAgentUpdateMessage(msg) { // already nulled out currentAssistantEl. The continue bar attaches to // the messages container, not to a specific message, so it's fine to // show unconditionally. - showContinueButton(); + { + const retryPayload = activeRetryPayloadForRequest(eventTabId, msg.requestId) + || retryPayloadForRunAssistant(eventAssistantEl); + showContinueButton({ foreground: retryPayload?.foreground === true }); + } break; case 'warning': @@ -8625,6 +8650,9 @@ function renderSubscribeError(textEl, content, resumeMode = '') { resumeBtn.dataset.resumeMode = ['ask', 'act', 'dev'].includes(resumeMode) ? resumeMode : (textEl.closest('.message.assistant')?.dataset.runMode || agentMode); + resumeBtn.dataset.resumeForeground = textEl.closest('.message.assistant')?.dataset.retryForeground === 'true' + ? 'true' + : 'false'; resumeBtn.dataset.bound = 'true'; resumeBtn.addEventListener('click', () => resumeAfterSubscription(resumeBtn)); actions.appendChild(resumeBtn); @@ -8680,6 +8708,10 @@ function renderCostAllowanceError(textEl, content, resumeMode = '', resumeOption continueBtn.dataset.resumeMode = ['ask', 'act', 'dev'].includes(resumeMode) ? resumeMode : (textEl.closest('.message.assistant')?.dataset.runMode || agentMode); + continueBtn.dataset.resumeForeground = resumeOptions?.retryPayload?.foreground === true + || textEl.closest('.message.assistant')?.dataset.retryForeground === 'true' + ? 'true' + : 'false'; continueBtn.hidden = true; continueBtn.dataset.bound = 'true'; continueBtn.addEventListener('click', () => resumeAfterSubscription(continueBtn)); @@ -8708,6 +8740,7 @@ function configureRetryButton(btn, retryPayload) { btn.dataset.retryText = String(retryPayload.text || ''); btn.dataset.retryMode = retryPayload.mode || 'ask'; btn.dataset.retryApiMutationsAllowed = retryPayload.apiMutationsAllowed ? 'true' : 'false'; + btn.dataset.retryForeground = retryPayload.foreground ? 'true' : 'false'; btn.dataset.retrySourceGrounding = retryPayload.sourceGrounding === SELECTION_ONLY_SOURCE_GROUNDING ? SELECTION_ONLY_SOURCE_GROUNDING : ''; @@ -8847,7 +8880,7 @@ function displayMaxAgentSteps(value) { return Number.isFinite(n) && n > 0 ? String(Math.floor(n)) : '130'; } -function showContinueButton() { +function showContinueButton(options = {}) { // Remove any existing continue button document.querySelectorAll('.continue-bar').forEach(el => el.remove()); resetChatNavigation(); @@ -8861,13 +8894,19 @@ function showContinueButton() { messagesEl.appendChild(bar); scrollToBottom({ force: true }); - document.getElementById('btn-continue').addEventListener('click', continueAgent); + const continueBtn = document.getElementById('btn-continue'); + continueBtn.dataset.foreground = options?.foreground === true ? 'true' : 'false'; + continueBtn.dataset.bound = 'true'; + continueBtn.addEventListener('click', () => continueAgent({ + foreground: continueBtn.dataset.foreground === 'true', + })); } async function continueAgent(options = {}) { const tabId = currentTabId; const requestId = createRunRequestId(tabId); const modeForSend = ['ask', 'act', 'dev'].includes(options?.mode) ? options.mode : agentMode; + const foregroundForSend = options?.foreground === true; clearActiveChatPayloadForTab(tabId); setTabProcessing(tabId, true); @@ -8888,6 +8927,7 @@ async function continueAgent(options = {}) { assistantEl = addMessage('assistant', ''); assistantEl.dataset.runRequestId = requestId; assistantEl.dataset.runMode = modeForSend; + assistantEl.dataset.retryForeground = foregroundForSend ? 'true' : 'false'; assistantEl.dataset.lastRenderedSeq = '0'; currentAssistantEl = assistantEl; const questionEl = precedingUserMessage(assistantEl); @@ -8901,6 +8941,7 @@ async function continueAgent(options = {}) { tabId, requestId, mode: modeForSend, + foreground: foregroundForSend, }); if (res?.conversationId) { chatHistoryConversationIdsByTab.set(tabId, res.conversationId); diff --git a/src/firefox/ARCHITECTURE.md b/src/firefox/ARCHITECTURE.md index 79d6d2701..274826a00 100644 --- a/src/firefox/ARCHITECTURE.md +++ b/src/firefox/ARCHITECTURE.md @@ -1,13 +1,13 @@ # WebBrain Firefox Extension — Architecture -> Version 25.9.7 · Manifest V2 · Background Page +> Version 26.0.0 · Manifest V2 · Background Page ## How Firefox Differs from Chrome Firefox uses Manifest V2 (background page, not service worker) and has **no access to the Chrome DevTools Protocol (CDP)**. Starting with v3.6.x, the Firefox build has been brought to functional parity with Chrome for the accessibility-tree (AX) subsystem — the same tree builder, the same four AX tools (`get_accessibility_tree`, `click_ax`, `type_ax`, `set_field`), and the same ref_id registry. What Firefox still lacks: - **No trusted events** — clicks and key presses are synthetic (`el.click()`, `new KeyboardEvent()`), and some sites reject `event.isTrusted === false`. All AX-tool click/type paths use synthetic dispatch in Firefox; the CDP-backed trusted-event path in Chrome has no Firefox equivalent. -- **No pixel-perfect / full-page screenshots** — uses `browser.tabs.captureVisibleTab()` instead of CDP `Page.captureScreenshot`. +- **No pixel-perfect / full-page screenshots** — uses `browser.tabs.captureTab()` instead of CDP `Page.captureScreenshot`; it can capture the run tab while that tab is inactive. Firefox has exposed `tabs.captureTab()` since Firefox 59, before WebBrain's Firefox 109 minimum, and the manifest declares the required `` permission. - **No shadow DOM piercing** — content script can read open shadow roots via `element.shadowRoot`, but cannot pierce closed roots. - **No offscreen document** — no HTTP fetch proxy for localhost LLM servers with Private Network Access / CORS issues. User must ensure their local LLM server sends permissive CORS headers. - **Some Chrome-only tools/features remain absent** — no CDP full-page screenshot, CDP upload automation, tab recording, offscreen fetch proxy, Chrome-only `shadow_dom_query`, or closed-shadow-root traversal. @@ -408,7 +408,7 @@ document.dispatchEvent(ev); ### Verify form -Reads all form field values via `browser.tabs.executeScript()` and captures a viewport screenshot via `browser.tabs.captureVisibleTab()`. The system prompt guides the LLM to call this before submitting important multi-field forms. +Reads all form field values via `browser.tabs.executeScript()` and captures a viewport screenshot via `browser.tabs.captureTab()`. The system prompt guides the LLM to call this before submitting important multi-field forms. --- @@ -456,10 +456,12 @@ Uses `browser.storage.local` instead of `chrome.storage.local` for config persis ## Scheduled Tasks (`scheduler.js`) -Firefox ships the same `ScheduledJobManager` class (`src/firefox/src/agent/scheduler.js`), using `browser.alarms` instead of `chrome.alarms`. Feature parity with the Chrome build except for two differences: +Firefox ships the same `ScheduledJobManager` class (`src/firefox/src/agent/scheduler.js`), using `browser.alarms` instead of `chrome.alarms`. Feature parity with the Chrome build except for one difference: - **No service-worker keepalive.** Chrome pings `chrome.runtime.getPlatformInfo` every 20 s during a job run to prevent the MV3 service worker from dying mid-run. Firefox has a persistent background page (MV2) that is always alive, so no keepalive is needed. -- **URL-target tabs open active.** On Firefox, URL-target tasks open their tab with `active: true` (Chrome opens them in the background). This is a cosmetic difference with no behavioural impact. + +Like Chrome, Firefox opens URL-target task tabs inactive and does not activate +an existing target tab before a scheduled run. All job kinds (`resume`, `task`), lifecycle states, retry/deferral logic, schedule types (`once`, `recurring`), LLM tools (`schedule_resume`, `schedule_task`), and storage key (`wb_scheduled_jobs`) are identical to Chrome. See `docs/architecture.md § Scheduled Tasks` for the full reference. diff --git a/src/firefox/manifest.json b/src/firefox/manifest.json index c779c1106..1c32fa044 100644 --- a/src/firefox/manifest.json +++ b/src/firefox/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 2, "name": "WebBrain", - "version": "25.9.7", + "version": "26.0.0", "description": "Open-source AI browser agent — chat with pages, automate tasks, multi-provider LLM support.", "permissions": [ "activeTab", diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index 0a7555ced..32f5b7068 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -2511,6 +2511,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const successfulEntries = frameEntries.filter(entry => entry?.payload && typeof entry.payload === 'object' ); + const hiddenChallenges = successfulEntries.flatMap(entry => { + const label = String(entry.payload?.hiddenChallenge?.label || ''); + if (!label) return []; + const normalized = detectChallengeDialog( + `dialog ${JSON.stringify(label.slice(0, 200))}`, + { allowGenericFailure: options?.allowGenericFailure === true }, + ); + if (!normalized?.normalizedLabel) return []; + return [{ + frameId: entry.frameId, + frameUrl: String(entry.payload?.frameContext?.frameUrl || ''), + label: normalized.label, + normalizedLabel: normalized.normalizedLabel, + }]; + }); const challenge = this._visibleChallengeDialogFromFrames( successfulEntries, navigationFrames, @@ -2521,9 +2536,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d && navigationFrames.some(frame => frame?.frameId === expectedFrameId); return { challenge, + challengeHidden: hiddenChallenges.length > 0, + hiddenChallenges, + // With no expected frame, "complete" still requires at least one frame + // to have actually been inspected. inspectionComplete: expectedFrameId === null - || inspectedFrameIds.has(expectedFrameId) - || (navigationInspectionComplete && !expectedFrameStillExists), + ? inspectedFrameIds.size > 0 + : (inspectedFrameIds.has(expectedFrameId) + || (navigationInspectionComplete && !expectedFrameStillExists)), }; } @@ -2549,6 +2569,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ? challenge.frameId : null, captchaChallengeFrameUrl: challenge.frameUrl || '', + captchaChallengeVisibilityConfirmed: true, }, {}, ); @@ -2567,6 +2588,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } const activeGate = this._captchaGateStates.get(tabId); + const treeFilter = String(toolArgs?.filter || 'all').toLowerCase(); + let observedChallengeFrameId = Number.isInteger(toolResult.captchaChallengeFrameId) + ? toolResult.captchaChallengeFrameId + : null; + let observedChallengeFrameUrl = String(toolResult.captchaChallengeFrameUrl || ''); let challenge = detectChallengeDialog(toolResult.pageContent, { allowGenericFailure: !!activeGate, }); @@ -2576,11 +2602,36 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d { allowGenericFailure: !!activeGate }, ); } + if ( + challenge + && treeFilter !== 'visible' + && toolResult.captchaChallengeVisibilityConfirmed !== true + ) { + const recheck = await this._detectChallengeDialogBeforeMutation(tabId, { + includeStatus: true, + expectedFrameId: observedChallengeFrameId, + allowGenericFailure: !!activeGate, + }); + if (recheck.challenge?.label) { + challenge = detectChallengeDialog( + `dialog ${JSON.stringify(String(recheck.challenge.label).slice(0, 200))}`, + { allowGenericFailure: !!activeGate }, + ); + observedChallengeFrameId = Number.isInteger(recheck.challenge.frameId) + ? recheck.challenge.frameId + : null; + observedChallengeFrameUrl = String(recheck.challenge.frameUrl || ''); + } + // A DOM scan cannot prove that the tree-observed dialog is the same + // element as a hidden match: a page may retain a hidden template while + // rendering the active challenge in a closed shadow root. Hidden, + // missing, and inconclusive scan results therefore remain diagnostic + // only and keep the gate fail-closed. + } let pageUrl = String(toolResult.currentUrl || toolResult.pageUrl || ''); if (!pageUrl) { try { pageUrl = await this._currentUrl(tabId); } catch {} } - const treeFilter = String(toolArgs?.filter || 'all').toLowerCase(); const requestedPage = toolArgs?.page; const requestedMaxDepth = toolArgs?.maxDepth; const parsedMaxDepth = Number(requestedMaxDepth); @@ -2778,10 +2829,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d key, status: publicGate.status, publicGate, - ...(Number.isInteger(toolResult.captchaChallengeFrameId) + ...(Number.isInteger(observedChallengeFrameId) ? { - challengeFrameId: toolResult.captchaChallengeFrameId, - challengeFrameUrl: String(toolResult.captchaChallengeFrameUrl || ''), + challengeFrameId: observedChallengeFrameId, + challengeFrameUrl: observedChallengeFrameUrl, } : {}), }); @@ -4836,10 +4887,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } /** - * Capture a viewport screenshot via the WebExtension tabs API. Firefox - * supports `scale: 1` on captureVisibleTab to force a CSS-pixel-aligned - * image (otherwise it captures at devicePixelRatio, causing the same - * coordinate-mismatch loop chrome had pre-1.5.1). Returns + * Capture a viewport screenshot for the run's tab without activating it. + * `tabs.captureTab()` has been available since Firefox 59; WebBrain's + * minimum is Firefox 109 and its `` permission authorizes capture. + * Firefox supports `scale: 1` here to force a CSS-pixel-aligned image + * (otherwise it captures at devicePixelRatio, causing the same coordinate- + * mismatch loop Chrome had pre-1.5.1). Returns * { dataUrl, width, height } in (possibly budget-resized) pixels, or null. * `opts` accepted for Chrome call-site parity; capture is always CSS-locked. */ @@ -4847,12 +4900,6 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d try { const tab = await browser.tabs.get(tabId); if (!tab) return null; - // captureVisibleTab takes a windowId and snapshots whatever is currently - // visible in that window — it does NOT take a tabId. If the agent's - // tab isn't the active tab, we'd silently capture an unrelated page - // and feed misleading visual context to the model. Skip in that case; - // the model will plan from text only this turn. - if (!tab.active) return null; const probe = await this._captureViewportProbe(tabId); const w = Math.max(1, Math.round(probe?.innerWidth || 1024)); const h = Math.max(1, Math.round(probe?.innerHeight || 768)); @@ -4863,7 +4910,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // scale: 1 forces 1 image pixel per CSS pixel (Firefox-specific option, // ignored by Chrome but Chrome path uses CDP anyway). const rawDataUrl = await this._withIndicatorsHidden(tabId, () => - browser.tabs.captureVisibleTab(tab.windowId, { + browser.tabs.captureTab(tabId, { format: 'jpeg', quality: 60, scale: 1, @@ -4871,7 +4918,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ); if (!rawDataUrl) return null; - // Firefox's captureVisibleTab doesn't take a clip/scale in a way that + // Firefox's captureTab doesn't take a clip/scale in a way that // lets us downsize during capture (scale:1 is viewport-lock, not a // factor). Capture at CSS size; shrink only when a side exceeds // maxImageDimension (coord-aligned budget). @@ -5013,7 +5060,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d async _captureVisibleMediaScreenshot(tabId) { try { const tab = await browser.tabs.get(tabId); - if (!tab?.active) return null; + if (!tab) return null; let width = 1024; let height = 768; try { @@ -5027,7 +5074,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } } catch (_) {} const cropDataUrl = await this._withIndicatorsHidden(tabId, () => - browser.tabs.captureVisibleTab(tab.windowId, { + browser.tabs.captureTab(tabId, { format: 'png', scale: 1, }) @@ -12370,27 +12417,20 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (name === 'screenshot') { try { - // Get the tab's window to capture. Firefox captureVisibleTab takes - // a windowId and snapshots whatever's currently visible in that - // window — not the tab we ask for. If the agent's tab isn't the - // active tab, refuse rather than capture an unrelated page. + // Firefox captureTab targets the run tab directly, so the user can + // continue working in another tab while the agent gathers vision. const tab = await browser.tabs.get(tabId); - if (!tab?.active) { - return { - success: false, - error: 'Cannot capture screenshot: this tab is not the active tab in its window. Switch to the tab before using /screenshot, or use a page-reading tool.', - }; - } + if (!tab) return { success: false, error: 'Screenshot failed: run tab is unavailable.' }; const probe = await this._captureViewportProbe(tabId); const cssW = Math.max(1, Math.round(probe?.innerWidth || 1024)); const cssH = Math.max(1, Math.round(probe?.innerHeight || 768)); const captureOnce = async () => { // scale: 1 CSS-locks the capture (Firefox-specific option). Without - // it captureVisibleTab snapshots at native devicePixelRatio, so on + // it captureTab snapshots at native devicePixelRatio, so on // hi-DPI displays the image would be DPR× the CSS viewport and any // pixel coordinate read off it would miss. const rawUrl = await this._withIndicatorsHidden(tabId, () => - browser.tabs.captureVisibleTab(tab.windowId, { + browser.tabs.captureTab(tabId, { format: 'png', quality: 80, scale: 1, @@ -12483,7 +12523,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (this._isActionMode(mode)) { try { const tab = await browser.tabs.get(tabId); - if (tab?.active) { + // Scheduled URL-target tabs intentionally stay inactive. Firefox can + // execute scripts in and capture those tabs directly, so completion + // verification must not depend on active-tab state. + if (tab) { // Probe page URL, title, and "work in progress" signals: open // dialogs/modals and visible forms. If any of these are present // while the model claims it created/added/saved/submitted, the @@ -12551,7 +12594,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d const plannerCanSeeImages = !!provider?.supportsVision; let dataUrl = plannerCanSeeImages ? await this._withIndicatorsHidden(tabId, () => - browser.tabs.captureVisibleTab(tab.windowId, { format: 'png', quality: 80 }) + browser.tabs.captureTab(tabId, { format: 'png', quality: 80 }) ) : null; if (dataUrl && this.screenshotRedaction) { @@ -13499,16 +13542,16 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } } catch {} - // Capture screenshot (requires active tab) + // Capture the run tab directly without activating it. try { const tab = await browser.tabs.get(tabId); - if (tab?.active) { + if (tab) { // Route through `_attachImage` (like the `screenshot` tool) so the // batch loop strips it and re-attaches it as an image_url block. // Left inline as `result.image`, the base64 data URL blows past the // tool-result char cap and gets truncated to unreadable garbage. let verifyShotUrl = await this._withIndicatorsHidden(tabId, () => - browser.tabs.captureVisibleTab(tab.windowId, { format: 'png', quality: 80 }) + browser.tabs.captureTab(tabId, { format: 'png', quality: 80 }) ); // Budget-resize for the model (issue #311 maxImageDimension). verifyShotUrl = (await this._shrinkImageForBudget(verifyShotUrl, 0, 0, this._budgetForCapture())).dataUrl; diff --git a/src/firefox/src/agent/captcha-frame-runtime.js b/src/firefox/src/agent/captcha-frame-runtime.js index 63f0fa6ac..d73743a85 100644 --- a/src/firefox/src/agent/captcha-frame-runtime.js +++ b/src/firefox/src/agent/captcha-frame-runtime.js @@ -630,11 +630,22 @@ export function detectCaptchaCandidatesInPage(scope = null) { const visibleElement = (element) => { if (!element) return false; try { - const style = typeof pageWindow?.getComputedStyle === 'function' + const elementStyle = typeof pageWindow?.getComputedStyle === 'function' ? pageWindow.getComputedStyle(element) : null; - if (style && (style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0)) return false; - if (element.hidden || element.getAttribute?.('aria-hidden') === 'true') return false; + if ( + elementStyle + && (elementStyle.visibility === 'hidden' || elementStyle.visibility === 'collapse') + ) return false; + let current = element; + for (let depth = 0; current && depth < 30; depth += 1) { + const style = typeof pageWindow?.getComputedStyle === 'function' + ? pageWindow.getComputedStyle(current) + : null; + if (style && (style.display === 'none' || Number(style.opacity) === 0)) return false; + if (current.hidden || current.getAttribute?.('aria-hidden') === 'true') return false; + current = current.parentElement || null; + } if (typeof element.getBoundingClientRect === 'function') { const rect = element.getBoundingClientRect(); if (!rect || rect.width <= 0 || rect.height <= 0) return false; @@ -660,13 +671,21 @@ export function detectCaptchaCandidatesInPage(scope = null) { .map(id => pageDocument.getElementById?.(id)?.textContent || '') .join(' '); } catch (_) {} + let renderedHeadings = ''; + try { + renderedHeadings = Array.from( + element.querySelectorAll?.('h1, h2, h3, [role="heading"]') || [] + ) + .filter(visibleElement) + .map(heading => heading.textContent || '') + .join(' '); + } catch (_) {} return [ element.getAttribute?.('aria-label'), labelledBy, - element.querySelector?.('h1, h2, h3, [role="heading"]')?.textContent, + renderedHeadings, element.getAttribute?.('title'), element.innerText, - element.textContent, ].some(value => challengeDialogRe.test(String(value || ''))); }); const elementInChallengeDialog = (element) => { diff --git a/src/firefox/src/agent/captcha-gate.js b/src/firefox/src/agent/captcha-gate.js index 4cffd9110..d2e2cfa3f 100644 --- a/src/firefox/src/agent/captcha-gate.js +++ b/src/firefox/src/agent/captcha-gate.js @@ -107,9 +107,17 @@ export function detectChallengeDialogInPage(options = null) { }; const visible = (element) => { try { - const style = getComputedStyle(element); - if (style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0) return false; - if (element.hidden || element.getAttribute?.('aria-hidden') === 'true') return false; + const elementStyle = getComputedStyle(element); + if (elementStyle.visibility === 'hidden' || elementStyle.visibility === 'collapse') return false; + let current = element; + for (let depth = 0; current && depth < 30; depth += 1) { + const style = getComputedStyle(current); + if (style.display === 'none' || Number(style.opacity) === 0) return false; + if (current.hidden || current.getAttribute?.('aria-hidden') === 'true') return false; + current = current.parentElement + || (typeof current.getRootNode === 'function' ? current.getRootNode()?.host : null) + || null; + } const rect = element.getBoundingClientRect(); if (rect.width <= 0 || rect.height <= 0) return false; const viewportWidth = typeof window !== 'undefined' && typeof window.innerWidth === 'number' @@ -139,9 +147,16 @@ export function detectChallengeDialogInPage(options = null) { visible: visible(element), }; }); + // A challenge dialog that exists in the DOM but is hidden or off-viewport + // is positive evidence the challenge is inactive. Report it separately so + // callers can distinguish "confirmed hidden" from "not found" — a dialog + // this scan cannot see at all (closed shadow root, unreachable frame) must + // not be treated as disproved. + let hiddenChallenge = null; const finish = challenge => includeFrameContext ? { challenge, + ...(hiddenChallenge ? { hiddenChallenge } : {}), frameContext: { frameUrl, frameName, @@ -149,23 +164,80 @@ export function detectChallengeDialogInPage(options = null) { }, } : challenge; - for (const element of document.querySelectorAll('dialog, [role="dialog"], [role="alertdialog"]')) { - if (!visible(element)) continue; + // The accessibility tree pierces open shadow roots on some sites, where + // frameworks (e.g. LinkedIn's interop shell) render whole dialogs that + // document.querySelectorAll cannot see. Scan open shadow roots too so this + // preflight can confirm the same dialogs the tree reports. + const dialogSelector = 'dialog, [role="dialog"], [role="alertdialog"]'; + const dialogCandidates = []; + const collectDialogs = (root, depth) => { + try { + for (const element of root.querySelectorAll(dialogSelector)) { + if (dialogCandidates.length >= 200) break; + dialogCandidates.push(element); + } + } catch {} + if (depth >= 4) return; + let descendants; + try { descendants = root.querySelectorAll('*'); } catch { return; } + for (const host of descendants) { + if (host.shadowRoot) collectDialogs(host.shadowRoot, depth + 1); + } + }; + collectDialogs(document, 0); + const recordHiddenChallenge = (element) => { + if (hiddenChallenge) return; + try { + const values = [ + element.getAttribute?.('aria-label'), + element.getAttribute?.('title'), + element.innerText, + element.textContent, + ...Array.from(element.querySelectorAll?.('h1, h2, h3, [role="heading"]') || []) + .map(heading => heading.textContent || ''), + ]; + for (const value of values) { + const text = String(value || ''); + if (!matchesChallenge(text)) continue; + const line = text.split(/\r?\n/).find(entry => matchesChallenge(entry)) || text; + const label = line.replace(/\s+/g, ' ').trim().slice(0, 200); + if (label) { + hiddenChallenge = { label }; + return; + } + } + } catch {} + }; + for (const element of dialogCandidates) { + if (!visible(element)) { + recordHiddenChallenge(element); + continue; + } let labelledBy = ''; try { + const idRoot = (typeof element.getRootNode === 'function' && element.getRootNode()) || document; labelledBy = String(element.getAttribute('aria-labelledby') || '') .split(/\s+/) .filter(Boolean) - .map(id => document.getElementById(id)?.textContent || '') + .map(id => (typeof idRoot.getElementById === 'function' ? idRoot : document) + .getElementById(id)?.textContent || '') + .join(' '); + } catch {} + let renderedHeadings = ''; + try { + renderedHeadings = Array.from( + element.querySelectorAll?.('h1, h2, h3, [role="heading"]') || [] + ) + .filter(visible) + .map(heading => heading.textContent || '') .join(' '); } catch {} const values = [ element.getAttribute?.('aria-label'), labelledBy, - element.querySelector?.('h1, h2, h3, [role="heading"]')?.textContent, + renderedHeadings, element.getAttribute?.('title'), element.innerText, - element.textContent, ]; for (const value of values) { const text = String(value || ''); diff --git a/src/firefox/src/agent/scheduler.js b/src/firefox/src/agent/scheduler.js index 3ad3132ba..9c374016d 100644 --- a/src/firefox/src/agent/scheduler.js +++ b/src/firefox/src/agent/scheduler.js @@ -1230,14 +1230,11 @@ export class ScheduledJobManager { try { const tab = await this.api.tabs.get(job.target.tabId); if (sameTargetUrl(job.target.url, tab?.url || '')) { - if (job.source !== 'watch') { - await this.api.tabs.update(job.target.tabId, { active: true }); - } return job.target.tabId; } if (job.source !== 'watch') { try { - await this.api.tabs.update(job.target.tabId, { url: job.target.url, active: true }); + await this.api.tabs.update(job.target.tabId, { url: job.target.url }); return job.target.tabId; } catch { /* create a fresh tab below */ } } else { @@ -1248,7 +1245,7 @@ export class ScheduledJobManager { } } catch { /* create a fresh tab below */ } } - const tab = await this.api.tabs.create({ url: job.target.url, active: job.source !== 'watch' }); + const tab = await this.api.tabs.create({ url: job.target.url, active: false }); await this._updateJob(job.id, () => ({ tabId: tab.id, target: { ...job.target, tabId: tab.id }, diff --git a/src/firefox/src/background.js b/src/firefox/src/background.js index 236a26b59..dd1b3d9df 100644 --- a/src/firefox/src/background.js +++ b/src/firefox/src/background.js @@ -1532,6 +1532,14 @@ function assertNoActiveTabRun(tabId) { } } +async function activateForegroundCompatibilityTab(tabId) { + const tab = await browser.tabs.get(tabId); + await browser.tabs.update(tabId, { active: true }); + if (tab?.windowId != null) { + await browser.windows.update(tab.windowId, { focused: true }); + } +} + const detachedRunStarts = new Map(); const detachedRunFailures = new Map(); const RUN_KEEPALIVE_INTERVAL_MS = 20_000; @@ -1871,7 +1879,11 @@ async function handleMessage(msg, sender) { assertRunCanStart(tabId, msg); const isWorkflowRun = !!msg.workflowId; const mode = isWorkflowRun ? 'act' : (msg.mode || 'ask'); - const runUi = await beginContinuationRunUiSnapshot(tabId, msg.requestId, { mode, kind: 'chat' }); + const runUi = await beginContinuationRunUiSnapshot(tabId, msg.requestId, { + mode, + kind: 'chat', + foreground: msg.foreground === true, + }); const releaseRunKeepalive = acquireRunKeepalive(); if (msg.apiMutationsAllowed) agent.setApiMutationsAllowed(tabId, true); @@ -1884,6 +1896,8 @@ async function handleMessage(msg, sender) { let result = ''; let runError = null; try { + if (msg.foreground) await activateForegroundCompatibilityTab(tabId); + // Capture belongs to the background run lifecycle so it survives the // sidebar closing or reloading while the agent is still working. runCaptureState = await runCaptureController.start(msg.runCapture, tabId); @@ -1898,6 +1912,7 @@ async function handleMessage(msg, sender) { const runOptions = { ...(isWorkflowRun ? { independentRun: true } : {}), ...(msg.recommendedAction ? { recommendedAction: msg.recommendedAction } : {}), + ...(msg.foreground ? { foreground: true } : {}), ...(msg.sourceGrounding === SELECTION_ONLY_SOURCE_GROUNDING ? { sourceGrounding: SELECTION_ONLY_SOURCE_GROUNDING } : {}), @@ -1988,7 +2003,11 @@ async function handleMessage(msg, sender) { if (!tabId) throw new Error('No tab ID'); assertNoActiveTabRun(tabId); const mode = msg.mode || 'ask'; - const runUi = beginRunUiSnapshot(tabId, msg.requestId); + const runUi = beginRunUiSnapshot(tabId, msg.requestId, { + mode, + kind: 'chat', + foreground: msg.foreground === true, + }); const releaseRunKeepalive = acquireRunKeepalive(); if (msg.apiMutationsAllowed) agent.setApiMutationsAllowed(tabId, true); @@ -1999,8 +2018,11 @@ async function handleMessage(msg, sender) { let result = ''; let runError = null; try { + if (msg.foreground) await activateForegroundCompatibilityTab(tabId); + const runOptions = { ...(msg.recommendedAction ? { recommendedAction: msg.recommendedAction } : {}), + ...(msg.foreground ? { foreground: true } : {}), ...(msg.sourceGrounding === SELECTION_ONLY_SOURCE_GROUNDING ? { sourceGrounding: SELECTION_ONLY_SOURCE_GROUNDING } : {}), @@ -2039,7 +2061,11 @@ async function handleMessage(msg, sender) { if (!tabId) throw new Error('No tab ID'); assertRunCanStart(tabId, msg); const mode = msg.mode || 'ask'; - const runUi = await beginContinuationRunUiSnapshot(tabId, msg.requestId, { mode, kind: 'continue' }); + const runUi = await beginContinuationRunUiSnapshot(tabId, msg.requestId, { + mode, + kind: 'continue', + foreground: msg.foreground === true, + }); const releaseRunKeepalive = acquireRunKeepalive(); sendIndicatorMessage(tabId, 'WB_SHOW_AGENT_INDICATORS'); @@ -2048,10 +2074,13 @@ async function handleMessage(msg, sender) { let result = ''; let runError = null; try { + if (msg.foreground) await activateForegroundCompatibilityTab(tabId); + result = await agent.continueProcessing(tabId, (type, data) => { updates.push({ type, data }); sendAgentUpdate(tabId, runUi.requestId, type, data); }, mode, { + ...(msg.foreground ? { foreground: true } : {}), detachedRequestId: runUi.requestId, isDetachedStartCancelled: () => isDetachedRunStartCancelled(tabId, msg), beforeConsequentialTool: () => flushRunUiSnapshot(tabId, runUi.requestId), diff --git a/src/firefox/src/run-capture.js b/src/firefox/src/run-capture.js index c2d37921d..a837cf3ff 100644 --- a/src/firefox/src/run-capture.js +++ b/src/firefox/src/run-capture.js @@ -50,19 +50,13 @@ function normalizeRunCaptureRequest(value) { } export async function captureAndSaveRunScreenshot(api, tabId, filename) { - let tab = tabId == null ? null : await api.tabs.get(tabId); - if (!tab || tab.windowId == null) { + const tab = tabId == null ? null : await api.tabs.get(tabId); + if (!tab) { throw new Error('The run tab is no longer available.'); } - // A run can activate a new tab. Return to the originating tab so the after - // image captures the page that the agent operated on. - if (!tab.active) { - tab = await api.tabs.update(tabId, { active: true }); - } - if (!tab?.active || tab.windowId == null) { - throw new Error('The run tab could not be activated for capture.'); - } - const dataUrl = await api.tabs.captureVisibleTab(tab.windowId, { format: 'png' }); + // Firefox has supported arbitrary-tab capture since Firefox 59. WebBrain's + // declared minimum is 109 and the manifest includes the required . + const dataUrl = await api.tabs.captureTab(tabId, { format: 'png' }); filename = await filenameInConfiguredDownloadDirectory(api, filename); const downloadId = await api.downloads.download({ url: dataUrl, diff --git a/src/firefox/src/run-ui-journal.js b/src/firefox/src/run-ui-journal.js index 2be6e5b90..57a0430fa 100644 --- a/src/firefox/src/run-ui-journal.js +++ b/src/firefox/src/run-ui-journal.js @@ -116,6 +116,7 @@ export class RunUiJournal { requestId: createRunRequestId(tabId, requestId), mode: String(metadata?.mode || ''), kind: metadata?.kind === 'continue' ? 'continue' : 'chat', + foreground: metadata?.foreground === true, runId: null, status: 'running', seq: 0, @@ -144,6 +145,7 @@ export class RunUiJournal { const snapshot = this.snapshots.get(tabId); if (!snapshot || String(snapshot.requestId) !== String(requestId)) return null; if (metadata?.mode) snapshot.mode = String(metadata.mode); + if (typeof metadata?.foreground === 'boolean') snapshot.foreground = metadata.foreground; if (!snapshot.kind && metadata?.kind) { snapshot.kind = metadata.kind === 'continue' ? 'continue' : 'chat'; } @@ -273,6 +275,7 @@ export class RunUiJournal { } if (typeof snapshot.mode !== 'string') snapshot.mode = ''; if (snapshot.kind !== 'continue' && snapshot.kind !== 'chat') snapshot.kind = 'chat'; + if (snapshot.foreground !== true) snapshot.foreground = false; if (typeof snapshot.streamedText !== 'string') snapshot.streamedText = ''; if (snapshot.streamedText.length > RUN_UI_STREAM_TEXT_LIMIT) { snapshot.streamedText = ''; diff --git a/src/firefox/src/ui/locales/ar.js b/src/firefox/src/ui/locales/ar.js index 1fed9e51c..435181cca 100644 --- a/src/firefox/src/ui/locales/ar.js +++ b/src/firefox/src/ui/locales/ar.js @@ -567,6 +567,7 @@ export default { 'sp.slash.show_scratchpad': 'عرض لوحة المسودة الحالية', 'sp.slash.clear_scratchpad': 'مسح منطقة المسودة الحالية', 'sp.slash.allow_api': 'السماح بتعديلات API لهذه المحادثة', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'تعطيل مطالبات الأذونات عالميًا', 'sp.slash.compact': 'ضغط سياق هذه المحادثة', 'sp.slash.verbose': 'تبديل عرض الأدوات بين مفصل ومضغوط', diff --git a/src/firefox/src/ui/locales/bn.js b/src/firefox/src/ui/locales/bn.js index e237dcb34..250b0c1ab 100644 --- a/src/firefox/src/ui/locales/bn.js +++ b/src/firefox/src/ui/locales/bn.js @@ -129,6 +129,7 @@ export default { 'sp.slash.save_workflow': "সর্বশেষ সফল রেকর্ড করা রান সংরক্ষণ করুন", 'sp.slash.delete_workflow': "আইডি দ্বারা একটি সংরক্ষিত ওয়ার্কফ্লো মুছুন", 'sp.slash.allow_api': "এই কথোপকথনের জন্য API মিউটেশনের অনুমতি দিন", + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': "বিশ্বব্যাপী অনুমতি প্রম্পট অক্ষম করুন", 'sp.slash.compact': "এই কথোপকথন প্রসঙ্গে কম্প্যাক্ট", 'sp.slash.verbose': "ভার্বোজ বা কমপ্যাক্ট টুল ডিসপ্লে টগল করুন", diff --git a/src/firefox/src/ui/locales/de.js b/src/firefox/src/ui/locales/de.js index f7ef74dcd..e9f62c868 100644 --- a/src/firefox/src/ui/locales/de.js +++ b/src/firefox/src/ui/locales/de.js @@ -122,6 +122,7 @@ export default { 'sp.slash.save_workflow': 'Letzten erfolgreichen aufgezeichneten Durchlauf speichern', 'sp.slash.delete_workflow': 'Gespeicherten Workflow nach ID löschen', 'sp.slash.allow_api': 'API-Mutationen für dieses Gespräch erlauben', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Berechtigungsabfragen global deaktivieren', 'sp.slash.compact': 'Diesen Gesprächskontext kompaktieren', 'sp.slash.verbose': 'Ausführliche oder kompakte Tool-Anzeige umschalten', diff --git a/src/firefox/src/ui/locales/en.js b/src/firefox/src/ui/locales/en.js index 053097457..58d6aa3be 100644 --- a/src/firefox/src/ui/locales/en.js +++ b/src/firefox/src/ui/locales/en.js @@ -129,6 +129,7 @@ export default { 'sp.slash.save_workflow': 'Save the latest successful recorded run', 'sp.slash.delete_workflow': 'Delete a saved workflow by ID', 'sp.slash.allow_api': 'Allow API mutations for this conversation', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Disable permission prompts globally', 'sp.slash.compact': 'Compact this conversation context', 'sp.slash.verbose': 'Toggle verbose or compact tool display', diff --git a/src/firefox/src/ui/locales/es.js b/src/firefox/src/ui/locales/es.js index baed3749d..08a1b5637 100644 --- a/src/firefox/src/ui/locales/es.js +++ b/src/firefox/src/ui/locales/es.js @@ -567,6 +567,7 @@ export default { 'sp.slash.show_scratchpad': 'Mostrar el bloc de notas actual', 'sp.slash.clear_scratchpad': 'Borrar el bloc de notas actual', 'sp.slash.allow_api': 'Permitir mutaciones de API en esta conversación', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Desactivar las solicitudes de permiso globalmente', 'sp.slash.compact': 'Compactar el contexto de esta conversación', 'sp.slash.verbose': 'Alternar entre vista de herramientas detallada o compacta', diff --git a/src/firefox/src/ui/locales/fa.js b/src/firefox/src/ui/locales/fa.js index 02d6ab1c0..4616ddd16 100644 --- a/src/firefox/src/ui/locales/fa.js +++ b/src/firefox/src/ui/locales/fa.js @@ -129,6 +129,7 @@ export default { 'sp.slash.save_workflow': "آخرین اجرای موفق ثبت شده را ذخیره کنید", 'sp.slash.delete_workflow': "یک گردش کار ذخیره شده توسط ID را حذف کنید", 'sp.slash.allow_api': "جهش های API را برای این مکالمه مجاز کنید", + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': "درخواست های مجوز را در سطح جهانی غیرفعال کنید", 'sp.slash.compact': "این زمینه مکالمه را فشرده کنید", 'sp.slash.verbose': "نمایش ابزار فشرده یا پرمخاطب را تغییر دهید", diff --git a/src/firefox/src/ui/locales/fr.js b/src/firefox/src/ui/locales/fr.js index 9592e91eb..6401ff881 100644 --- a/src/firefox/src/ui/locales/fr.js +++ b/src/firefox/src/ui/locales/fr.js @@ -567,6 +567,7 @@ export default { 'sp.slash.show_scratchpad': 'Afficher le bloc-notes actuel', 'sp.slash.clear_scratchpad': 'Effacer le bloc-notes actuel', 'sp.slash.allow_api': 'Autoriser les mutations API pour cette conversation', + 'sp.slash.foreground': 'Exécuter cette tâche au premier plan pour assurer la compatibilité visuelle', 'sp.slash.dangerously_skip_permissions': 'Désactiver les demandes d\'autorisation globalement', 'sp.slash.compact': 'Compacter le contexte de cette conversation', 'sp.slash.verbose': "Basculer l'affichage des outils entre détaillé et compact", diff --git a/src/firefox/src/ui/locales/he.js b/src/firefox/src/ui/locales/he.js index 1495a74e2..65149f205 100644 --- a/src/firefox/src/ui/locales/he.js +++ b/src/firefox/src/ui/locales/he.js @@ -117,6 +117,7 @@ export default { "sp.slash.show_memory": "הצג זיכרון משתמש שמור", "sp.slash.forget_memory": "שכח זיכרון שמור לפי מזהה", "sp.slash.allow_api": "אפשר שינויי API בשיחה הזו", + "sp.slash.foreground": "Run this task in the foreground for visual compatibility", "sp.slash.dangerously_skip_permissions": "השבת בקשות הרשאה באופן גלובלי", "sp.slash.compact": "דחוס את הקשר השיחה הזה", "sp.slash.verbose": "החלף את תצוגת הכלים המילולית או הקומפקטית", diff --git a/src/firefox/src/ui/locales/hi.js b/src/firefox/src/ui/locales/hi.js index fa8e090bc..6b440a835 100644 --- a/src/firefox/src/ui/locales/hi.js +++ b/src/firefox/src/ui/locales/hi.js @@ -129,6 +129,7 @@ export default { 'sp.slash.save_workflow': "नवीनतम सफल रिकॉर्ड किए गए रन को सहेजें", 'sp.slash.delete_workflow': "आईडी के आधार पर सहेजे गए वर्कफ़्लो को हटाएँ", 'sp.slash.allow_api': "इस वार्तालाप के लिए एपीआई म्यूटेशन की अनुमति दें", + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': "विश्व स्तर पर अनुमति संकेतों को अक्षम करें", 'sp.slash.compact': "इस वार्तालाप प्रसंग को संक्षिप्त करें", 'sp.slash.verbose': "वर्बोज़ या कॉम्पैक्ट टूल डिस्प्ले को टॉगल करें", diff --git a/src/firefox/src/ui/locales/id.js b/src/firefox/src/ui/locales/id.js index 4d4b3d818..1abdd73e4 100644 --- a/src/firefox/src/ui/locales/id.js +++ b/src/firefox/src/ui/locales/id.js @@ -567,6 +567,7 @@ export default { 'sp.slash.show_scratchpad': 'Tampilkan catatan saat ini', 'sp.slash.clear_scratchpad': 'Kosongkan bak pasir saat ini', 'sp.slash.allow_api': 'Izinkan mutasi API untuk percakapan ini', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Nonaktifkan pertanyaan izin secara global', 'sp.slash.compact': 'Kompak konteks percakapan ini', 'sp.slash.verbose': 'Beralih tampilan alat antara verbose dan kompak', diff --git a/src/firefox/src/ui/locales/ja.js b/src/firefox/src/ui/locales/ja.js index fb2cf1dcd..120784388 100644 --- a/src/firefox/src/ui/locales/ja.js +++ b/src/firefox/src/ui/locales/ja.js @@ -567,6 +567,7 @@ export default { 'sp.slash.show_scratchpad': '現在のスクラッチパッドを表示', 'sp.slash.clear_scratchpad': '現在のスクラッチパッドをクリア', 'sp.slash.allow_api': 'この会話でAPIの変更を許可', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': '権限の確認をグローバルに無効化', 'sp.slash.compact': 'この会話のコンテキストをコンパクト化', 'sp.slash.verbose': 'ツール表示を詳細とコンパクトで切り替え', diff --git a/src/firefox/src/ui/locales/ko.js b/src/firefox/src/ui/locales/ko.js index af6976403..1090ee417 100644 --- a/src/firefox/src/ui/locales/ko.js +++ b/src/firefox/src/ui/locales/ko.js @@ -567,6 +567,7 @@ export default { 'sp.slash.show_scratchpad': '현재 스크래치패드 표시', 'sp.slash.clear_scratchpad': '현재 스크래치패드 지우기', 'sp.slash.allow_api': '이 대화에서 API 변경 허용', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': '권한 확인을 전역으로 비활성화', 'sp.slash.compact': '이 대화 컨텍스트 압축', 'sp.slash.verbose': '도구 표시를 상세 또는 압축으로 전환', diff --git a/src/firefox/src/ui/locales/ms.js b/src/firefox/src/ui/locales/ms.js index c63037e47..da0def023 100644 --- a/src/firefox/src/ui/locales/ms.js +++ b/src/firefox/src/ui/locales/ms.js @@ -567,6 +567,7 @@ export default { 'sp.slash.show_scratchpad': 'Tunjukkan pad nota semasa', 'sp.slash.clear_scratchpad': 'Kosongkan pad nota semasa', 'sp.slash.allow_api': 'Benarkan mutasi API untuk perbualan ini', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Nyahaktifkan gesaan kebenaran secara global', 'sp.slash.compact': 'Padatkan konteks perbualan ini', 'sp.slash.verbose': 'Togol paparan alat antara terperinci dan padat', diff --git a/src/firefox/src/ui/locales/nl.js b/src/firefox/src/ui/locales/nl.js index 638a2b399..f2cc101ca 100644 --- a/src/firefox/src/ui/locales/nl.js +++ b/src/firefox/src/ui/locales/nl.js @@ -125,6 +125,7 @@ export default { 'sp.slash.save_workflow': 'Sla de laatste succesvolle opgenomen uitvoering op', 'sp.slash.delete_workflow': 'Verwijder een opgeslagen workflow op ID', 'sp.slash.allow_api': 'Sta API-wijzigingen toe voor dit gesprek', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Schakel machtigingsvragen globaal uit', 'sp.slash.compact': 'Compact deze gesprekscontext', 'sp.slash.verbose': 'Schakel uitvoerige of compacte toolweergave', diff --git a/src/firefox/src/ui/locales/pl.js b/src/firefox/src/ui/locales/pl.js index 0c5e749b3..aa0f37f15 100644 --- a/src/firefox/src/ui/locales/pl.js +++ b/src/firefox/src/ui/locales/pl.js @@ -87,6 +87,7 @@ export default { 'sp.slash.show_scratchpad': 'Pokaż bieżący notatnik', 'sp.slash.clear_scratchpad': 'Wyczyść bieżący notatnik', 'sp.slash.allow_api': 'Zezwól na modyfikacje API w tej rozmowie', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Wyłącz pytania o uprawnienia globalnie', 'sp.slash.compact': 'Skompaktuj kontekst tej rozmowy', 'sp.slash.verbose': 'Przełącz szczegółowy lub kompaktowy widok narzędzi', diff --git a/src/firefox/src/ui/locales/pt.js b/src/firefox/src/ui/locales/pt.js index a48ec1c76..f598b31f3 100644 --- a/src/firefox/src/ui/locales/pt.js +++ b/src/firefox/src/ui/locales/pt.js @@ -129,6 +129,7 @@ export default { 'sp.slash.save_workflow': "Salve a última execução registrada com sucesso", 'sp.slash.delete_workflow': "Excluir um fluxo de trabalho salvo por ID", 'sp.slash.allow_api': "Permitir mutações de API para esta conversa", + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': "Desabilitar prompts de permissão globalmente", 'sp.slash.compact': "Compacte este contexto de conversa", 'sp.slash.verbose': "Alternar exibição de ferramenta detalhada ou compacta", diff --git a/src/firefox/src/ui/locales/ru.js b/src/firefox/src/ui/locales/ru.js index 166fd0cd8..b8e6f12a2 100644 --- a/src/firefox/src/ui/locales/ru.js +++ b/src/firefox/src/ui/locales/ru.js @@ -567,6 +567,7 @@ export default { 'sp.slash.show_scratchpad': 'Показать текущий блокнот', 'sp.slash.clear_scratchpad': 'Очистить текущий блокнот', 'sp.slash.allow_api': 'Разрешить изменения API для этого разговора', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Disable permission prompts globally', 'sp.slash.compact': 'Сжать контекст этого разговора', 'sp.slash.verbose': 'Переключить подробный или компактный вид инструментов', diff --git a/src/firefox/src/ui/locales/th.js b/src/firefox/src/ui/locales/th.js index c7eeecbe6..14038fd23 100644 --- a/src/firefox/src/ui/locales/th.js +++ b/src/firefox/src/ui/locales/th.js @@ -567,6 +567,7 @@ export default { 'sp.slash.show_scratchpad': 'แสดงสมุดบันทึกปัจจุบัน', 'sp.slash.clear_scratchpad': 'ล้างกระดานร่างปัจจุบัน', 'sp.slash.allow_api': 'อนุญาตการเปลี่ยนแปลง API สำหรับการสนทนานี้', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'ปิดการถามสิทธิ์โดยทั่วไป', 'sp.slash.compact': 'บีบอัดบริบทการสนทนานี้', 'sp.slash.verbose': 'สลับการแสดงเครื่องมือระหว่างละเอียดและกระชับ', diff --git a/src/firefox/src/ui/locales/tl.js b/src/firefox/src/ui/locales/tl.js index cc74ad5af..f1d009b88 100644 --- a/src/firefox/src/ui/locales/tl.js +++ b/src/firefox/src/ui/locales/tl.js @@ -567,6 +567,7 @@ export default { 'sp.slash.show_scratchpad': 'Ipakita ang kasalukuyang scratchpad', 'sp.slash.clear_scratchpad': 'I-clear ang kasalukuyang scratchpad', 'sp.slash.allow_api': 'Payagan ang mga API mutation para sa pag-uusap na ito', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'I-disable ang mga prompt ng pahintulot sa pangkalahatan', 'sp.slash.compact': 'I-compact ang konteksto ng pag-uusap na ito', 'sp.slash.verbose': 'I-toggle ang pagpapakita ng tool sa pagitan ng detalyado at compact', diff --git a/src/firefox/src/ui/locales/tr.js b/src/firefox/src/ui/locales/tr.js index 0184c835a..b7cf96220 100644 --- a/src/firefox/src/ui/locales/tr.js +++ b/src/firefox/src/ui/locales/tr.js @@ -606,6 +606,7 @@ export default { 'sp.slash.show_scratchpad': 'Mevcut not defterini göster', 'sp.slash.clear_scratchpad': 'Geçerli not defterini temizle', 'sp.slash.allow_api': 'Bu konuşma için API değişikliklerine izin ver', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'İzin istemlerini küresel olarak devre dışı bırak', 'sp.slash.compact': 'Bu konuşmanın bağlamını sıkıştır', 'sp.slash.verbose': 'Araç görünümünü ayrıntılı ve kompakt arasında geçiş yap', diff --git a/src/firefox/src/ui/locales/uk.js b/src/firefox/src/ui/locales/uk.js index ec1e282ea..3c2fc1594 100644 --- a/src/firefox/src/ui/locales/uk.js +++ b/src/firefox/src/ui/locales/uk.js @@ -567,6 +567,7 @@ export default { 'sp.slash.show_scratchpad': 'Показати поточний блокнот', 'sp.slash.clear_scratchpad': 'Очистити поточний чернетник', 'sp.slash.allow_api': 'Дозволити зміни API для цієї розмови', + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': 'Вимкнути запити дозволів глобально', 'sp.slash.compact': 'Стиснути контекст цієї розмови', 'sp.slash.verbose': 'Перемкнути докладний або компактний вигляд інструментів', diff --git a/src/firefox/src/ui/locales/vi.js b/src/firefox/src/ui/locales/vi.js index f83391e38..6189a9ed1 100644 --- a/src/firefox/src/ui/locales/vi.js +++ b/src/firefox/src/ui/locales/vi.js @@ -129,6 +129,7 @@ export default { 'sp.slash.save_workflow': "Lưu lần chạy được ghi thành công mới nhất", 'sp.slash.delete_workflow': "Xóa quy trình làm việc đã lưu theo ID", 'sp.slash.allow_api': "Cho phép đột biến API cho cuộc trò chuyện này", + 'sp.slash.foreground': 'Run this task in the foreground for visual compatibility', 'sp.slash.dangerously_skip_permissions': "Tắt lời nhắc cấp phép trên toàn cầu", 'sp.slash.compact': "Thu gọn bối cảnh hội thoại này", 'sp.slash.verbose': "Chuyển đổi hiển thị công cụ dài dòng hoặc nhỏ gọn", diff --git a/src/firefox/src/ui/locales/zh.js b/src/firefox/src/ui/locales/zh.js index 5e9a8cd2d..13f6b35a7 100644 --- a/src/firefox/src/ui/locales/zh.js +++ b/src/firefox/src/ui/locales/zh.js @@ -567,6 +567,7 @@ export default { 'sp.slash.show_scratchpad': '显示当前草稿板', 'sp.slash.clear_scratchpad': '清除当前草稿板', 'sp.slash.allow_api': '允许此对话进行 API 修改', + 'sp.slash.foreground': '在前台运行此任务以确保视觉兼容性', 'sp.slash.dangerously_skip_permissions': '全局禁用权限提示', 'sp.slash.compact': '压缩此对话上下文', 'sp.slash.verbose': '切换工具显示的详细或紧凑模式', diff --git a/src/firefox/src/ui/settings.js b/src/firefox/src/ui/settings.js index c0d24f3ae..684d630c6 100644 --- a/src/firefox/src/ui/settings.js +++ b/src/firefox/src/ui/settings.js @@ -52,7 +52,7 @@ import { ADDITIONAL_PROVIDER_UI } from '../providers/provider-catalog.js'; // Version shown in the subtitle. Kept here so it only needs one update per // release; the subtitle string itself is translated. -const EXT_VERSION = '25.9.7'; +const EXT_VERSION = '26.0.0'; const providersContainer = document.getElementById('providers'); const displaySettings = document.getElementById('display-settings'); diff --git a/src/firefox/src/ui/sidepanel.js b/src/firefox/src/ui/sidepanel.js index b763e4a19..4850a8270 100644 --- a/src/firefox/src/ui/sidepanel.js +++ b/src/firefox/src/ui/sidepanel.js @@ -499,6 +499,7 @@ const SLASH_COMMANDS = [ ], }, { value: '/allow-api', usage: '/allow-api [prompt]', descriptionKey: 'sp.slash.allow_api', action: 'enable', acceptsPayload: true }, + { value: '/foreground', usage: '/foreground [prompt]', descriptionKey: 'sp.slash.foreground', action: 'enable', acceptsPayload: true }, { value: '/dangerously-skip-permissions', usage: '/dangerously-skip-permissions [prompt]', descriptionKey: 'sp.slash.dangerously_skip_permissions', action: 'disable', acceptsPayload: true, outOfBand: true }, { value: '/compact', usage: '/compact [prompt]', descriptionKey: 'sp.slash.compact', action: 'compact', acceptsPayload: true }, { value: '/verbose', usage: '/verbose', descriptionKey: 'sp.slash.verbose', action: 'toggle', outOfBand: true }, @@ -3581,6 +3582,7 @@ async function adoptRestoredRunState(tabId, state) { tabId, requestId, mode, + foreground: runUi.foreground === true, }, { probeFirst: true, requireDurableSubmittedTurn: runUi.kind !== 'continue', @@ -3626,6 +3628,7 @@ async function applyActiveRunState(numericTabId, state) { || addMessage('assistant', ''); currentAssistantEl = runAssistantEl; runAssistantEl.dataset.runRequestId = String(runUi.requestId); + runAssistantEl.dataset.retryForeground = runUi.foreground === true ? 'true' : 'false'; if (runUi.runId) runAssistantEl.dataset.runId = String(runUi.runId); const lastRenderedSeq = Number(runAssistantEl.dataset.lastRenderedSeq || 0); const replayEvents = Array.isArray(runUi.events) ? runUi.events : []; @@ -3931,7 +3934,9 @@ function rebindContinueButtons() { document.querySelectorAll('.continue-btn').forEach(btn => { if (btn.dataset.bound) return; btn.dataset.bound = 'true'; - btn.addEventListener('click', continueAgent); + btn.addEventListener('click', () => continueAgent({ + foreground: btn.dataset.foreground === 'true', + })); }); } @@ -4911,7 +4916,10 @@ function resumeAfterSubscription(btn) { ? btn.dataset.resumeMode : agentMode; setMode(mode); - void continueAgent({ mode }); + void continueAgent({ + mode, + foreground: btn?.dataset?.resumeForeground === 'true', + }); } function rebindSubscribeButtons() { @@ -4950,6 +4958,7 @@ function retryPayloadFromButton(btn) { text, mode, apiMutationsAllowed: btn.dataset.retryApiMutationsAllowed === 'true', + foreground: btn.dataset.retryForeground === 'true', ...(btn.dataset.retrySourceGrounding === SELECTION_ONLY_SOURCE_GROUNDING ? { sourceGrounding: SELECTION_ONLY_SOURCE_GROUNDING } : {}), @@ -4984,6 +4993,7 @@ function bindErrorRetryButton(btn) { __retry: { mode: payload.mode, apiMutationsAllowed: payload.apiMutationsAllowed, + foreground: payload.foreground, ...(payload.sourceGrounding ? { sourceGrounding: payload.sourceGrounding } : {}), attachments: payload.attachments, }, @@ -5022,6 +5032,7 @@ function retryPayloadForRunAssistant(assistantEl) { ? assistantEl.dataset.runMode : agentMode, apiMutationsAllowed: assistantEl?.dataset.retryApiMutationsAllowed === 'true', + foreground: assistantEl?.dataset.retryForeground === 'true', ...(assistantEl?.dataset.retrySourceGrounding === SELECTION_ONLY_SOURCE_GROUNDING ? { sourceGrounding: SELECTION_ONLY_SOURCE_GROUNDING } : {}), @@ -6246,6 +6257,10 @@ async function parseSlashCommands(text, tabId = currentTabId, options = {}) { return payload; } + if (command.value === '/foreground') { + return payload; + } + if (command.value === '/dangerously-skip-permissions') { await browser.storage.local.set({ [PERMISSION_GATE_KEY]: false }).catch(() => {}); askBeforeConsequential = false; @@ -6582,6 +6597,9 @@ async function sendMessage(extraChatParams = {}) { const apiMutationsAllowedForSend = retryOptions ? !!retryOptions.apiMutationsAllowed : isApiMutationsAllowedForTab(tabId) || /^\/allow-api\b/i.test(text); + const foregroundForSend = retryOptions + ? !!retryOptions.foreground + : /^\/foreground\b/i.test(text); resetComposerHistoryNavigation(tabId); saveInputDraftForTab(tabId, ''); hideSlashCommandAutocomplete(); @@ -6640,6 +6658,7 @@ async function sendMessage(extraChatParams = {}) { text, mode: modeForSend, apiMutationsAllowed: apiMutationsAllowedForSend, + foreground: foregroundForSend, ...(sourceGrounding ? { sourceGrounding } : {}), attachments: attachmentsForSend, }; @@ -6659,6 +6678,7 @@ async function sendMessage(extraChatParams = {}) { assistantEl.dataset.runRequestId = requestId; assistantEl.dataset.runMode = modeForSend; assistantEl.dataset.retryApiMutationsAllowed = apiMutationsAllowedForSend ? 'true' : 'false'; + assistantEl.dataset.retryForeground = foregroundForSend ? 'true' : 'false'; assistantEl.dataset.retrySourceGrounding = sourceGrounding || ''; assistantEl.dataset.retryAttachmentCount = String(attachmentsForSend.length); assistantEl.dataset.lastRenderedSeq = '0'; @@ -6684,6 +6704,7 @@ async function sendMessage(extraChatParams = {}) { locale: getLocale(), intentFailureMessage: t('sp.plan.intent_unavailable'), apiMutationsAllowed: apiMutationsAllowedForSend, + foreground: foregroundForSend, ...(runCaptureDirective ? { runCapture: { kind: runCaptureDirective.kind, @@ -7024,7 +7045,11 @@ function handleAgentUpdateMessage(msg) { hideActivity(); // Don't gate on currentAssistantEl — race with sendResponse means it // may already be null by the time this fires. - showContinueButton(); + { + const retryPayload = activeRetryPayloadForRequest(eventTabId, msg.requestId) + || retryPayloadForRunAssistant(eventAssistantEl); + showContinueButton({ foreground: retryPayload?.foreground === true }); + } break; case 'warning': @@ -8271,6 +8296,9 @@ function renderSubscribeError(textEl, content, resumeMode = '') { resumeBtn.dataset.resumeMode = ['ask', 'act', 'dev'].includes(resumeMode) ? resumeMode : (textEl.closest('.message.assistant')?.dataset.runMode || agentMode); + resumeBtn.dataset.resumeForeground = textEl.closest('.message.assistant')?.dataset.retryForeground === 'true' + ? 'true' + : 'false'; resumeBtn.dataset.bound = 'true'; resumeBtn.addEventListener('click', () => resumeAfterSubscription(resumeBtn)); actions.appendChild(resumeBtn); @@ -8326,6 +8354,10 @@ function renderCostAllowanceError(textEl, content, resumeMode = '', resumeOption continueBtn.dataset.resumeMode = ['ask', 'act', 'dev'].includes(resumeMode) ? resumeMode : (textEl.closest('.message.assistant')?.dataset.runMode || agentMode); + continueBtn.dataset.resumeForeground = resumeOptions?.retryPayload?.foreground === true + || textEl.closest('.message.assistant')?.dataset.retryForeground === 'true' + ? 'true' + : 'false'; continueBtn.hidden = true; continueBtn.dataset.bound = 'true'; continueBtn.addEventListener('click', () => resumeAfterSubscription(continueBtn)); @@ -8354,6 +8386,7 @@ function configureRetryButton(btn, retryPayload) { btn.dataset.retryText = String(retryPayload.text || ''); btn.dataset.retryMode = retryPayload.mode || 'ask'; btn.dataset.retryApiMutationsAllowed = retryPayload.apiMutationsAllowed ? 'true' : 'false'; + btn.dataset.retryForeground = retryPayload.foreground ? 'true' : 'false'; btn.dataset.retrySourceGrounding = retryPayload.sourceGrounding === SELECTION_ONLY_SOURCE_GROUNDING ? SELECTION_ONLY_SOURCE_GROUNDING : ''; @@ -8494,7 +8527,7 @@ function displayMaxAgentSteps(value) { return Number.isFinite(n) && n > 0 ? String(Math.floor(n)) : '130'; } -function showContinueButton() { +function showContinueButton(options = {}) { document.querySelectorAll('.continue-bar').forEach(el => el.remove()); resetChatNavigation(); @@ -8507,13 +8540,19 @@ function showContinueButton() { messagesEl.appendChild(bar); scrollToBottom({ force: true }); - document.getElementById('btn-continue').addEventListener('click', continueAgent); + const continueBtn = document.getElementById('btn-continue'); + continueBtn.dataset.foreground = options?.foreground === true ? 'true' : 'false'; + continueBtn.dataset.bound = 'true'; + continueBtn.addEventListener('click', () => continueAgent({ + foreground: continueBtn.dataset.foreground === 'true', + })); } async function continueAgent(options = {}) { const tabId = currentTabId; const requestId = createRunRequestId(tabId); const modeForSend = ['ask', 'act', 'dev'].includes(options?.mode) ? options.mode : agentMode; + const foregroundForSend = options?.foreground === true; clearActiveChatPayloadForTab(tabId); setTabProcessing(tabId, true); @@ -8533,6 +8572,7 @@ async function continueAgent(options = {}) { assistantEl = addMessage('assistant', ''); assistantEl.dataset.runRequestId = requestId; assistantEl.dataset.runMode = modeForSend; + assistantEl.dataset.retryForeground = foregroundForSend ? 'true' : 'false'; assistantEl.dataset.lastRenderedSeq = '0'; currentAssistantEl = assistantEl; const questionEl = precedingUserMessage(assistantEl); @@ -8546,6 +8586,7 @@ async function continueAgent(options = {}) { tabId, requestId, mode: modeForSend, + foreground: foregroundForSend, }); if (res?.conversationId) { chatHistoryConversationIdsByTab.set(tabId, res.conversationId); diff --git a/test/run.js b/test/run.js index 2c176a38e..c99a712d8 100644 --- a/test/run.js +++ b/test/run.js @@ -1125,6 +1125,79 @@ test('remaining model-facing screenshot fallbacks apply redaction', () => { 'Firefox done verification should not embed base64 in the done result'); }); +test('Firefox done verifies inactive scheduled tabs before accepting completion', async () => { + const originalBrowser = globalThis.browser; + const tabId = 417; + let executeScriptCalls = 0; + const captures = []; + try { + globalThis.browser = { + tabs: { + get: async () => ({ + id: tabId, + active: false, + url: 'https://example.test/items/new', + title: 'Create item', + }), + executeScript: async () => { + executeScriptCalls += 1; + return [{ + url: 'https://example.test/items/new', + title: 'Create item', + openDialogCount: 1, + dialogTitles: ['Create item'], + visibleFormCount: 1, + relevantFormCount: 1, + formDescriptors: [{ + label: 'Create item', + relevant: true, + utility: false, + editableCount: 2, + submitCount: 1, + }], + liveRegionMessages: [], + successMessages: [], + }]; + }, + captureTab: async (capturedTabId, options) => { + captures.push({ tabId: capturedTabId, options }); + return 'data:image/png;base64,AA=='; + }, + sendMessage: async () => ({}), + }, + }; + + const agent = new AgentFx({ + getActive: () => ({ supportsVision: true }), + getVisionProvider: async () => null, + }); + agent.conversationModes.set(tabId, 'act'); + + const result = await agent.executeTool(tabId, 'done', { + summary: 'Created the item.', + outcome: 'success', + }); + + assert.equal(executeScriptCalls, 1, + 'inactive Firefox runs should still probe dialogs and forms before done'); + assert.deepEqual(captures, [{ + tabId, + options: { format: 'png', quality: 80 }, + }], 'inactive Firefox runs should capture their own tab for done verification'); + assert.equal(result.blockedDone, true, + 'an unfinished inactive-tab form should block a successful done result'); + + const source = fs.readFileSync(path.join(ROOT, 'src/firefox/src/agent/agent.js'), 'utf8'); + const doneStart = source.indexOf("if (name === 'done')"); + const doneEnd = source.indexOf('// Network & download tools', doneStart); + assert.doesNotMatch(source.slice(doneStart, doneEnd), /if \(tab\?\.active\)/, + 'Firefox done verification must not be gated on active-tab state'); + } finally { + if (originalBrowser === undefined) delete globalThis.browser; + else globalThis.browser = originalBrowser; + } +}); + test('firefox auto and media screenshot helpers redact model-facing data URLs', () => { const source = fs.readFileSync(path.join(ROOT, 'src/firefox/src/agent/agent.js'), 'utf8'); // Signature may include optional opts for Chrome call-site parity. @@ -5214,8 +5287,11 @@ test('CAPTCHA dialog parsing handles descendant-only labels and escaped quotes', } }); -test('CAPTCHA mutation preflight ignores hidden and off-viewport verification dialogs', async () => { - for (const [build, gate] of [['chrome', CaptchaGateCh], ['firefox', CaptchaGateFx]]) { +test('CAPTCHA preflight ignores hidden dialogs while ambiguous all-tree reads fail closed', async () => { + for (const [build, gate, AgentClass] of [ + ['chrome', CaptchaGateCh, AgentCh], + ['firefox', CaptchaGateFx, AgentFx], + ]) { const hiddenCases = [ captchaEl('div', { role: 'dialog', @@ -5236,8 +5312,73 @@ test('CAPTCHA mutation preflight ignores hidden and off-viewport verification di for (const dialog of hiddenCases) { await withCaptchaFakePage(build, [dialog], async () => { assert.equal(gate.detectChallengeDialogInPage(), null, `${build}: inactive dialog armed the mutation preflight`); + const agent = new AgentClass({}); + const observation = await agent._observeCaptchaChallenge( + 1, + 'get_accessibility_tree', + { + pageContent: 'dialog "Security verification" [ref_1]\n button "Dismiss" [ref_2]', + pageUrl: 'https://example.test/form', + }, + {}, + ); + assert.ok( + observation.gate, + `${build}: ambiguous all-tree challenge was cleared by non-unique hidden evidence`, + ); + assert.equal( + agent._captchaGateStates.has(1), + true, + `${build}: ambiguous all-tree challenge failed open`, + ); }); } + const visibilityOverrideNodes = [ + captchaEl('div', { 'data-test-visibility': 'hidden' }, [ + captchaEl('div', { + role: 'dialog', + innerText: 'Security verification', + 'data-test-visibility': 'visible', + }, [ + captchaEl('div', { + class: 'g-recaptcha', + 'data-sitekey': 'VISIBILITY_OVERRIDE_KEY', + 'data-test-visibility': 'visible', + }), + captchaEl('iframe', { + src: 'https://www.google.com/recaptcha/api2/anchor?k=VISIBILITY_OVERRIDE_KEY', + 'data-test-visibility': 'visible', + }), + ]), + ]), + ]; + await withCaptchaFakePage(build, visibilityOverrideNodes, async () => { + assert.equal( + gate.detectChallengeDialogInPage()?.label, + 'Security verification', + `${build}: visible descendant under visibility-hidden ancestor was suppressed`, + ); + const runtime = await import(pathToFileURL( + path.join(ROOT, `src/${build}/src/agent/captcha-frame-runtime.js`), + ).href); + const detected = runtime.detectCaptchaCandidatesInPage({ + window: { + location: globalThis.location, + innerWidth: globalThis.innerWidth, + innerHeight: globalThis.innerHeight, + getComputedStyle: globalThis.getComputedStyle, + }, + document: globalThis.document, + }); + const candidate = detected.candidates.find( + entry => entry.websiteKey === 'VISIBILITY_OVERRIDE_KEY', + ); + assert.equal( + candidate?.visible, + true, + `${build}: visible CAPTCHA widget under visibility-hidden ancestor was demoted`, + ); + }); await withCaptchaFakePage(build, [ captchaEl('div', { role: 'dialog', innerText: 'Verify you are a human' }), ], async () => { @@ -5258,6 +5399,196 @@ test('CAPTCHA mutation preflight ignores hidden and off-viewport verification di `${build}: active-gate failure context was ignored`, ); }); + + const hiddenStageNodes = [ + captchaEl('div', { + role: 'dialog', + innerText: 'Account details', + textContent: 'Account details Security verification', + }, [ + captchaEl('h2', { textContent: 'Account details' }), + captchaEl('section', { + hidden: true, + innerText: 'Security verification', + }, [ + captchaEl('h2', { textContent: 'Security verification' }), + captchaEl('div', { + class: 'g-recaptcha', + 'data-sitekey': 'INACTIVE_STAGE_KEY', + }), + ]), + ]), + ]; + await withCaptchaFakePage(build, hiddenStageNodes, async () => { + assert.equal( + gate.detectChallengeDialogInPage(), + null, + `${build}: hidden CAPTCHA stage labelled the visible modal as a challenge`, + ); + }); + const hiddenStageCandidate = await detectCaptchaOnFakePage(build, hiddenStageNodes); + assert.equal( + hiddenStageCandidate?.dialogAssociated, + false, + `${build}: hidden CAPTCHA stage associated its widget with the visible modal`, + ); + } +}); + +test('CAPTCHA visibility recheck sees shadow-rooted dialogs and keeps the gate on inconclusive scans', async () => { + for (const [build, gate, AgentClass] of [ + ['chrome', CaptchaGateCh, AgentCh], + ['firefox', CaptchaGateFx, AgentFx], + ]) { + // A visible challenge dialog rendered inside an open shadow root (the + // LinkedIn interop-outlet pattern): invisible to document.querySelectorAll + // but reported by the shadow-piercing accessibility tree. + const shadowDialogNodes = [ + captchaEl('div', { + id: 'interop-outlet', + shadow: [ + captchaEl('div', { role: 'dialog', innerText: 'Security verification' }, [ + captchaEl('h2', { textContent: 'Security verification' }), + ]), + ], + }), + ]; + await withCaptchaFakePage(build, shadowDialogNodes, async () => { + assert.equal( + gate.detectChallengeDialogInPage()?.label, + 'Security verification', + `${build}: shadow-rooted challenge dialog was invisible to the DOM scan`, + ); + const agent = new AgentClass({}); + const observation = await agent._observeCaptchaChallenge( + 1, + 'get_accessibility_tree', + { + pageContent: 'dialog "Security verification" [ref_1]\n button "Verify" [ref_2]', + pageUrl: 'https://example.test/form', + }, + {}, + ); + assert.ok(observation.gate, `${build}: shadow-rooted challenge from an all-tree read did not arm the gate`); + assert.equal(agent._captchaGateStates.has(1), true, `${build}: shadow-rooted challenge did not persist a gate`); + }); + + // A hidden challenge is disproof only for the same normalized label in + // the same frame. An unrelated hidden reCAPTCHA must not clear a visible + // tree-only challenge (for example one inside a closed shadow root). + await withCaptchaFakePage(build, [ + captchaEl('div', { + role: 'dialog', + innerText: 'reCAPTCHA', + hidden: true, + }), + ], async () => { + const agent = new AgentClass({}); + const recheck = await agent._detectChallengeDialogBeforeMutation(1, { + includeStatus: true, + }); + assert.deepEqual(recheck.hiddenChallenges, [{ + frameId: 0, + frameUrl: 'https://example.test/form', + label: 'reCAPTCHA', + normalizedLabel: 'recaptcha', + }], `${build}: hidden challenge evidence lost its frame or normalized label`); + + const observation = await agent._observeCaptchaChallenge( + 1, + 'get_accessibility_tree', + { + pageContent: 'dialog "Security verification" [ref_1]\n button "Verify" [ref_2]', + pageUrl: 'https://example.test/form', + }, + {}, + ); + assert.ok(observation.gate, `${build}: unrelated hidden challenge cleared the tree-only CAPTCHA gate`); + assert.equal(agent._captchaGateStates.has(1), true, `${build}: unrelated hidden challenge failed open`); + + const sameLabelAgent = new AgentClass({}); + const sameLabelObservation = await sameLabelAgent._observeCaptchaChallenge( + 1, + 'get_accessibility_tree', + { + pageContent: 'dialog "reCAPTCHA" [ref_1]\n button "Verify" [ref_2]', + pageUrl: 'https://example.test/form', + }, + {}, + ); + assert.ok( + sameLabelObservation.gate, + `${build}: same-label hidden template cleared the tree-only CAPTCHA gate`, + ); + assert.equal( + sameLabelAgent._captchaGateStates.has(1), + true, + `${build}: same-label hidden template failed open`, + ); + }); + + const crossFrameAgent = new AgentClass({}); + crossFrameAgent._detectChallengeDialogBeforeMutation = async () => ({ + challenge: null, + challengeHidden: true, + hiddenChallenges: [{ + frameId: 7, + frameUrl: 'https://challenge.example.test/hidden', + label: 'Security verification', + normalizedLabel: 'security verification', + }], + inspectionComplete: true, + }); + const crossFrameObservation = await crossFrameAgent._observeCaptchaChallenge( + 1, + 'get_accessibility_tree', + { + pageContent: 'dialog "Security verification" [ref_1]\n button "Verify" [ref_2]', + pageUrl: 'https://example.test/form', + captchaChallengeFrameId: 0, + }, + {}, + ); + assert.ok(crossFrameObservation.gate, `${build}: hidden challenge from another frame cleared the gate`); + assert.equal(crossFrameAgent._captchaGateStates.has(1), true, `${build}: cross-frame hidden challenge failed open`); + + // Hidden ancestry must still win even across the shadow boundary: a + // challenge dialog inside the shadow root of a hidden host stays inert. + await withCaptchaFakePage(build, [ + captchaEl('div', { + hidden: true, + shadow: [ + captchaEl('div', { role: 'dialog', innerText: 'Security verification' }), + ], + }), + ], async () => { + assert.equal( + gate.detectChallengeDialogInPage(), + null, + `${build}: shadow dialog under a hidden host armed the preflight`, + ); + }); + + // When script injection fails outright the scan proves nothing — the + // tree's observation must be kept rather than failing the gate open. + // (The "scan succeeds but cannot find the dialog at all" case is covered + // by the unsupported-Arkose routing test: absence is not disproof.) + await withCaptchaFakePage(build, [], async () => { + globalThis.chrome.scripting.executeScript = async () => { throw new Error('injection blocked'); }; + globalThis.browser.tabs.executeScript = async () => { throw new Error('injection blocked'); }; + const agent = new AgentClass({}); + const observation = await agent._observeCaptchaChallenge( + 1, + 'get_accessibility_tree', + { + pageContent: 'dialog "Security verification" [ref_1]\n button "Verify" [ref_2]', + pageUrl: 'https://example.test/form', + }, + {}, + ); + assert.ok(observation.gate, `${build}: inconclusive DOM scan cleared the tree challenge and failed open`); + assert.equal(agent._captchaGateStates.has(1), true, `${build}: inconclusive DOM scan did not persist a gate`); + }); } }); @@ -6421,6 +6752,79 @@ test('blank screenshot retry: skips retry when a blank page has no content signa } }); +test('Chrome background capture emulates focus without foregrounding and restores CDP state', async () => { + const originalSendCommand = cdpClientCh.sendCommand; + const calls = []; + cdpClientCh.sendCommand = async (tabId, method, params) => { + calls.push({ tabId, method, params }); + return {}; + }; + try { + const agent = new AgentCh({}); + const tabId = 42; + + agent._runningTabs.add(tabId); + assert.deepEqual( + await agent._preparePageForCapture(tabId), + { foreground: false, focusEmulated: true }, + ); + assert.deepEqual(calls, [{ + tabId, + method: 'Emulation.setFocusEmulationEnabled', + params: { enabled: true }, + }]); + + await agent._clearBackgroundFocusEmulation(tabId); + assert.deepEqual(calls.at(-1), { + tabId, + method: 'Emulation.setFocusEmulationEnabled', + params: { enabled: false }, + }); + + calls.length = 0; + agent._configureCapturePolicyForRun(tabId, { foreground: true }); + assert.deepEqual( + await agent._preparePageForCapture(tabId), + { foreground: true, focusEmulated: false }, + ); + assert.deepEqual(calls, [{ tabId, method: 'Page.bringToFront', params: undefined }]); + + calls.length = 0; + agent._configureCapturePolicyForRun(tabId, { cloudRun: true }); + await agent._preparePageForCapture(tabId); + assert.deepEqual(calls, [{ tabId, method: 'Page.bringToFront', params: undefined }], + 'managed cloud runs should retain foreground capture behavior'); + + calls.length = 0; + agent._runningTabs.delete(tabId); + agent._configureCapturePolicyForRun(tabId, {}); + assert.deepEqual( + await agent._preparePageForCapture(tabId), + { foreground: false, focusEmulated: false }, + ); + assert.deepEqual(calls, [], 'capture outside a run should not mutate focus state'); + } finally { + cdpClientCh.sendCommand = originalSendCommand; + } +}); + +test('local screenshot implementations do not foreground ordinary run tabs', () => { + const chromeAgent = fs.readFileSync(path.join(ROOT, 'src/chrome/src/agent/agent.js'), 'utf8'); + const firefoxAgent = fs.readFileSync(path.join(ROOT, 'src/firefox/src/agent/agent.js'), 'utf8'); + const firefoxScheduler = fs.readFileSync(path.join(ROOT, 'src/firefox/src/agent/scheduler.js'), 'utf8'); + + assert.equal( + [...chromeAgent.matchAll(/Page\.bringToFront/g)].length, + 2, + 'Chrome should mention bringToFront only in policy documentation and its foreground compatibility branch', + ); + assert.match(chromeAgent, /if \(this\._foregroundCaptureTabs\.has\(tabId\)\)[\s\S]*?'Page\.bringToFront'/); + assert.doesNotMatch(firefoxAgent, /captureVisibleTab/); + assert.match(firefoxAgent, /browser\.tabs\.captureTab\(tabId,/); + assert.doesNotMatch(firefoxScheduler, /active:\s*job\.source !== 'watch'/); + assert.match(firefoxScheduler, /tabs\.create\(\{ url: job\.target\.url, active: false \}\)/); +}); + // ──────────────────────────────────────────────────────────────────────── // Markdown link sanitizer // ──────────────────────────────────────────────────────────────────────── @@ -14354,6 +14758,12 @@ test('canonical slash parser handles flags, values, casing, termination, and har const scheduleList = chrome.parseSlashInvocation(' /SCHEDULE --LIST '); assert.equal(scheduleList.action, 'list'); assert.equal(chrome.slashInvocationIsOutOfBand(scheduleList), true); + for (const [label, runtime] of [['chrome', chrome], ['firefox', firefox]]) { + const foreground = runtime.parseSlashInvocation('/FoReGrOuNd finish the checkout'); + assert.equal(foreground.action, 'enable', `${label}: /foreground should select the compatibility action`); + assert.equal(foreground.payload, 'finish the checkout', `${label}: /foreground should preserve the task payload`); + assert.equal(foreground.command.acceptsPayload, true, `${label}: /foreground should accept an inline prompt`); + } for (const command of chrome.SLASH_COMMANDS.filter((item) => !item.unsupported)) { const invocation = chrome.parseSlashInvocation(`${command.value} --help`); assert.equal(invocation.action, 'help', `${command.value} --help should route to command help`); @@ -14434,6 +14844,57 @@ test('canonical slash parser handles flags, values, casing, termination, and har } }); +test('/foreground is a run-scoped compatibility override in both local builds', () => { + for (const [label, panelRel, backgroundRel, apiName] of [ + ['chrome', 'src/chrome/src/ui/sidepanel.js', 'src/chrome/src/background.js', 'chrome'], + ['firefox', 'src/firefox/src/ui/sidepanel.js', 'src/firefox/src/background.js', 'browser'], + ]) { + const panel = fs.readFileSync(path.join(ROOT, panelRel), 'utf8'); + const background = fs.readFileSync(path.join(ROOT, backgroundRel), 'utf8'); + + assert.match(panel, /const foregroundForSend = retryOptions[\s\S]*?\^\\\/foreground\\b/i, + `${label}: send flow should recognize /foreground before slash parsing`); + assert.match(panel, /foreground: foregroundForSend,[\s\S]*?sendRunWithReconnect\('chat_start',[\s\S]*?foreground: foregroundForSend,/, + `${label}: fresh runs and retries should carry the foreground bit`); + assert.match(panel, /case 'max_steps_reached':[\s\S]*?showContinueButton\(\{ foreground: retryPayload\?\.foreground === true \}\)/, + `${label}: max-step continuations should inherit foreground mode`); + assert.match(panel, /function showContinueButton\(options = \{\}\)[\s\S]*?continueBtn\.dataset\.foreground[\s\S]*?continueAgent\(\{[\s\S]*?foreground:/, + `${label}: persisted Continue buttons should retain foreground mode`); + assert.match(panel, /async function continueAgent\(options = \{\}\)[\s\S]*?const foregroundForSend = options\?\.foreground === true;[\s\S]*?dataset\.retryForeground = foregroundForSend[\s\S]*?sendRunWithReconnect\('continue_start',[\s\S]*?foreground: foregroundForSend,/, + `${label}: continuation runs should forward and retain foreground mode`); + assert.match(panel, /async function adoptRestoredRunState\([\s\S]*?sendRunWithReconnect\('continue_start', \{[\s\S]*?foreground: runUi\.foreground === true,/, + `${label}: remounted runs should recover foreground mode from the durable journal`); + assert.match(panel, /async function applyActiveRunState\([\s\S]*?dataset\.retryForeground = runUi\.foreground === true/, + `${label}: replayed max-step events should rebuild foreground continuation state`); + assert.match(background, new RegExp( + `async function activateForegroundCompatibilityTab\\(tabId\\)[\\s\\S]*?${apiName}\\.tabs\\.update\\(tabId, \\{ active: true \\}\\)[\\s\\S]*?${apiName}\\.windows\\.update\\(tab\\.windowId, \\{ focused: true \\}\\)`, + ), `${label}: compatibility mode should activate the run tab and focus its window`); + assert.match(background, /if \(msg\.foreground\) await activateForegroundCompatibilityTab\(tabId\);[\s\S]*?\.\.\.\(msg\.foreground \? \{ foreground: true \} : \{\}\)/, + `${label}: background dispatch should scope foreground behavior to the requested run`); + assert.match(background, /case 'continue':[\s\S]*?foreground: msg\.foreground === true,[\s\S]*?agent\.continueProcessing\([\s\S]*?\.\.\.\(msg\.foreground \? \{ foreground: true \} : \{\}\)/, + `${label}: background continuations should persist foreground mode and configure the agent`); + + for (const [caseName, nextCaseName] of [ + ['chat', 'chat_stream'], + ['chat_stream', 'continue'], + ['continue', 'clear_conversation'], + ]) { + const start = background.indexOf(`case '${caseName}': {`); + const end = background.indexOf(`case '${nextCaseName}': {`, start); + const body = background.slice(start, end); + const tryIndex = body.indexOf('try {'); + const activationIndex = body.indexOf('await activateForegroundCompatibilityTab(tabId)'); + const finallyIndex = body.lastIndexOf('finally {'); + const releaseIndex = body.lastIndexOf('releaseRunKeepalive();'); + assert.ok(start >= 0 && end > start, `${label}: ${caseName} handler should be extractable`); + assert.ok(tryIndex >= 0 && activationIndex > tryIndex, + `${label}: ${caseName} activation failures should enter run cleanup`); + assert.ok(finallyIndex > activationIndex && releaseIndex > finallyIndex, + `${label}: ${caseName} should release its keepalive after activation failures`); + } + } +}); + test('hidden trailing run-capture suffixes wrap normal prompts without entering slash help', () => { for (const [label, panelRel, backgroundRel, captureRel] of [ ['chrome', 'src/chrome/src/ui/sidepanel.js', 'src/chrome/src/background.js', 'src/chrome/src/run-capture.js'], @@ -14516,7 +14977,7 @@ test('hidden trailing run-capture suffixes wrap normal prompts without entering assert.match(host, /const filename = recordingState\.filename \|\| `webbrain-recording-\$\{stamp\}\.webm`;/, 'chrome: custom filename should override only the default timestamped recording name'); }); -test('run screenshot capture reactivates the originating tab before saving', async () => { +test('run screenshot capture keeps Firefox backgrounded while Chrome retains its diagnostic activation path', async () => { for (const [label, captureRel] of [ ['chrome', 'src/chrome/src/run-capture.js'], ['firefox', 'src/firefox/src/run-capture.js'], @@ -14537,6 +14998,10 @@ test('run screenshot capture reactivates the originating tab before saving', asy captures.push({ windowId, options }); return 'data:image/png;base64,capture'; }, + captureTab: async (tabId, options) => { + captures.push({ tabId, options }); + return 'data:image/png;base64,capture'; + }, }, downloads: { download: async (options) => { @@ -14553,8 +15018,13 @@ test('run screenshot capture reactivates the originating tab before saving', asy const runtime = await loadRunCaptureRuntime(captureRel); const result = await runtime.captureAndSaveRunScreenshot(api, 42, 'run-after.png'); - assert.deepEqual(updates, [{ tabId: 42, changes: { active: true } }], `${label}: inactive run tab should be reactivated`); - assert.deepEqual(captures, [{ windowId: 7, options: { format: 'png' } }], `${label}: capture should use the reactivated tab's window`); + if (label === 'chrome') { + assert.deepEqual(updates, [{ tabId: 42, changes: { active: true } }], 'chrome: diagnostic capture should retain its existing activation path'); + assert.deepEqual(captures, [{ windowId: 7, options: { format: 'png' } }], 'chrome: diagnostic capture should use the activated tab window'); + } else { + assert.deepEqual(updates, [], 'firefox: inactive run tab should not be activated for capture'); + assert.deepEqual(captures, [{ tabId: 42, options: { format: 'png' } }], 'firefox: capture should target the run tab directly'); + } assert.equal(downloads[0].filename, 'run-after.png', `${label}: after screenshot should be saved under the requested filename`); assert.deepEqual(result, { filename: '/Users/test/Downloads/WebBrain/run-after.png', @@ -15437,7 +15907,7 @@ test('sidepanel subscribe error card clears DOM without HTML reinterpretation', assert.match(body, /resumeBtn\.textContent = t\('sp\.subscribe\.resume'\);/, `${label}: subscribe card should render a localized resume action`); assert.match(body, /resumeBtn\.dataset\.resumeMode = \['ask', 'act', 'dev'\]\.includes\(resumeMode\)[\s\S]*?\? resumeMode[\s\S]*?: \(textEl\.closest\('\.message\.assistant'\)\?\.dataset\.runMode \|\| agentMode\);/, `${label}: resume action should prefer an explicitly captured failed run mode`); assert.match(body, /resumeAfterSubscription\(resumeBtn\)/, `${label}: subscribe card should use the shared resume handler`); - assert.match(panel, /function resumeAfterSubscription\(btn\) \{[\s\S]*?const mode = \['ask', 'act', 'dev'\]\.includes\(btn\?\.dataset\?\.resumeMode\)[\s\S]*?setMode\(mode\);[\s\S]*?continueAgent\(\{ mode \}\);[\s\S]*?\}/, `${label}: subscribe resume should synchronize the visible mode before continuing`); + assert.match(panel, /function resumeAfterSubscription\(btn\) \{[\s\S]*?const mode = \['ask', 'act', 'dev'\]\.includes\(btn\?\.dataset\?\.resumeMode\)[\s\S]*?setMode\(mode\);[\s\S]*?continueAgent\(\{[\s\S]*?mode,[\s\S]*?foreground: btn\?\.dataset\?\.resumeForeground === 'true',[\s\S]*?\}\);[\s\S]*?\}/, `${label}: subscribe resume should synchronize the visible mode and preserve foreground mode before continuing`); assert.match(panel, /btn\.addEventListener\('click', \(\) => resumeAfterSubscription\(btn\)\);/, `${label}: restored subscribe cards should use the shared resume handler`); const errorUpdateStart = panel.indexOf('function renderAgentErrorUpdate('); const errorUpdateEnd = panel.indexOf('\n}\n\nfunction rebindRestoredMessageControls', errorUpdateStart); @@ -16191,7 +16661,7 @@ test('sidepanel rebinds interactive controls after restoring serialized tab chat assert.match(panel, /function bindMessageCopyButton\(btn\) \{[\s\S]*?addEventListener\('click'/, `${label}: message copy rebinding should attach a click listener`); assert.match(panel, /btn\.__wbCopyBound = true;/, `${label}: message copy binding should use an in-memory flag so restored HTML can rebind`); assert.match(panel, /document\.querySelectorAll\('\.code-copy-btn'\)[\s\S]*?addEventListener\('click'/, `${label}: code copy buttons should be rebound`); - assert.match(panel, /function rebindContinueButtons\(\) \{[\s\S]*?document\.querySelectorAll\('\.continue-btn'\)[\s\S]*?addEventListener\('click', continueAgent\)/, `${label}: restored Continue buttons should be rebound`); + assert.match(panel, /function rebindContinueButtons\(\) \{[\s\S]*?document\.querySelectorAll\('\.continue-btn'\)[\s\S]*?addEventListener\('click', \(\) => continueAgent\(\{[\s\S]*?foreground: btn\.dataset\.foreground === 'true'/, `${label}: restored Continue buttons should be rebound with their foreground mode`); assert.match(panel, /function rebindClarifyCards\(\) \{[\s\S]*?document\.querySelectorAll\('\.clarify-card'\)[\s\S]*?submitClarify\(card, tabId, clarifyId/, `${label}: restored clarify cards should be rebound`); assert.match(panel, /function rebindScheduleComposers\(\) \{[\s\S]*?document\.querySelectorAll\('form\.schedule-composer'\)[\s\S]*?bindScheduleComposer\(form\)/, `${label}: restored schedule composers should be rebound`); assert.match(panel, /function rebindRestoredMessageControls\(\) \{[\s\S]*?rebindCopyButtons\(\);[\s\S]*?rebindContinueButtons\(\);[\s\S]*?rebindClarifyCards\(\);[\s\S]*?rebindScheduleComposers\(\);[\s\S]*?\}/, `${label}: restored tab chat should rebind all durable message controls`); @@ -20246,7 +20716,7 @@ test('sidepanel long replies use reading-first turn navigation', () => { ); assert.match( panel, - /function showContinueButton\(\) \{[\s\S]*?resetChatNavigation\(\);[\s\S]*?messagesEl\.appendChild\(bar\);[\s\S]*?scrollToBottom\(\{ force: true \}\);/, + /function showContinueButton\(options = \{\}\) \{[\s\S]*?resetChatNavigation\(\);[\s\S]*?messagesEl\.appendChild\(bar\);[\s\S]*?scrollToBottom\(\{ force: true \}\);/, `${label}: the blocking Continue prompt should replace turn navigation instead of being covered by it`, ); assert.match( @@ -21609,16 +22079,19 @@ test('Chrome ScheduledJobManager keeps alarm-triggered runs alive until completi assert.equal(keepAliveStops, 1, 'keepalive should stop after the run settles'); }); -test('Firefox ScheduledJobManager activates URL-target tabs before running', async () => { +test('Firefox ScheduledJobManager keeps URL-target tabs inactive while running', async () => { const now = Date.UTC(2026, 0, 1, 12, 0, 0); let h; + let observedInactiveTarget = false; h = makeSchedulerHarness(SchedulerFx, { now, processMessage: async (tabId) => { - assert.equal(h.tabs.get(tabId)?.active, true, 'Firefox URL-target scheduled runs need an active tab for screenshots'); + assert.equal(h.tabs.get(tabId)?.active, false, 'Firefox URL-target scheduled runs should stay in the background'); + observedInactiveTarget = true; return 'done'; }, }); + h.tabs.set(77, { ...h.tabs.get(77), active: false }); const created = await h.manager.createTaskJob({ tabId: 77, conversationId: 'conv-1', @@ -21634,8 +22107,7 @@ test('Firefox ScheduledJobManager activates URL-target tabs before running', asy }); await h.manager.handleAlarm(h.alarmName(created.jobId)); - const helperTab = [...h.tabs.values()].find((tab) => tab.url === 'https://example.org/app'); - assert.equal(helperTab?.active, true, 'new Firefox URL helper tabs should be active'); + assert.equal(observedInactiveTarget, true, 'Firefox should run the scheduled task against its inactive helper tab'); }); test('ScheduledJobManager restoreAlarms requeues stranded transient jobs', async () => { @@ -25272,7 +25744,7 @@ test('user full-page screenshot responses preserve compositor fallback warnings' warning: 'Full-page screenshot assembly failed (canvas too large). Showing the first captured tile instead.', }); const agent = new AgentCh({}); - agent._bringToFrontForCapture = async () => {}; + agent._preparePageForCapture = async () => {}; agent._withIndicatorsHidden = async (_tabId, capture) => capture(); const result = await agent.captureFullPageScreenshotForUser(42); @@ -25285,6 +25757,135 @@ test('user full-page screenshot responses preserve compositor fallback warnings' } }); +test('Chrome full-page screenshot paths reject blank background captures after retries', async () => { + const originalChrome = globalThis.chrome; + const originalAttach = cdpClientCh.attach; + const originalCapture = cdpClientCh.captureFullPageScreenshot; + const originalDelays = AgentCh.BLANK_SCREENSHOT_RETRY_DELAYS_MS; + let captureCalls = 0; + try { + globalThis.chrome = { + ...(originalChrome || {}), + tabs: { + ...(originalChrome?.tabs || {}), + get: async (tabId) => ({ + id: tabId, + active: false, + url: 'https://example.test/background', + }), + }, + }; + cdpClientCh.attach = async () => ({ attached: true }); + cdpClientCh.captureFullPageScreenshot = async () => { + captureCalls += 1; + return { data: 'Ymxhbms=' }; + }; + AgentCh.BLANK_SCREENSHOT_RETRY_DELAYS_MS = [0]; + + const agent = new AgentCh({ + getActive: () => ({ supportsVision: true }), + getVisionProvider: async () => null, + }); + agent._preparePageForCapture = async () => {}; + agent._withIndicatorsHidden = async (_tabId, capture) => capture(); + agent._captureViewportProbe = async () => ({ + readyState: 'complete', + documentTextChars: 200, + visibleTextChars: 100, + domNodes: 50, + imageCount: 0, + scrollHeight: 1200, + innerHeight: 800, + }); + agent._analyzeScreenshotBlankness = async () => ({ + blank: true, + reason: 'near-all-white frame', + meanLuma: 255, + lumaStdDev: 0, + whiteRatio: 1, + blackRatio: 0, + }); + + const userResult = await agent.captureFullPageScreenshotForUser(42); + assert.deepEqual(userResult, { + ok: false, + error: 'Background full-page screenshot remained blank after retries', + }); + + const toolResult = await agent.executeTool(42, 'full_page_screenshot', {}); + assert.deepEqual(toolResult, { + success: false, + error: 'Full page screenshot failed: Background full-page screenshot remained blank after retries', + }); + assert.equal(captureCalls, 4, 'each full-page path should retry once before rejecting the blank frame'); + } finally { + AgentCh.BLANK_SCREENSHOT_RETRY_DELAYS_MS = originalDelays; + cdpClientCh.attach = originalAttach; + cdpClientCh.captureFullPageScreenshot = originalCapture; + if (originalChrome === undefined) delete globalThis.chrome; + else globalThis.chrome = originalChrome; + } +}); + +test('Chrome full-page blank guard ignores document length as the lone content signal', async () => { + const originalChrome = globalThis.chrome; + const originalAttach = cdpClientCh.attach; + const originalCapture = cdpClientCh.captureFullPageScreenshot; + let captureCalls = 0; + try { + globalThis.chrome = { + ...(originalChrome || {}), + tabs: { + ...(originalChrome?.tabs || {}), + get: async (tabId) => ({ + id: tabId, + active: false, + url: 'https://example.test/uniform-long-page', + }), + }, + }; + cdpClientCh.attach = async () => ({ attached: true }); + cdpClientCh.captureFullPageScreenshot = async () => { + captureCalls += 1; + return { data: 'dW5pZm9ybQ==' }; + }; + + const agent = new AgentCh({}); + agent._preparePageForCapture = async () => {}; + agent._withIndicatorsHidden = async (_tabId, capture) => capture(); + agent._captureViewportProbe = async () => ({ + readyState: 'complete', + documentTextChars: 0, + visibleTextChars: 0, + domNodes: 20, + imageCount: 0, + scrollHeight: 5000, + innerHeight: 800, + }); + agent._analyzeScreenshotBlankness = async () => ({ + blank: true, + reason: 'near-all-white frame', + meanLuma: 255, + lumaStdDev: 0, + whiteRatio: 1, + blackRatio: 0, + }); + + const result = await agent.captureFullPageScreenshotForUser(42); + assert.deepEqual(result, { + ok: true, + dataUrl: 'data:image/png;base64,dW5pZm9ybQ==', + warning: null, + }); + assert.equal(captureCalls, 1, 'a uniform long page should not retry from scroll length alone'); + } finally { + cdpClientCh.attach = originalAttach; + cdpClientCh.captureFullPageScreenshot = originalCapture; + if (originalChrome === undefined) delete globalThis.chrome; + else globalThis.chrome = originalChrome; + } +}); + test('user full-page screenshots apply adapter capture policy without LLM adapter injection', async () => { const originalChrome = globalThis.chrome; const originalAttach = cdpClientCh.attach; @@ -25308,7 +25909,7 @@ test('user full-page screenshots apply adapter capture policy without LLM adapte }; const agent = new AgentCh({}); agent.useSiteAdapters = false; - agent._bringToFrontForCapture = async () => {}; + agent._preparePageForCapture = async () => {}; agent._withIndicatorsHidden = async (_tabId, capture) => capture(); const result = await agent.captureFullPageScreenshotForUser(42); @@ -25358,7 +25959,7 @@ test('agent full-page screenshot tool applies adapter capture policy without LLM }); agent.useSiteAdapters = false; agent.screenshotRedaction = true; - agent._bringToFrontForCapture = async () => {}; + agent._preparePageForCapture = async () => {}; agent._withIndicatorsHidden = async (_tabId, capture) => capture(); agent._shrinkImageForBudget = async (dataUrl) => ({ dataUrl, width: 400, height: 1500 }); agent._redactScreenshotDataUrl = async (_tabId, dataUrl, options) => { @@ -49932,12 +50533,13 @@ test('run UI journal: concurrent tabs, bounded replay, terminal snapshots, and s test('run UI journal: resumed requests preserve sequence and replay boundaries', () => { for (const [label, Journal] of [['chrome', RunUiJournalCh], ['firefox', RunUiJournalFx]]) { const journal = new Journal(); - journal.begin(7, 'resume-request'); + journal.begin(7, 'resume-request', { foreground: true }); journal.record(7, 'resume-request', 'thinking', { content: 'Before restart' }, 'run-before'); journal.record(7, 'resume-request', 'plan_review', { planId: 'plan-before' }, 'run-before'); journal.acknowledge(7, 'resume-request', 1); const before = journal.get(7); + assert.equal(before.foreground, true, `${label}: foreground mode should be part of the durable run snapshot`); assert.equal(before.seq, 2, `${label}: fixture should start above sequence zero`); assert.equal(before.ackedSeq, 1, `${label}: fixture should retain an acknowledged replay boundary`); @@ -49945,6 +50547,7 @@ test('run UI journal: resumed requests preserve sequence and replay boundaries', restarted.restore(7, structuredClone(before)); const resumed = restarted.resume(7, 'resume-request'); assert.ok(resumed, `${label}: matching request should resume its existing journal`); + assert.equal(resumed.foreground, true, `${label}: restored continuations should retain foreground mode`); assert.equal(resumed.seq, 2, `${label}: resume must preserve the prior sequence`); assert.equal(resumed.ackedSeq, 1, `${label}: resume must preserve the acknowledged replay boundary`); assert.equal(resumed.truncatedBeforeSeq, 1, `${label}: resume must preserve the compaction boundary`); @@ -49958,6 +50561,9 @@ test('run UI journal: resumed requests preserve sequence and replay boundaries', const next = restarted.record(7, 'resume-request', 'thinking', { content: 'After restart' }, 'run-after'); assert.equal(next.seq, 3, `${label}: first resumed event should advance beyond the old sequence`); + + const backgroundRun = new Journal().begin(8, 'background-request'); + assert.equal(backgroundRun.foreground, false, `${label}: ordinary runs should remain background by default`); assert.deepEqual( restarted.get(7).events.map(event => event.seq), [2, 3], @@ -53290,6 +53896,21 @@ function captchaEl(tag, attrs = {}, children = []) { querySelectorAll: (sel) => captchaMatchAll(children, sel), }; for (const child of children) child.parentElement = el; + if (Array.isArray(attrs.shadow)) { + // Open shadow root: content is reachable only via el.shadowRoot, never + // through document/parent querySelectorAll — mirroring the real DOM. + const shadowChildren = attrs.shadow; + el.shadowRoot = { + host: el, + children: shadowChildren, + querySelector: (sel) => captchaMatchAll(shadowChildren, sel)[0] || null, + querySelectorAll: (sel) => captchaMatchAll(shadowChildren, sel), + }; + for (const child of shadowChildren) { + child.parentElement = null; + child.getRootNode = () => el.shadowRoot; + } + } el.contains = (candidate) => { if (candidate === el) return true; return children.some(child => child === candidate || child.contains?.(candidate)); @@ -53304,6 +53925,7 @@ function captchaMatchAll(roots, selectorList) { })(roots); const selectors = String(selectorList).split(',').map(s => s.trim()).filter(Boolean); const matchesOne = (el, sel) => { + if (sel === '*') return true; // Tokenize into tag / .class / [attr…] parts. Splitting naively on "." // would break selectors like script[src*="recaptcha/api.js"]. const parts = sel.match(/\[[^\]]*\]|\.[\w-]+|^[a-zA-Z][\w-]*/g) || []; @@ -53351,7 +53973,7 @@ async function withCaptchaFakePage(build, nodes, callback) { globalThis.innerHeight = 720; globalThis.getComputedStyle = (element) => ({ display: element.hidden ? 'none' : 'block', - visibility: 'visible', + visibility: element.getAttribute?.('data-test-visibility') || 'visible', opacity: '1', }); // The detector reaches for the extension API by name; give each build the