From dbfb2eaf937ac62f813952cac1d81b9c0c93910a Mon Sep 17 00:00:00 2001 From: fkb032 <249513614+fkb032@users.noreply.github.com> Date: Thu, 7 May 2026 10:53:39 -0700 Subject: [PATCH 01/16] fix: stabilize peekaboo macincloud capture --- .github/workflows/peekaboo-e2e.yml | 57 +++++++++++++++++-- apps/native/src-tauri/src/e2e_runtime.rs | 10 ++++ .../src-tauri/src/evolve/search_packages.rs | 19 ++++--- apps/native/src-tauri/src/main.rs | 50 ++++++++++++++++ .../widget/controls/directory-picker.test.tsx | 2 + .../peekaboo-workflow-contract-self-test.mjs | 7 ++- 6 files changed, 132 insertions(+), 13 deletions(-) diff --git a/.github/workflows/peekaboo-e2e.yml b/.github/workflows/peekaboo-e2e.yml index 70e649af9..03189c534 100644 --- a/.github/workflows/peekaboo-e2e.yml +++ b/.github/workflows/peekaboo-e2e.yml @@ -456,19 +456,69 @@ jobs: trusted_secret_scan_file="$FETCH_REPORT_DIR/trusted-secret-scan.json" trusted_secret_scan_tmp="$(mktemp "${RUNNER_TEMP:-/tmp}/nixmac-trusted-secret-scan.XXXXXX.json")" node --input-type=module - "$FETCH_REPORT_DIR" > "$trusted_secret_scan_tmp" <<'NODE' - import { readdirSync, readFileSync, lstatSync } from 'node:fs'; + import { closeSync, lstatSync, openSync, readSync, readdirSync } from 'node:fs'; import path from 'node:path'; const root = process.argv[2]; const secretPattern = /(?:sk-[A-Za-z0-9_-]{16,}|xai-[A-Za-z0-9_-]{16,}|gh[opsu]_[A-Za-z0-9_]{16,}|github_pat_[A-Za-z0-9_]{22,}|xox[abprs]-[A-Za-z0-9-]{16,}|AKIA[0-9A-Z]{16}|Bearer\s+[A-Za-z0-9._-]{16,}|(?:OPENROUTER|OPENAI|ANTHROPIC|GROQ|XAI|MISTRAL|COHERE|GITHUB|SLACK|AWS)_API_KEY=(?!\[REDACTED\])[^\s"'<>]+)/i; + const secretPatternGlobal = new RegExp(secretPattern.source, `${secretPattern.flags.replace('g', '')}g`); + const TEXT_SNIFF_BYTES = 8192; + const TEXT_SCAN_CHUNK_BYTES = 65536; + // Larger than real token lengths, so chunk-boundary matches stay bounded-memory. + const TEXT_SCAN_OVERLAP_CHARS = 2048; const violations = []; let scannedPaths = 0; let scannedFiles = 0; + function readSample(full, maxBytes = TEXT_SNIFF_BYTES) { + const fd = openSync(full, 'r'); + try { + const buffer = Buffer.allocUnsafe(maxBytes); + const bytesRead = readSync(fd, buffer, 0, maxBytes, 0); + return buffer.subarray(0, bytesRead); + } finally { + closeSync(fd); + } + } function looksLikeTextFile(full) { - const sample = readFileSync(full).subarray(0, 8192); + const sample = readSample(full); if (sample.includes(0)) return false; const suspiciousControlBytes = [...sample].filter((byte) => byte < 9 || (byte > 13 && byte < 32)).length; return sample.length === 0 || suspiciousControlBytes / sample.length < 0.02; } + function scanChunkForSecret(text, hasPotentialNextChunk) { + secretPatternGlobal.lastIndex = 0; + for (const match of text.matchAll(secretPatternGlobal)) { + const matchEnd = (match.index ?? 0) + match[0].length; + if (hasPotentialNextChunk && matchEnd === text.length) { + const maybeValue = match[0].split('=').at(-1) ?? ''; + if ('[REDACTED]'.startsWith(maybeValue)) { + return { status: 'deferred-redaction', carryStart: match.index ?? 0 }; + } + } + return { status: 'match', carryStart: null }; + } + return { status: 'none', carryStart: null }; + } + function textFileHasSecret(full) { + const fd = openSync(full, 'r'); + const buffer = Buffer.allocUnsafe(TEXT_SCAN_CHUNK_BYTES); + let position = 0; + let carry = ''; + try { + for (;;) { + const bytesRead = readSync(fd, buffer, 0, buffer.length, position); + if (bytesRead === 0) return false; + const text = carry + buffer.subarray(0, bytesRead).toString('utf8'); + const scanStatus = scanChunkForSecret(text, bytesRead === buffer.length); + if (scanStatus.status === 'match') return true; + carry = scanStatus.status === 'deferred-redaction' + ? text.slice(scanStatus.carryStart) + : text.slice(-TEXT_SCAN_OVERLAP_CHARS); + position += bytesRead; + } + } finally { + closeSync(fd); + } + } function walk(dir) { for (const entry of readdirSync(dir)) { const full = path.join(dir, entry); @@ -481,8 +531,7 @@ jobs: walk(full); } else if (stat.isFile() && looksLikeTextFile(full)) { scannedFiles += 1; - const text = readFileSync(full, 'utf8'); - if (secretPattern.test(text)) violations.push(`content:${relativePath}`); + if (textFileHasSecret(full)) violations.push(`content:${relativePath}`); } } } diff --git a/apps/native/src-tauri/src/e2e_runtime.rs b/apps/native/src-tauri/src/e2e_runtime.rs index 2040fe1eb..df2d1ad8c 100644 --- a/apps/native/src-tauri/src/e2e_runtime.rs +++ b/apps/native/src-tauri/src/e2e_runtime.rs @@ -6,15 +6,23 @@ //! debug builds can still receive deterministic test controls. Release builds //! ignore the file. +#[cfg(debug_assertions)] use serde::Deserialize; +#[cfg(debug_assertions)] use std::collections::HashMap; +#[cfg(debug_assertions)] use std::fs; +#[cfg(debug_assertions)] use std::path::PathBuf; +#[cfg(debug_assertions)] use std::time::{SystemTime, UNIX_EPOCH}; +#[cfg(debug_assertions)] const RUNTIME_FILE_NAME: &str = "e2e-runtime.json"; +#[cfg(debug_assertions)] const BUNDLE_ID: &str = "com.darkmatter.nixmac"; +#[cfg(debug_assertions)] #[derive(Debug, Deserialize)] struct E2eRuntimeFile { #[serde(rename = "schemaVersion")] @@ -26,6 +34,7 @@ struct E2eRuntimeFile { values: HashMap, } +#[cfg(debug_assertions)] fn runtime_file_path() -> Option { let home = dirs::home_dir()?; Some( @@ -36,6 +45,7 @@ fn runtime_file_path() -> Option { ) } +#[cfg(debug_assertions)] fn now_unix() -> Option { SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/apps/native/src-tauri/src/evolve/search_packages.rs b/apps/native/src-tauri/src/evolve/search_packages.rs index dce77905d..a05135d2c 100644 --- a/apps/native/src-tauri/src/evolve/search_packages.rs +++ b/apps/native/src-tauri/src/evolve/search_packages.rs @@ -88,7 +88,11 @@ fn process_search_output( if let Some(value) = parsed.as_object() { for (attr_path, pkg) in value { - let name = attr_path.split('.').last().unwrap_or(attr_path).to_string(); + let name = attr_path + .split('.') + .next_back() + .unwrap_or(attr_path) + .to_string(); let package_type = if let Some(classifier) = package_classifier { classifier(&name) } else { @@ -138,7 +142,10 @@ fn process_channel_results( .split('.') .next_back() .unwrap_or(&result.attr_path); - if !structured.iter().any(|item| item.attr_path == result.attr_path) { + if !structured + .iter() + .any(|item| item.attr_path == result.attr_path) + { structured.push(result); } } @@ -242,11 +249,7 @@ fn classify_derivation(drv: &str) -> SearchResultInstallTarget { /// (Homebrew Cask-like) or a CLI / nix-native package. fn classify_package(channel: &str, attr_path: &str) -> SearchResultInstallTarget { let mut cmd = Command::new("nix"); - cmd.args(&[ - "derivation", - "show", - &format!("{}#{}", channel, attr_path), - ]); + cmd.args(["derivation", "show", &format!("{}#{}", channel, attr_path)]); let output = match cmd.output() { Ok(output) => output, @@ -335,7 +338,7 @@ mod tests { channel: "test-channel".to_string(), version: "13.2".to_string(), description: "Extensible package for writing and formatting TeX files in GNU Emacs and XEmacs".to_string(), - install_via: SearchResultInstallTarget::Either, + install_via: SearchResultInstallTarget::Either, })), ("empty", 0, None )]; let fake_package_classifier = |_package_name: &str| SearchResultInstallTarget::Either; diff --git a/apps/native/src-tauri/src/main.rs b/apps/native/src-tauri/src/main.rs index 77d588ff9..024979b7b 100644 --- a/apps/native/src-tauri/src/main.rs +++ b/apps/native/src-tauri/src/main.rs @@ -164,6 +164,38 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" clientTimestampUnixMs: Date.now(), }).catch(() => {}); }; + const getCssValue = (style, property) => { + if (!style) return null; + const value = style.getPropertyValue(property); + return value === "" ? null : value; + }; + const captureStyleProbe = (label) => { + try { + const root = document.getElementById("root"); + const shell = root?.firstElementChild ?? null; + const htmlStyle = window.getComputedStyle(document.documentElement); + const bodyStyle = document.body ? window.getComputedStyle(document.body) : null; + const shellStyle = shell ? window.getComputedStyle(shell) : null; + logCaptureBreadcrumb( + `e2e-capture-style-${label}`, + JSON.stringify({ + captureMode, + capturePaint: document.documentElement.dataset.nixmacE2eCapturePaint ?? null, + rootChildren: root?.childElementCount ?? null, + bodyChildren: document.body?.childElementCount ?? null, + shellClassName: typeof shell?.className === "string" ? shell.className : null, + htmlBackgroundColor: htmlStyle.backgroundColor, + bodyBackgroundColor: bodyStyle?.backgroundColor ?? null, + shellBackgroundColor: shellStyle?.backgroundColor ?? null, + shellBackdropFilter: getCssValue(shellStyle, "backdrop-filter"), + shellWebkitBackdropFilter: getCssValue(shellStyle, "-webkit-backdrop-filter"), + shellOpacity: shellStyle?.opacity ?? null, + }), + ); + } catch (error) { + logCaptureBreadcrumb("e2e-capture-style-probe-error", String(error)); + } + }; const applyCaptureBackground = () => { document.documentElement.classList.add("dark"); document.documentElement.dataset.nixmacE2eCapture = captureMode; @@ -186,6 +218,12 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" html[data-nixmac-e2e-capture="${captureMode}"] .bg-zinc-900\\/95 { background-color: ${captureBackground} !important; } + html[data-nixmac-e2e-capture="${captureMode}"] *, + html[data-nixmac-e2e-capture="${captureMode}"] *::before, + html[data-nixmac-e2e-capture="${captureMode}"] *::after { + -webkit-backdrop-filter: none !important; + backdrop-filter: none !important; + } `; document.head.appendChild(style); } @@ -199,7 +237,19 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" bodyChildren: document.body?.childElementCount ?? null, }), ); + captureStyleProbe("initial-raf"); }); + window.addEventListener( + "nixmac:app-mounted", + () => { + requestAnimationFrame(() => { + requestAnimationFrame(() => { + captureStyleProbe("app-mounted-plus-2raf"); + }); + }); + }, + { once: true }, + ); }; if (document.head) { applyCaptureBackground(); diff --git a/apps/native/src/components/widget/controls/directory-picker.test.tsx b/apps/native/src/components/widget/controls/directory-picker.test.tsx index 75c7f9b04..8ceb33220 100644 --- a/apps/native/src/components/widget/controls/directory-picker.test.tsx +++ b/apps/native/src/components/widget/controls/directory-picker.test.tsx @@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useWidgetStore } from "@/stores/widget-store"; import type { SetDirResult } from "@/types/shared"; import { DirectoryPicker } from "@/components/widget/controls/directory-picker"; +import type { SetDirResult } from "@/types/shared"; // --------------------------------------------------------------------------- // Mocks @@ -234,6 +235,7 @@ describe("", () => { evolveState: {} as never, hosts: ["mbp"], }); + mockListHosts.mockResolvedValue(["mbp"]); render(); typeAndBlur(screen.getByLabelText("Config directory"), "/Users/me/.darwin"); diff --git a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs index 971b1af69..a7ba693cf 100644 --- a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs +++ b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs @@ -117,7 +117,10 @@ assert.match(proof, /state_secret_scan_passed="\$\(jq -r '\(\.peekaboo\.secretSc assert.match(proof, /ServerAliveInterval=15[\s\S]*run-peekaboo-suite --allow-cleanup/, 'long-running Peekaboo SSH run must use keepalives'); assert.match(proof, /trusted-secret-scan\.json[\s\S]*mktemp[\s\S]*secretPattern[\s\S]*github_pat_[\s\S]*lstatSync[\s\S]*isSymbolicLink\(\)[\s\S]*trusted_secret_scan_passed/, 'workflow must independently re-scan fetched report text artifacts before public publishing without following symlinks'); assert.match(proof, /scannedPaths[\s\S]*path\.relative\(root, full\)\.split\(path\.sep\)\.join\('\/'\)[\s\S]*secretPattern\.test\(relativePath\)[\s\S]*path:\$\{relativePath\}[\s\S]*isSymbolicLink\(\)/, 'trusted report scan must inspect artifact paths, including symlink names, before refusing to follow symlinks'); -assert.match(proof, /function looksLikeTextFile\(full\)[\s\S]*subarray\(0, 8192\)[\s\S]*sample\.includes\(0\)[\s\S]*suspiciousControlBytes/, 'trusted report scan must sniff text-like files instead of relying only on a small extension allowlist'); +assert.match(proof, /function readSample\(full, maxBytes = TEXT_SNIFF_BYTES\)[\s\S]*openSync\(full, 'r'\)[\s\S]*readSync\(fd, buffer, 0, maxBytes, 0\)[\s\S]*closeSync\(fd\)/, 'trusted report scan must use a bounded file sample instead of reading whole artifacts before sniffing text'); +assert.match(proof, /function looksLikeTextFile\(full\)[\s\S]*const sample = readSample\(full\)[\s\S]*sample\.includes\(0\)[\s\S]*suspiciousControlBytes/, 'trusted report scan must sniff text-like files instead of relying only on a small extension allowlist'); +assert.match(proof, /function scanChunkForSecret\(text, hasPotentialNextChunk\)[\s\S]*text\.matchAll\(secretPatternGlobal\)[\s\S]*'\[REDACTED\]'\.startsWith\(maybeValue\)[\s\S]*status: 'deferred-redaction'/, 'trusted report content scan must only defer boundary-ending matches that may be split redaction markers'); +assert.match(proof, /function textFileHasSecret\(full\)[\s\S]*TEXT_SCAN_CHUNK_BYTES[\s\S]*readSync\(fd, buffer, 0, buffer\.length, position\)[\s\S]*scanChunkForSecret\(text, bytesRead === buffer\.length\)[\s\S]*scanStatus\.status === 'match'[\s\S]*TEXT_SCAN_OVERLAP_CHARS[\s\S]*closeSync\(fd\)/, 'trusted report content scan must stream text artifacts in bounded chunks instead of reading full files into memory'); assert.doesNotMatch(proof, /textExtPattern/, 'trusted report content scan must not skip extensionless or renamed text diagnostics'); assert.match(proof, /NIXMAC_APP_PATH=\$\(printf '%q' "\$REMOTE_APP_PATH"\)[\s\S]*run-peekaboo-suite --allow-cleanup/, 'Peekaboo run must use the freshly built PR app bundle'); assert.match(proof, /remote_env_parts=\([\s\S]*E2E_TERMINAL_CLEANUP_MODE=kill[\s\S]*E2E_HIDE_RECORDING_TERMINAL=1[\s\S]*E2E_CLOSE_RECORDING_TERMINAL=1/, 'MacInCloud remote runner must force stale recorder Terminal cleanup and keep recorder windows hidden/closed'); @@ -204,6 +207,8 @@ assert.match(nativeMain, /crate::e2e_runtime::value\("RUST_LOG"\)[\s\S]*EnvFilte assert.match(nativeMain, /on_page_load\(move[\s\S]*main webview page load[\s\S]*PageLoadEvent::Finished[\s\S]*store\(true, Ordering::SeqCst\)/, 'Native app must log main WebView page-load lifecycle and mark finished loads for E2E diagnostics'); assert.match(nativeMain, /fn e2e_solid_capture_enabled\(\) -> bool \{\n\s+cfg!\(debug_assertions\) && crate::e2e_runtime::enabled\("NIXMAC_E2E_SOLID_CAPTURE"\)/, 'Native app must expose an E2E-only solid capture gate'); assert.match(nativeMain, /document\.documentElement\.dataset\.nixmacE2eCapture = captureMode[\s\S]*html\[data-nixmac-e2e-capture="\$\{captureMode\}"\][\s\S]*requestAnimationFrame\(\(\) => \{\n\s+document\.documentElement\.dataset\.nixmacE2eCapturePaint = "raf";[\s\S]*"e2e-capture-paint-raf"/, 'Native app capture script must set matching capture selectors and breadcrumb the paint marker'); +assert.match(nativeMain, /html\[data-nixmac-e2e-capture="\$\{captureMode\}"\] \*,[\s\S]*html\[data-nixmac-e2e-capture="\$\{captureMode\}"\] \*::before,[\s\S]*html\[data-nixmac-e2e-capture="\$\{captureMode\}"\] \*::after[\s\S]*-webkit-backdrop-filter: none !important;[\s\S]*backdrop-filter: none !important;/, 'Native app capture script must disable backdrop filtering in E2E capture mode so MacInCloud software capture can see the actual app shell'); +assert.match(nativeMain, /const captureStyleProbe = \(label\) => \{[\s\S]*window\.getComputedStyle\(shell\)[\s\S]*`e2e-capture-style-\$\{label\}`[\s\S]*shellBackdropFilter[\s\S]*shellWebkitBackdropFilter[\s\S]*captureStyleProbe\("app-mounted-plus-2raf"\)/, 'Native app capture script must log post-mount computed styles that distinguish CSS application failures from compositor failures'); assert.match(nativeCaptureWindowSetup, /let e2e_solid_capture = e2e_solid_capture_enabled\(\);[\s\S]*let e2e_css_capture = e2e_solid_capture \|\| e2e_opaque_window[\s\S]*transparent\(!e2e_opaque_window\)/, 'Native app must keep default solid capture CSS-backed and transparent while limiting native opacity to opaque debug mode'); assert.match(nativeCaptureWindowSetup, /if e2e_opaque_window \{\n\s+main_window_builder = main_window_builder\s+\.background_color\(tauri::utils::config::Color\(10, 10, 10, 255\)\);[\s\S]*if e2e_css_capture \{\n\s+main_window_builder =\s+main_window_builder\.initialization_script\(E2E_CAPTURE_DARK_BACKGROUND_SCRIPT\);/, 'Native app must keep native dark background only in the opaque debug path while applying CSS capture for solid and opaque modes'); assert.match(nativeMain, /NIXMAC_E2E_OPAQUE_WINDOW native window diagnostics[\s\S]*isOpaque[\s\S]*alphaValue[\s\S]*hasShadow/, 'Native app must log native opaque-window diagnostics for MacInCloud capture debugging'); From 4f1efeb0959ba635a955522febf59918d6add545 Mon Sep 17 00:00:00 2001 From: fkb032 <249513614+fkb032@users.noreply.github.com> Date: Thu, 7 May 2026 12:25:37 -0700 Subject: [PATCH 02/16] fix: make peekaboo e2e capture window readable --- apps/native/src-tauri/src/main.rs | 21 ++++++++++++++++--- .../peekaboo-workflow-contract-self-test.mjs | 3 ++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/apps/native/src-tauri/src/main.rs b/apps/native/src-tauri/src/main.rs index 024979b7b..7f5cb2ded 100644 --- a/apps/native/src-tauri/src/main.rs +++ b/apps/native/src-tauri/src/main.rs @@ -825,7 +825,7 @@ fn run_gui_mode( log::debug!("Main nixmac window created"); #[cfg(target_os = "macos")] - if e2e_opaque_window { + if e2e_css_capture { use objc::msg_send; use objc::runtime::Object; use objc::sel; @@ -834,12 +834,27 @@ fn run_gui_mode( match main_window.ns_window() { Ok(ns_window) => unsafe { let ns_window = ns_window as *mut Object; + let sharing_type_before: usize = msg_send![ns_window, sharingType]; + if let Err(error) = main_window.set_content_protected(false) { + log::warn!( + "NIXMAC_E2E_CAPTURE native window could not disable content protection: {}", + error + ); + } + let sharing_type_after: usize = msg_send![ns_window, sharingType]; let is_opaque: bool = msg_send![ns_window, isOpaque]; let alpha_value: f64 = msg_send![ns_window, alphaValue]; let level: i64 = msg_send![ns_window, level]; let has_shadow: bool = msg_send![ns_window, hasShadow]; log::info!( - "NIXMAC_E2E_OPAQUE_WINDOW native window diagnostics: isOpaque={} alphaValue={:.3} level={} hasShadow={}", + "NIXMAC_E2E_CAPTURE native window diagnostics: mode={} sharingTypeBefore={} sharingTypeAfter={} isOpaque={} alphaValue={:.3} level={} hasShadow={}", + if e2e_opaque_window { + "opaque" + } else { + "solid" + }, + sharing_type_before, + sharing_type_after, is_opaque, alpha_value, level, @@ -848,7 +863,7 @@ fn run_gui_mode( }, Err(error) => { log::warn!( - "NIXMAC_E2E_OPAQUE_WINDOW native window diagnostics unavailable: {}", + "NIXMAC_E2E_CAPTURE native window diagnostics unavailable: {}", error ); } diff --git a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs index a7ba693cf..48da393d3 100644 --- a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs +++ b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs @@ -211,7 +211,8 @@ assert.match(nativeMain, /html\[data-nixmac-e2e-capture="\$\{captureMode\}"\] \* assert.match(nativeMain, /const captureStyleProbe = \(label\) => \{[\s\S]*window\.getComputedStyle\(shell\)[\s\S]*`e2e-capture-style-\$\{label\}`[\s\S]*shellBackdropFilter[\s\S]*shellWebkitBackdropFilter[\s\S]*captureStyleProbe\("app-mounted-plus-2raf"\)/, 'Native app capture script must log post-mount computed styles that distinguish CSS application failures from compositor failures'); assert.match(nativeCaptureWindowSetup, /let e2e_solid_capture = e2e_solid_capture_enabled\(\);[\s\S]*let e2e_css_capture = e2e_solid_capture \|\| e2e_opaque_window[\s\S]*transparent\(!e2e_opaque_window\)/, 'Native app must keep default solid capture CSS-backed and transparent while limiting native opacity to opaque debug mode'); assert.match(nativeCaptureWindowSetup, /if e2e_opaque_window \{\n\s+main_window_builder = main_window_builder\s+\.background_color\(tauri::utils::config::Color\(10, 10, 10, 255\)\);[\s\S]*if e2e_css_capture \{\n\s+main_window_builder =\s+main_window_builder\.initialization_script\(E2E_CAPTURE_DARK_BACKGROUND_SCRIPT\);/, 'Native app must keep native dark background only in the opaque debug path while applying CSS capture for solid and opaque modes'); -assert.match(nativeMain, /NIXMAC_E2E_OPAQUE_WINDOW native window diagnostics[\s\S]*isOpaque[\s\S]*alphaValue[\s\S]*hasShadow/, 'Native app must log native opaque-window diagnostics for MacInCloud capture debugging'); +assert.match(nativeMain, /if e2e_css_capture \{[\s\S]*match main_window\.ns_window\(\) \{[\s\S]*set_content_protected\(false\)/, 'Native app must explicitly make E2E capture windows readable from the native diagnostics branch'); +assert.match(nativeMain, /NIXMAC_E2E_CAPTURE native window diagnostics:[\s\S]*sharingTypeBefore=\{\}[\s\S]*sharingTypeAfter=\{\}[\s\S]*isOpaque[\s\S]*alphaValue[\s\S]*hasShadow/, 'Native app must log native sharing diagnostics for MacInCloud capture debugging'); assert.match(nativeSolidCaptureBranch, /NIXMAC_E2E_SOLID_CAPTURE enabled/, 'Native app must keep an explicit solid-capture branch for diagnostics'); assert.doesNotMatch(nativeSolidCaptureBranch, /background_color\(tauri::utils::config::Color\(10, 10, 10, 255\)\)/, 'Native app must not apply native background_color from the default solid-capture path'); assert.match(nativeMain, /fn e2e_webview_watchdog_enabled\(\) -> bool \{\n\s+cfg!\(debug_assertions\) && crate::e2e_runtime::enabled\("NIXMAC_E2E_WEBVIEW_WATCHDOG"\)/, 'Native app must expose an E2E-only WebView watchdog gate independent of opaque capture'); From ba72dcc6d8eb8a0a3b1f5ee77282cf0dfde4c568 Mon Sep 17 00:00:00 2001 From: fkb032 <249513614+fkb032@users.noreply.github.com> Date: Thu, 7 May 2026 12:56:08 -0700 Subject: [PATCH 03/16] fix: restore peekaboo capture contrast --- apps/native/src-tauri/src/main.rs | 110 +++++++++++++++++- .../peekaboo-workflow-contract-self-test.mjs | 6 +- 2 files changed, 109 insertions(+), 7 deletions(-) diff --git a/apps/native/src-tauri/src/main.rs b/apps/native/src-tauri/src/main.rs index 7f5cb2ded..fc7faecdf 100644 --- a/apps/native/src-tauri/src/main.rs +++ b/apps/native/src-tauri/src/main.rs @@ -154,7 +154,6 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" (() => { const styleId = "nixmac-e2e-capture-background"; const captureMode = "solid"; - const captureBackground = "hsl(0 0% 3.9%)"; const logCaptureBreadcrumb = (label, detail) => { const invoke = window.__TAURI__?.core?.invoke || window.__TAURI_INTERNALS__?.invoke; if (typeof invoke !== "function") return; @@ -169,10 +168,49 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" const value = style.getPropertyValue(property); return value === "" ? null : value; }; + const firstMatchingElement = (selectors) => { + for (const selector of selectors) { + const element = document.querySelector(selector); + if (element) return element; + } + return null; + }; + const firstTextElement = () => { + const candidates = document.body?.querySelectorAll("h1,h2,h3,p,span,button,[data-slot='card']") ?? []; + return [...candidates].find((element) => { + const style = window.getComputedStyle(element); + return ( + element.textContent?.trim() && + style.display !== "none" && + style.visibility !== "hidden" && + style.opacity !== "0" + ); + }) ?? null; + }; + const sampleStyle = (element) => { + if (!element) return null; + const style = window.getComputedStyle(element); + return { + tagName: element.tagName?.toLowerCase() ?? null, + className: typeof element.className === "string" ? element.className : null, + text: element.textContent?.trim().slice(0, 80) ?? null, + backgroundColor: style.backgroundColor, + borderTopColor: style.borderTopColor, + color: style.color, + display: style.display, + opacity: style.opacity, + visibility: style.visibility, + }; + }; const captureStyleProbe = (label) => { try { const root = document.getElementById("root"); const shell = root?.firstElementChild ?? null; + const card = firstMatchingElement(["[data-slot='card']", ".bg-card", ".bg-card\\/50", ".bg-card\\/80", ".bg-card\\/95"]); + const header = firstMatchingElement(["[data-tauri-drag-region='true'].border-b", ".border-b"]); + const button = document.querySelector("button"); + const svg = document.querySelector("svg"); + const textElement = firstTextElement(); const htmlStyle = window.getComputedStyle(document.documentElement); const bodyStyle = document.body ? window.getComputedStyle(document.body) : null; const shellStyle = shell ? window.getComputedStyle(shell) : null; @@ -190,6 +228,13 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" shellBackdropFilter: getCssValue(shellStyle, "backdrop-filter"), shellWebkitBackdropFilter: getCssValue(shellStyle, "-webkit-backdrop-filter"), shellOpacity: shellStyle?.opacity ?? null, + bodyColor: bodyStyle?.color ?? null, + captureReady: document.documentElement.dataset.nixmacE2eCaptureReady ?? null, + cardStyle: sampleStyle(card), + headerStyle: sampleStyle(header), + buttonStyle: sampleStyle(button), + svgStyle: sampleStyle(svg), + textStyle: sampleStyle(textElement), }), ); } catch (error) { @@ -206,17 +251,59 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" html[data-nixmac-e2e-capture="${captureMode}"], html[data-nixmac-e2e-capture="${captureMode}"] body, html[data-nixmac-e2e-capture="${captureMode}"] #root { - background: ${captureBackground} !important; - background-color: ${captureBackground} !important; + --nixmac-e2e-capture-background: hsl(var(--background, 0 0% 3.9%)); + --nixmac-e2e-capture-surface: hsl(var(--accent, 0 0% 14.9%)); + --nixmac-e2e-capture-card: hsl(var(--card, 0 0% 3.9%)); + --nixmac-e2e-capture-foreground: hsl(var(--foreground, 0 0% 98%)); + --nixmac-e2e-capture-muted: hsl(var(--muted-foreground, 0 0% 63.9%)); + --nixmac-e2e-capture-border: hsl(var(--border, 0 0% 14.9%)); + --nixmac-e2e-capture-primary: hsl(var(--primary, 0 0% 98%)); + --nixmac-e2e-capture-primary-foreground: hsl(var(--primary-foreground, 0 0% 9%)); + background: var(--nixmac-e2e-capture-background) !important; + background-color: var(--nixmac-e2e-capture-background) !important; + color: var(--nixmac-e2e-capture-foreground) !important; + } + html[data-nixmac-e2e-capture="${captureMode}"] #root > * { + background-color: var(--nixmac-e2e-capture-background) !important; + color: var(--nixmac-e2e-capture-foreground) !important; } html[data-nixmac-e2e-capture="${captureMode}"] .bg-background\\/80, html[data-nixmac-e2e-capture="${captureMode}"] .bg-background\\/90, - html[data-nixmac-e2e-capture="${captureMode}"] .bg-background\\/95, + html[data-nixmac-e2e-capture="${captureMode}"] .bg-background\\/95 { + background-color: var(--nixmac-e2e-capture-background) !important; + } html[data-nixmac-e2e-capture="${captureMode}"] .bg-card\\/50, html[data-nixmac-e2e-capture="${captureMode}"] .bg-card\\/80, html[data-nixmac-e2e-capture="${captureMode}"] .bg-card\\/95, html[data-nixmac-e2e-capture="${captureMode}"] .bg-zinc-900\\/95 { - background-color: ${captureBackground} !important; + background-color: var(--nixmac-e2e-capture-surface) !important; + } + html[data-nixmac-e2e-capture="${captureMode}"] [data-slot="card"], + html[data-nixmac-e2e-capture="${captureMode}"] .bg-card { + background-color: var(--nixmac-e2e-capture-card) !important; + border-color: var(--nixmac-e2e-capture-border) !important; + color: var(--nixmac-e2e-capture-foreground) !important; + } + html[data-nixmac-e2e-capture="${captureMode}"] [class*="text-foreground"], + html[data-nixmac-e2e-capture="${captureMode}"] [class*="text-card-foreground"] { + color: var(--nixmac-e2e-capture-foreground) !important; + } + html[data-nixmac-e2e-capture="${captureMode}"] [class*="text-muted-foreground"] { + color: var(--nixmac-e2e-capture-muted) !important; + } + html[data-nixmac-e2e-capture="${captureMode}"] [class~="text-primary"] { + color: var(--nixmac-e2e-capture-primary) !important; + } + html[data-nixmac-e2e-capture="${captureMode}"] .bg-primary { + background-color: var(--nixmac-e2e-capture-primary) !important; + color: var(--nixmac-e2e-capture-primary-foreground) !important; + } + html[data-nixmac-e2e-capture="${captureMode}"] [class*="border"] { + border-color: var(--nixmac-e2e-capture-border) !important; + } + html[data-nixmac-e2e-capture="${captureMode}"] svg { + color: currentColor; + stroke: currentColor; } html[data-nixmac-e2e-capture="${captureMode}"] *, html[data-nixmac-e2e-capture="${captureMode}"] *::before, @@ -238,12 +325,25 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" }), ); captureStyleProbe("initial-raf"); + requestAnimationFrame(() => { + document.documentElement.dataset.nixmacE2eCaptureReady = "1"; + logCaptureBreadcrumb( + "e2e-capture-ready-2raf", + JSON.stringify({ + captureMode, + rootChildren: document.getElementById("root")?.childElementCount ?? null, + bodyChildren: document.body?.childElementCount ?? null, + }), + ); + captureStyleProbe("capture-ready-2raf"); + }); }); window.addEventListener( "nixmac:app-mounted", () => { requestAnimationFrame(() => { requestAnimationFrame(() => { + document.documentElement.dataset.nixmacE2eCaptureReady = "app-mounted"; captureStyleProbe("app-mounted-plus-2raf"); }); }); diff --git a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs index 48da393d3..c2605ddde 100644 --- a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs +++ b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs @@ -206,9 +206,11 @@ assert.match(nativeMain, /crate::e2e_runtime::value\("NIXMAC_LOGFILE"\)/, 'Nativ assert.match(nativeMain, /crate::e2e_runtime::value\("RUST_LOG"\)[\s\S]*EnvFilter::try_from_default_env/, 'Native app logging must read RUST_LOG through the E2E runtime file before falling back to process env/default filters'); assert.match(nativeMain, /on_page_load\(move[\s\S]*main webview page load[\s\S]*PageLoadEvent::Finished[\s\S]*store\(true, Ordering::SeqCst\)/, 'Native app must log main WebView page-load lifecycle and mark finished loads for E2E diagnostics'); assert.match(nativeMain, /fn e2e_solid_capture_enabled\(\) -> bool \{\n\s+cfg!\(debug_assertions\) && crate::e2e_runtime::enabled\("NIXMAC_E2E_SOLID_CAPTURE"\)/, 'Native app must expose an E2E-only solid capture gate'); -assert.match(nativeMain, /document\.documentElement\.dataset\.nixmacE2eCapture = captureMode[\s\S]*html\[data-nixmac-e2e-capture="\$\{captureMode\}"\][\s\S]*requestAnimationFrame\(\(\) => \{\n\s+document\.documentElement\.dataset\.nixmacE2eCapturePaint = "raf";[\s\S]*"e2e-capture-paint-raf"/, 'Native app capture script must set matching capture selectors and breadcrumb the paint marker'); +assert.match(nativeMain, /document\.documentElement\.dataset\.nixmacE2eCapture = captureMode[\s\S]*html\[data-nixmac-e2e-capture="\$\{captureMode\}"\][\s\S]*requestAnimationFrame\(\(\) => \{\n\s+document\.documentElement\.dataset\.nixmacE2eCapturePaint = "raf";[\s\S]*"e2e-capture-paint-raf"[\s\S]*document\.documentElement\.dataset\.nixmacE2eCaptureReady = "1"[\s\S]*"e2e-capture-ready-2raf"/, 'Native app capture script must set matching capture selectors and breadcrumb a double-RAF paint-ready marker'); assert.match(nativeMain, /html\[data-nixmac-e2e-capture="\$\{captureMode\}"\] \*,[\s\S]*html\[data-nixmac-e2e-capture="\$\{captureMode\}"\] \*::before,[\s\S]*html\[data-nixmac-e2e-capture="\$\{captureMode\}"\] \*::after[\s\S]*-webkit-backdrop-filter: none !important;[\s\S]*backdrop-filter: none !important;/, 'Native app capture script must disable backdrop filtering in E2E capture mode so MacInCloud software capture can see the actual app shell'); -assert.match(nativeMain, /const captureStyleProbe = \(label\) => \{[\s\S]*window\.getComputedStyle\(shell\)[\s\S]*`e2e-capture-style-\$\{label\}`[\s\S]*shellBackdropFilter[\s\S]*shellWebkitBackdropFilter[\s\S]*captureStyleProbe\("app-mounted-plus-2raf"\)/, 'Native app capture script must log post-mount computed styles that distinguish CSS application failures from compositor failures'); +assert.match(nativeMain, /--nixmac-e2e-capture-background: hsl\(var\(--background[\s\S]*--nixmac-e2e-capture-surface: hsl\(var\(--accent[\s\S]*--nixmac-e2e-capture-foreground: hsl\(var\(--foreground[\s\S]*--nixmac-e2e-capture-border: hsl\(var\(--border/, 'Native app capture script must derive capture colors from the real nixmac dark theme tokens'); +assert.doesNotMatch(nativeMain, /\.bg-card\\\/50,[\s\S]*\.bg-card\\\/95,[\s\S]*background-color: \$\{captureBackground\}/, 'Native app capture script must not flatten card surfaces to the root capture background'); +assert.match(nativeMain, /const captureStyleProbe = \(label\) => \{[\s\S]*window\.getComputedStyle\(shell\)[\s\S]*cardStyle: sampleStyle\(card\)[\s\S]*headerStyle: sampleStyle\(header\)[\s\S]*buttonStyle: sampleStyle\(button\)[\s\S]*svgStyle: sampleStyle\(svg\)[\s\S]*textStyle: sampleStyle\(textElement\)[\s\S]*captureStyleProbe\("app-mounted-plus-2raf"\)/, 'Native app capture script must log post-mount computed foreground, card, border, button, and icon styles that distinguish CSS failures from compositor failures'); assert.match(nativeCaptureWindowSetup, /let e2e_solid_capture = e2e_solid_capture_enabled\(\);[\s\S]*let e2e_css_capture = e2e_solid_capture \|\| e2e_opaque_window[\s\S]*transparent\(!e2e_opaque_window\)/, 'Native app must keep default solid capture CSS-backed and transparent while limiting native opacity to opaque debug mode'); assert.match(nativeCaptureWindowSetup, /if e2e_opaque_window \{\n\s+main_window_builder = main_window_builder\s+\.background_color\(tauri::utils::config::Color\(10, 10, 10, 255\)\);[\s\S]*if e2e_css_capture \{\n\s+main_window_builder =\s+main_window_builder\.initialization_script\(E2E_CAPTURE_DARK_BACKGROUND_SCRIPT\);/, 'Native app must keep native dark background only in the opaque debug path while applying CSS capture for solid and opaque modes'); assert.match(nativeMain, /if e2e_css_capture \{[\s\S]*match main_window\.ns_window\(\) \{[\s\S]*set_content_protected\(false\)/, 'Native app must explicitly make E2E capture windows readable from the native diagnostics branch'); From 96066c660d3afff152af11d454ed815beada9c1f Mon Sep 17 00:00:00 2001 From: fkb032 <249513614+fkb032@users.noreply.github.com> Date: Thu, 7 May 2026 13:32:21 -0700 Subject: [PATCH 04/16] chore: add peekaboo capture diagnostics --- apps/native/src-tauri/src/main.rs | 199 ++++++++++++++++++ .../peekaboo-workflow-contract-self-test.mjs | 3 + 2 files changed, 202 insertions(+) diff --git a/apps/native/src-tauri/src/main.rs b/apps/native/src-tauri/src/main.rs index fc7faecdf..1081c6fb9 100644 --- a/apps/native/src-tauri/src/main.rs +++ b/apps/native/src-tauri/src/main.rs @@ -150,6 +150,124 @@ fn e2e_schedule_webview_boot_probe(window: WebviewWindow, label: &'static str, d fn e2e_schedule_webview_boot_probe(_window: WebviewWindow, _label: &'static str, _delay: Duration) { } +#[cfg(all(debug_assertions, target_os = "macos"))] +fn e2e_request_native_webview_capture_probe(window: &WebviewWindow, label: &'static str) { + use objc::msg_send; + use objc::runtime::{Object, YES}; + use objc::sel; + use objc::sel_impl; + + #[repr(C)] + struct E2eNativePoint { + x: f64, + y: f64, + } + + #[repr(C)] + struct E2eNativeSize { + width: f64, + height: f64, + } + + #[repr(C)] + struct E2eNativeRect { + origin: E2eNativePoint, + size: E2eNativeSize, + } + + if let Err(error) = window.with_webview(move |webview| unsafe { + let webview = webview.inner() as *mut Object; + let is_hidden: bool = msg_send![webview, isHidden]; + let alpha_value: f64 = msg_send![webview, alphaValue]; + let wants_layer: bool = msg_send![webview, wantsLayer]; + let frame: E2eNativeRect = msg_send![webview, frame]; + let bounds: E2eNativeRect = msg_send![webview, bounds]; + let responds_to_draws_background: bool = + msg_send![webview, respondsToSelector: sel!(drawsBackground)]; + let draws_background = if responds_to_draws_background { + let value: bool = msg_send![webview, drawsBackground]; + Some(value) + } else { + None + }; + let layer: *mut Object = msg_send![webview, layer]; + let layer_has_contents = if layer.is_null() { + false + } else { + let contents: *mut Object = msg_send![layer, contents]; + !contents.is_null() + }; + let layer_opacity: f32 = if layer.is_null() { + -1.0 + } else { + msg_send![layer, opacity] + }; + let layer_hidden: bool = if layer.is_null() { + false + } else { + msg_send![layer, isHidden] + }; + + // Best-effort AppKit invalidation for the virtualized MacInCloud capture path. + // WKWebView content is out-of-process, so this is evidence plus a hint, not a + // guarantee that WebContent will repaint. + let _: () = msg_send![webview, setNeedsDisplay: YES]; + let _: () = msg_send![webview, displayIfNeeded]; + + log::info!( + "NIXMAC_E2E_CAPTURE webview diagnostics: label={} drawsBackground={:?} respondsToDrawsBackground={} hidden={} alphaValue={:.3} wantsLayer={} frame={:.0},{:.0},{:.0},{:.0} bounds={:.0},{:.0},{:.0},{:.0} layerPresent={} layerHidden={} layerOpacity={:.3} layerHasContents={} appKitDisplayHint=true", + label, + draws_background, + responds_to_draws_background, + is_hidden, + alpha_value, + wants_layer, + frame.origin.x, + frame.origin.y, + frame.size.width, + frame.size.height, + bounds.origin.x, + bounds.origin.y, + bounds.size.width, + bounds.size.height, + !layer.is_null(), + layer_hidden, + layer_opacity, + layer_has_contents + ); + }) { + log::warn!( + "NIXMAC_E2E_CAPTURE webview diagnostics unavailable for {}: {}", + label, + error + ); + } +} + +#[cfg(not(all(debug_assertions, target_os = "macos")))] +#[allow(dead_code)] +fn e2e_request_native_webview_capture_probe(_window: &WebviewWindow, _label: &'static str) {} + +#[cfg(debug_assertions)] +fn e2e_schedule_native_webview_capture_probe( + window: WebviewWindow, + label: &'static str, + delay: Duration, +) { + std::thread::spawn(move || { + std::thread::sleep(delay); + e2e_request_native_webview_capture_probe(&window, label); + }); +} + +#[cfg(not(debug_assertions))] +fn e2e_schedule_native_webview_capture_probe( + _window: WebviewWindow, + _label: &'static str, + _delay: Duration, +) { +} + const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" (() => { const styleId = "nixmac-e2e-capture-background"; @@ -202,6 +320,63 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" visibility: style.visibility, }; }; + const rectFor = (element) => { + if (!element) return null; + const rect = element.getBoundingClientRect(); + return { + x: Math.round(rect.x), + y: Math.round(rect.y), + width: Math.round(rect.width), + height: Math.round(rect.height), + top: Math.round(rect.top), + right: Math.round(rect.right), + bottom: Math.round(rect.bottom), + left: Math.round(rect.left), + }; + }; + const summarizeElement = (element) => { + if (!element) return null; + return { + tagName: element.tagName?.toLowerCase() ?? null, + className: typeof element.className === "string" ? element.className : null, + text: element.textContent?.trim().slice(0, 80) ?? null, + rect: rectFor(element), + }; + }; + const elementAtCenter = (element) => { + const rect = element?.getBoundingClientRect(); + if (!rect || rect.width <= 0 || rect.height <= 0) return null; + const x = Math.max(0, Math.min(window.innerWidth - 1, rect.left + rect.width / 2)); + const y = Math.max(0, Math.min(window.innerHeight - 1, rect.top + rect.height / 2)); + const hit = document.elementFromPoint(x, y); + return { + point: { x: Math.round(x), y: Math.round(y) }, + hit: summarizeElement(hit), + textContainsHit: Boolean(hit && (element === hit || element.contains(hit))), + hitContainsText: Boolean(hit && hit.contains(element)), + }; + }; + const canvasReadbackProbe = () => { + try { + const canvas = document.createElement("canvas"); + canvas.width = 2; + canvas.height = 2; + const context = canvas.getContext("2d"); + if (!context) return { ok: false, error: "2d context unavailable" }; + context.fillStyle = "rgb(255, 255, 255)"; + context.fillRect(0, 0, 2, 2); + context.fillStyle = "rgb(17, 17, 17)"; + context.fillRect(1, 1, 1, 1); + const pixel = context.getImageData(0, 0, 1, 1).data; + return { + ok: true, + firstPixel: [pixel[0], pixel[1], pixel[2], pixel[3]], + dataUrlPrefix: canvas.toDataURL("image/png").slice(0, 32), + }; + } catch (error) { + return { ok: false, error: String(error) }; + } + }; const captureStyleProbe = (label) => { try { const root = document.getElementById("root"); @@ -219,9 +394,14 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" JSON.stringify({ captureMode, capturePaint: document.documentElement.dataset.nixmacE2eCapturePaint ?? null, + visibilityState: document.visibilityState, + devicePixelRatio: window.devicePixelRatio, + viewport: { width: window.innerWidth, height: window.innerHeight }, rootChildren: root?.childElementCount ?? null, bodyChildren: document.body?.childElementCount ?? null, shellClassName: typeof shell?.className === "string" ? shell.className : null, + rootRect: rectFor(root), + shellRect: rectFor(shell), htmlBackgroundColor: htmlStyle.backgroundColor, bodyBackgroundColor: bodyStyle?.backgroundColor ?? null, shellBackgroundColor: shellStyle?.backgroundColor ?? null, @@ -235,6 +415,9 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" buttonStyle: sampleStyle(button), svgStyle: sampleStyle(svg), textStyle: sampleStyle(textElement), + textRect: rectFor(textElement), + textElementAtCenter: elementAtCenter(textElement), + canvasReadback: canvasReadbackProbe(), }), ); } catch (error) { @@ -861,6 +1044,7 @@ fn run_gui_mode( let main_webview_loaded = Arc::new(AtomicBool::new(false)); let main_webview_loaded_for_page_load = Arc::clone(&main_webview_loaded); let e2e_page_load_boot_probe = e2e_webview_watchdog; + let e2e_page_load_native_capture_probe = e2e_css_capture; let mut main_window_builder = WebviewWindowBuilder::new(app, "main", WebviewUrl::App("index.html".into())) @@ -903,6 +1087,13 @@ fn run_gui_mode( Duration::from_secs(5), ); } + if e2e_page_load_native_capture_probe { + e2e_schedule_native_webview_capture_probe( + window.clone(), + "native-page-load-finished-plus-5s", + Duration::from_secs(5), + ); + } } }); @@ -970,6 +1161,14 @@ fn run_gui_mode( } } + if e2e_css_capture { + e2e_schedule_native_webview_capture_probe( + main_window.clone(), + "native-post-build-plus-2s", + Duration::from_secs(2), + ); + } + if e2e_webview_watchdog { e2e_schedule_webview_boot_probe( main_window.clone(), diff --git a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs index c2605ddde..e87f7c8a6 100644 --- a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs +++ b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs @@ -211,10 +211,13 @@ assert.match(nativeMain, /html\[data-nixmac-e2e-capture="\$\{captureMode\}"\] \* assert.match(nativeMain, /--nixmac-e2e-capture-background: hsl\(var\(--background[\s\S]*--nixmac-e2e-capture-surface: hsl\(var\(--accent[\s\S]*--nixmac-e2e-capture-foreground: hsl\(var\(--foreground[\s\S]*--nixmac-e2e-capture-border: hsl\(var\(--border/, 'Native app capture script must derive capture colors from the real nixmac dark theme tokens'); assert.doesNotMatch(nativeMain, /\.bg-card\\\/50,[\s\S]*\.bg-card\\\/95,[\s\S]*background-color: \$\{captureBackground\}/, 'Native app capture script must not flatten card surfaces to the root capture background'); assert.match(nativeMain, /const captureStyleProbe = \(label\) => \{[\s\S]*window\.getComputedStyle\(shell\)[\s\S]*cardStyle: sampleStyle\(card\)[\s\S]*headerStyle: sampleStyle\(header\)[\s\S]*buttonStyle: sampleStyle\(button\)[\s\S]*svgStyle: sampleStyle\(svg\)[\s\S]*textStyle: sampleStyle\(textElement\)[\s\S]*captureStyleProbe\("app-mounted-plus-2raf"\)/, 'Native app capture script must log post-mount computed foreground, card, border, button, and icon styles that distinguish CSS failures from compositor failures'); +assert.match(nativeMain, /const rectFor = \(element\)[\s\S]*document\.elementFromPoint\(x, y\)[\s\S]*const canvasReadbackProbe = \(\)[\s\S]*getImageData\(0, 0, 1, 1\)[\s\S]*devicePixelRatio: window\.devicePixelRatio[\s\S]*textElementAtCenter: elementAtCenter\(textElement\)[\s\S]*canvasReadback: canvasReadbackProbe\(\)/, 'Native app capture script must log layout, hit-test, and canvas readback evidence for black-capture diagnosis'); assert.match(nativeCaptureWindowSetup, /let e2e_solid_capture = e2e_solid_capture_enabled\(\);[\s\S]*let e2e_css_capture = e2e_solid_capture \|\| e2e_opaque_window[\s\S]*transparent\(!e2e_opaque_window\)/, 'Native app must keep default solid capture CSS-backed and transparent while limiting native opacity to opaque debug mode'); assert.match(nativeCaptureWindowSetup, /if e2e_opaque_window \{\n\s+main_window_builder = main_window_builder\s+\.background_color\(tauri::utils::config::Color\(10, 10, 10, 255\)\);[\s\S]*if e2e_css_capture \{\n\s+main_window_builder =\s+main_window_builder\.initialization_script\(E2E_CAPTURE_DARK_BACKGROUND_SCRIPT\);/, 'Native app must keep native dark background only in the opaque debug path while applying CSS capture for solid and opaque modes'); assert.match(nativeMain, /if e2e_css_capture \{[\s\S]*match main_window\.ns_window\(\) \{[\s\S]*set_content_protected\(false\)/, 'Native app must explicitly make E2E capture windows readable from the native diagnostics branch'); assert.match(nativeMain, /NIXMAC_E2E_CAPTURE native window diagnostics:[\s\S]*sharingTypeBefore=\{\}[\s\S]*sharingTypeAfter=\{\}[\s\S]*isOpaque[\s\S]*alphaValue[\s\S]*hasShadow/, 'Native app must log native sharing diagnostics for MacInCloud capture debugging'); +assert.match(nativeMain, /fn e2e_request_native_webview_capture_probe[\s\S]*with_webview\(move \|webview\|[\s\S]*respondsToSelector: sel!\(drawsBackground\)[\s\S]*setNeedsDisplay: YES[\s\S]*displayIfNeeded[\s\S]*NIXMAC_E2E_CAPTURE webview diagnostics:[\s\S]*fn e2e_schedule_native_webview_capture_probe/, 'Native app must provide guarded WebView diagnostics and a best-effort AppKit display hint for MacInCloud black-capture diagnosis'); +assert.match(nativeMain, /e2e_page_load_native_capture_probe[\s\S]*native-page-load-finished-plus-5s[\s\S]*e2e_schedule_native_webview_capture_probe[\s\S]*native-post-build-plus-2s/, 'Native app must schedule native WebView capture diagnostics after build and after page-load'); assert.match(nativeSolidCaptureBranch, /NIXMAC_E2E_SOLID_CAPTURE enabled/, 'Native app must keep an explicit solid-capture branch for diagnostics'); assert.doesNotMatch(nativeSolidCaptureBranch, /background_color\(tauri::utils::config::Color\(10, 10, 10, 255\)\)/, 'Native app must not apply native background_color from the default solid-capture path'); assert.match(nativeMain, /fn e2e_webview_watchdog_enabled\(\) -> bool \{\n\s+cfg!\(debug_assertions\) && crate::e2e_runtime::enabled\("NIXMAC_E2E_WEBVIEW_WATCHDOG"\)/, 'Native app must expose an E2E-only WebView watchdog gate independent of opaque capture'); From 2a356827d8082761e2db057f020c46d2325af9fb Mon Sep 17 00:00:00 2001 From: fkb032 <249513614+fkb032@users.noreply.github.com> Date: Thu, 7 May 2026 14:38:09 -0700 Subject: [PATCH 05/16] fix: add native webview snapshots for peekaboo proof --- Cargo.lock | 1 + apps/native/src-tauri/Cargo.toml | 1 + apps/native/src-tauri/src/main.rs | 374 ++++++++++++++++++ tests/e2e/adapters/nixmac.sh | 32 +- tests/e2e/lib/nixmac_product_proof.sh | 72 +++- tools/computer-use-e2e/peekaboo-runner.mjs | 4 +- .../peekaboo-workflow-contract-self-test.mjs | 17 + tools/computer-use-e2e/run-local.mjs | 8 +- 8 files changed, 497 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 37b6c5ccf..15ef15e25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3626,6 +3626,7 @@ dependencies = [ "async-openai", "async-trait", "backtrace", + "block", "chrono", "clap", "cocoa", diff --git a/apps/native/src-tauri/Cargo.toml b/apps/native/src-tauri/Cargo.toml index 30f7bfe7f..ee50d2ce6 100644 --- a/apps/native/src-tauri/Cargo.toml +++ b/apps/native/src-tauri/Cargo.toml @@ -81,6 +81,7 @@ tauri-plugin-updater = "2.0.0" url = "2" [target.'cfg(target_os = "macos")'.dependencies] +block = "0.1" cocoa = "0.26" objc = "0.2" diff --git a/apps/native/src-tauri/src/main.rs b/apps/native/src-tauri/src/main.rs index 1081c6fb9..fc0170610 100644 --- a/apps/native/src-tauri/src/main.rs +++ b/apps/native/src-tauri/src/main.rs @@ -65,6 +65,10 @@ fn e2e_webview_watchdog_enabled() -> bool { cfg!(debug_assertions) && crate::e2e_runtime::enabled("NIXMAC_E2E_WEBVIEW_WATCHDOG") } +#[cfg(all(debug_assertions, target_os = "macos"))] +static E2E_NATIVE_SNAPSHOT_COUNTER: std::sync::atomic::AtomicUsize = + std::sync::atomic::AtomicUsize::new(0); + #[cfg(debug_assertions)] fn e2e_request_webview_boot_probe(window: &WebviewWindow, label: &'static str) { let label_json = @@ -268,6 +272,365 @@ fn e2e_schedule_native_webview_capture_probe( ) { } +#[cfg(all(debug_assertions, target_os = "macos"))] +fn e2e_clean_snapshot_label(label: &str) -> String { + let cleaned = label + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() || ch == '.' || ch == '_' || ch == '-' { + ch + } else { + '-' + } + }) + .collect::() + .trim_matches('-') + .chars() + .take(80) + .collect::(); + if cleaned.is_empty() { + "snapshot".to_string() + } else { + cleaned + } +} + +#[cfg(all(debug_assertions, target_os = "macos"))] +fn e2e_native_snapshot_root_dir() -> Option { + crate::e2e_runtime::value("NIXMAC_E2E_DIAGNOSTICS_DIR") + .map(std::path::PathBuf::from) + .map(|path| path.join("native-webview-snapshots")) +} + +#[cfg(all(debug_assertions, target_os = "macos"))] +fn e2e_native_snapshot_paths(label: &str) -> Option<(std::path::PathBuf, std::path::PathBuf)> { + let root = e2e_native_snapshot_root_dir()?; + let sequence = E2E_NATIVE_SNAPSHOT_COUNTER.fetch_add(1, Ordering::SeqCst) + 1; + let epoch_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_millis()) + .unwrap_or(0); + let stem = format!( + "native-webview-{sequence:04}-{}-{epoch_ms}", + e2e_clean_snapshot_label(label) + ); + Some(( + root.join(format!("{stem}.png")), + root.join(format!("{stem}.json")), + )) +} + +#[cfg(all(debug_assertions, target_os = "macos"))] +fn e2e_write_native_snapshot_status( + status_path: &std::path::Path, + status: &str, + label: &str, + output_path: &std::path::Path, + message: Option<&str>, +) { + if let Some(parent) = status_path.parent() { + let _ = std::fs::create_dir_all(parent); + } + let payload = serde_json::json!({ + "status": status, + "label": label, + "path": output_path, + "message": message, + "capturedAtUnixMs": std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_millis()) + .unwrap_or(0), + "source": "WKWebView.takeSnapshotWithConfiguration" + }); + let tmp_status_path = status_path.with_extension("json.tmp"); + if std::fs::write( + &tmp_status_path, + serde_json::to_string_pretty(&payload).unwrap_or_else(|_| "{}".to_string()), + ) + .and_then(|()| std::fs::rename(&tmp_status_path, status_path)) + .is_err() + { + let _ = std::fs::remove_file(&tmp_status_path); + } +} + +#[cfg(all(debug_assertions, target_os = "macos"))] +fn e2e_request_native_webview_snapshot( + window: &WebviewWindow, + label: String, + output_path: std::path::PathBuf, + status_path: std::path::PathBuf, +) { + use block::ConcreteBlock; + use objc::class; + use objc::msg_send; + use objc::runtime::Object; + use objc::sel; + use objc::sel_impl; + + if let Some(parent) = output_path.parent() { + if let Err(error) = std::fs::create_dir_all(parent) { + log::warn!( + "NIXMAC_E2E_NATIVE_SNAPSHOT could not create output dir for {}: {}", + label, + error + ); + e2e_write_native_snapshot_status( + &status_path, + "failed", + &label, + &output_path, + Some(&format!("failed to create output directory: {error}")), + ); + return; + } + } + if let Some(parent) = status_path.parent() { + let _ = std::fs::create_dir_all(parent); + } + e2e_write_native_snapshot_status(&status_path, "pending", &label, &output_path, None); + + let error_label = label.clone(); + let error_output_path = output_path.clone(); + let error_status_path = status_path.clone(); + if let Err(error) = window.with_webview(move |webview| unsafe { + let label_for_log = label.clone(); + let label_for_status = label.clone(); + let output_for_block = output_path.clone(); + let status_for_block = status_path.clone(); + let completion = ConcreteBlock::new(move |image: *mut Object, error: *mut Object| { + if !error.is_null() { + log::warn!( + "NIXMAC_E2E_NATIVE_SNAPSHOT failed for {}: WebKit returned an error", + label_for_status + ); + e2e_write_native_snapshot_status( + &status_for_block, + "failed", + &label_for_status, + &output_for_block, + Some("WebKit returned an error"), + ); + return; + } + if image.is_null() { + log::warn!( + "NIXMAC_E2E_NATIVE_SNAPSHOT failed for {}: WebKit returned no image", + label_for_status + ); + e2e_write_native_snapshot_status( + &status_for_block, + "failed", + &label_for_status, + &output_for_block, + Some("WebKit returned no image"), + ); + return; + } + + let tiff_data: *mut Object = msg_send![image, TIFFRepresentation]; + if tiff_data.is_null() { + e2e_write_native_snapshot_status( + &status_for_block, + "failed", + &label_for_status, + &output_for_block, + Some("NSImage TIFFRepresentation was nil"), + ); + return; + } + let bitmap_rep: *mut Object = + msg_send![class!(NSBitmapImageRep), imageRepWithData: tiff_data]; + if bitmap_rep.is_null() { + e2e_write_native_snapshot_status( + &status_for_block, + "failed", + &label_for_status, + &output_for_block, + Some("NSBitmapImageRep could not read snapshot data"), + ); + return; + } + let properties: *mut Object = msg_send![class!(NSDictionary), dictionary]; + const NS_BITMAP_IMAGE_FILE_TYPE_PNG: usize = 4; + let png_data: *mut Object = msg_send![ + bitmap_rep, + representationUsingType: NS_BITMAP_IMAGE_FILE_TYPE_PNG + properties: properties + ]; + if png_data.is_null() { + e2e_write_native_snapshot_status( + &status_for_block, + "failed", + &label_for_status, + &output_for_block, + Some("NSBitmapImageRep PNG representation was nil"), + ); + return; + } + let bytes: *const std::ffi::c_void = msg_send![png_data, bytes]; + let len: usize = msg_send![png_data, length]; + if bytes.is_null() || len == 0 { + e2e_write_native_snapshot_status( + &status_for_block, + "failed", + &label_for_status, + &output_for_block, + Some("PNG data was empty"), + ); + return; + } + let slice = std::slice::from_raw_parts(bytes.cast::(), len); + let tmp_output = output_for_block.with_extension("png.tmp"); + let write_result = std::fs::write(&tmp_output, slice) + .and_then(|()| std::fs::rename(&tmp_output, &output_for_block)); + match write_result { + Ok(()) => { + log::info!( + "NIXMAC_E2E_NATIVE_SNAPSHOT wrote {} for {}", + output_for_block.display(), + label_for_status + ); + e2e_write_native_snapshot_status( + &status_for_block, + "passed", + &label_for_status, + &output_for_block, + None, + ); + } + Err(error) => { + let _ = std::fs::remove_file(&tmp_output); + log::warn!( + "NIXMAC_E2E_NATIVE_SNAPSHOT failed to write {} for {}: {}", + output_for_block.display(), + label_for_status, + error + ); + e2e_write_native_snapshot_status( + &status_for_block, + "failed", + &label_for_status, + &output_for_block, + Some(&format!("failed to write PNG atomically: {error}")), + ); + } + } + }) + .copy(); + let webview = webview.inner() as *mut Object; + let configuration: *mut Object = std::ptr::null_mut(); + let _: () = msg_send![ + webview, + takeSnapshotWithConfiguration: configuration + completionHandler: &*completion + ]; + // The completion is async and owned by WebKit after dispatch. Leaking the + // copied block is acceptable in debug-only E2E runs and avoids use-after-free + // while the virtualized host is under load. + std::mem::forget(completion); + log::debug!( + "NIXMAC_E2E_NATIVE_SNAPSHOT requested {} for {}", + output_path.display(), + label_for_log + ); + }) { + log::warn!( + "NIXMAC_E2E_NATIVE_SNAPSHOT unavailable for {}: {}", + error_label, + error + ); + e2e_write_native_snapshot_status( + &error_status_path, + "failed", + &error_label, + &error_output_path, + Some(&format!("webview unavailable: {error}")), + ); + } +} + +#[cfg(all(debug_assertions, target_os = "macos"))] +fn e2e_schedule_native_webview_snapshot( + window: WebviewWindow, + label: &'static str, + delay: Duration, +) { + std::thread::spawn(move || { + std::thread::sleep(delay); + if let Some((output_path, status_path)) = e2e_native_snapshot_paths(label) { + e2e_request_native_webview_snapshot( + &window, + label.to_string(), + output_path, + status_path, + ); + } + }); +} + +#[cfg(all(debug_assertions, target_os = "macos"))] +fn e2e_start_native_webview_snapshot_request_poller(window: WebviewWindow) { + let Some(root) = e2e_native_snapshot_root_dir() else { + return; + }; + let request_dir = root.join("requests"); + if let Err(error) = std::fs::create_dir_all(&request_dir) { + log::warn!( + "NIXMAC_E2E_NATIVE_SNAPSHOT could not create request dir {}: {}", + request_dir.display(), + error + ); + return; + } + log::info!( + "NIXMAC_E2E_NATIVE_SNAPSHOT polling requests in {}", + request_dir.display() + ); + std::thread::spawn(move || { + let started = std::time::Instant::now(); + let max_runtime = Duration::from_secs(2 * 60 * 60); + while started.elapsed() < max_runtime { + let Ok(entries) = std::fs::read_dir(&request_dir) else { + std::thread::sleep(Duration::from_millis(250)); + continue; + }; + for entry in entries.flatten() { + let request_path = entry.path(); + let Some(file_name) = request_path.file_name().and_then(|name| name.to_str()) + else { + continue; + }; + let Some(request_id) = file_name.strip_suffix(".request") else { + continue; + }; + let label = std::fs::read_to_string(&request_path) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| request_id.to_string()); + let _ = std::fs::remove_file(&request_path); + let output_path = root.join(format!("{request_id}.png")); + let status_path = root.join(format!("{request_id}.json")); + e2e_request_native_webview_snapshot(&window, label, output_path, status_path); + } + std::thread::sleep(Duration::from_millis(250)); + } + log::warn!("NIXMAC_E2E_NATIVE_SNAPSHOT request poller stopped after max runtime"); + }); +} + +#[cfg(not(all(debug_assertions, target_os = "macos")))] +fn e2e_schedule_native_webview_snapshot( + _window: WebviewWindow, + _label: &'static str, + _delay: Duration, +) { +} + +#[cfg(not(all(debug_assertions, target_os = "macos")))] +fn e2e_start_native_webview_snapshot_request_poller(_window: WebviewWindow) {} + const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" (() => { const styleId = "nixmac-e2e-capture-background"; @@ -1093,6 +1456,11 @@ fn run_gui_mode( "native-page-load-finished-plus-5s", Duration::from_secs(5), ); + e2e_schedule_native_webview_snapshot( + window.clone(), + "page-load-finished-plus-5s", + Duration::from_secs(5), + ); } } }); @@ -1162,11 +1530,17 @@ fn run_gui_mode( } if e2e_css_capture { + e2e_start_native_webview_snapshot_request_poller(main_window.clone()); e2e_schedule_native_webview_capture_probe( main_window.clone(), "native-post-build-plus-2s", Duration::from_secs(2), ); + e2e_schedule_native_webview_snapshot( + main_window.clone(), + "post-build-plus-2s", + Duration::from_secs(2), + ); } if e2e_webview_watchdog { diff --git a/tests/e2e/adapters/nixmac.sh b/tests/e2e/adapters/nixmac.sh index 802fc86ff..b78347527 100644 --- a/tests/e2e/adapters/nixmac.sh +++ b/tests/e2e/adapters/nixmac.sh @@ -170,7 +170,37 @@ nixmac_text() { } nixmac_screenshot() { - screenshot "${1:-nixmac}" "$NIXMAC_APP_NAME" + local label="${1:-nixmac}" + local system_path native_path native_dest system_diag_dir system_diag_path + + system_path=$(screenshot "$label" "$NIXMAC_APP_NAME" | tail -n 1) + if ! declare -f nixmac_pp_capture_native_visual_signal >/dev/null 2>&1; then + printf '%s\n' "$system_path" + return 0 + fi + + native_path=$(nixmac_pp_capture_native_visual_signal "$label") || { + printf '%s\n' "$system_path" + return 0 + } + + mkdir -p "$E2E_SCREENSHOT_DIR" 2>/dev/null || true + native_dest="$E2E_SCREENSHOT_DIR/${label//[^a-zA-Z0-9._-]/_}-webkit-snapshot-$(date +%s)-$$-$RANDOM.png" + cp "$native_path" "$native_dest" 2>/dev/null || { + printf '%s\n' "$system_path" + return 0 + } + + if [ -n "$system_path" ] && [ -f "$system_path" ]; then + system_diag_dir="${NIXMAC_E2E_DIAGNOSTICS_DIR:-$E2E_DIAGNOSTIC_DIR}/system-captures" + mkdir -p "$system_diag_dir" 2>/dev/null || true + system_diag_path="$system_diag_dir/$(basename "$system_path" .png)-peekaboo-system.png" + mv "$system_path" "$system_diag_path" 2>/dev/null || true + log "Promoted native WKWebView snapshot for $label; retained system capture diagnostic: $system_diag_path" + else + log "Promoted native WKWebView snapshot for $label" + fi + printf '%s\n' "$native_dest" } nixmac_click_button() { diff --git a/tests/e2e/lib/nixmac_product_proof.sh b/tests/e2e/lib/nixmac_product_proof.sh index 037d2e214..256e663ad 100644 --- a/tests/e2e/lib/nixmac_product_proof.sh +++ b/tests/e2e/lib/nixmac_product_proof.sh @@ -395,22 +395,80 @@ console.log(`central app content visual signal ready: YAVG ${yAvg}, Y range ${yM NODE } +nixmac_pp_request_native_webview_snapshot() { + local label="${1:-snapshot}" + local timeout="${2:-15}" + local root request_dir safe_label request_id request_path output_path status_path deadline status + + [ -n "${NIXMAC_E2E_DIAGNOSTICS_DIR:-}" ] || return 1 + root="$NIXMAC_E2E_DIAGNOSTICS_DIR/native-webview-snapshots" + request_dir="$root/requests" + mkdir -p "$request_dir" "$root" || return 1 + safe_label=$(printf '%s' "$label" | tr -cs 'A-Za-z0-9._-' '-' | sed -E 's/^-+|-+$//g' | cut -c1-80) + [ -n "$safe_label" ] || safe_label="snapshot" + request_id="${safe_label}-$(date +%s)-$$-$RANDOM" + request_path="$request_dir/$request_id.request" + output_path="$root/$request_id.png" + status_path="$root/$request_id.json" + + printf '%s\n' "$label" > "$request_path" || return 1 + deadline=$(($(date +%s) + timeout)) + while [ "$(date +%s)" -le "$deadline" ]; do + if [ -s "$output_path" ]; then + printf '%s\n' "$output_path" + return 0 + fi + if [ -s "$status_path" ]; then + status=$(jq -r '.status // ""' "$status_path" 2>/dev/null || true) + if [ "$status" = "failed" ]; then + debug "Native WKWebView snapshot failed for $label: $(jq -r '.message // "unknown"' "$status_path" 2>/dev/null || echo unknown)" + return 1 + fi + fi + sleep 0.2 + done + + debug "Native WKWebView snapshot timed out for $label" + return 1 +} + +nixmac_pp_capture_native_visual_signal() { + local label="${1:-ready-shell}" + local path result + + path=$(nixmac_pp_request_native_webview_snapshot "$label" 5) || return 1 + result=$(nixmac_pp_screenshot_has_visual_signal "$path" 2>&1) || { + debug "Native WKWebView visual signal not established for $path: $result" + return 1 + } + debug "Native WKWebView snapshot visual signal ready for $path: $result" + printf '%s\n' "$path" +} + nixmac_pp_capture_ready_visual_signal() { local label="${1:-ready-shell}" - local dir path result + local dir path result native_path dir="$E2E_DIAGNOSTIC_DIR/visual-readiness" mkdir -p "$dir" path="$dir/${label//[^a-zA-Z0-9._-]/_}.png" - peekaboo_run see --app "$NIXMAC_APP_NAME" --path "$path" >/dev/null 2>&1 \ - || peekaboo_run image --mode screen --path "$path" >/dev/null 2>&1 \ - || return 1 - [ -s "$path" ] || return 1 - result=$(nixmac_pp_screenshot_has_visual_signal "$path" 2>&1) || { + if peekaboo_run see --app "$NIXMAC_APP_NAME" --path "$path" >/dev/null 2>&1 \ + || peekaboo_run image --mode screen --path "$path" >/dev/null 2>&1; then + result=$(nixmac_pp_screenshot_has_visual_signal "$path" 2>&1) && { + debug "$result" + return 0 + } + debug "Ready-shell system visual signal not established for $path: $result" + else + result="system screenshot capture failed" + debug "Ready-shell system visual signal not established for $path: $result" + fi + + native_path=$(nixmac_pp_capture_native_visual_signal "$label") || { debug "Ready-shell visual signal not established for $path: $result" return 1 } - debug "$result" + debug "Ready-shell visual signal established from native WKWebView snapshot: $native_path" } nixmac_pp_wait_for_ready_app_shell() { diff --git a/tools/computer-use-e2e/peekaboo-runner.mjs b/tools/computer-use-e2e/peekaboo-runner.mjs index b69cf2950..37c33a014 100644 --- a/tools/computer-use-e2e/peekaboo-runner.mjs +++ b/tools/computer-use-e2e/peekaboo-runner.mjs @@ -173,7 +173,9 @@ function artifactEntries(dirPath, runDir) { label: entry.replace(/\.[^.]+$/, ''), path: path.relative(runDir, fullPath), capturedAt: new Date(fileStat.mtimeMs).toISOString(), - note: 'Captured by Peekaboo runner.', + note: /webkit-snapshot/i.test(entry) + ? 'WKWebView internal snapshot captured from the running nixmac WebContent surface.' + : 'Captured by Peekaboo runner.', bytes: fileStat.size, }; }); diff --git a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs index e87f7c8a6..a5fa2906c 100644 --- a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs +++ b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs @@ -24,6 +24,7 @@ const frontendWidgetPath = path.join(repoRoot, 'apps/native/src/components/widge const frontendEditorPanelPath = path.join(repoRoot, 'apps/native/src/components/widget/overlays/editor-panel.tsx'); const frontendBootDiagnosticsPath = path.join(repoRoot, 'apps/native/src/lib/e2e-boot-diagnostics.ts'); const tauriApiPath = path.join(repoRoot, 'apps/native/src/tauri-api.ts'); +const cargoTomlPath = path.join(repoRoot, 'apps/native/src-tauri/Cargo.toml'); const workflow = readFileSync(workflowPath, 'utf8'); const productProof = readFileSync(productProofPath, 'utf8'); const peekabooShell = readFileSync(peekabooShellPath, 'utf8'); @@ -42,6 +43,7 @@ const frontendWidget = readFileSync(frontendWidgetPath, 'utf8'); const frontendEditorPanel = readFileSync(frontendEditorPanelPath, 'utf8'); const frontendBootDiagnostics = readFileSync(frontendBootDiagnosticsPath, 'utf8'); const tauriApi = readFileSync(tauriApiPath, 'utf8'); +const cargoToml = readFileSync(cargoTomlPath, 'utf8'); function section(startPattern, endPattern = null) { const source = typeof startPattern === 'object' && startPattern.sourceText ? startPattern.sourceText : workflow; @@ -218,6 +220,15 @@ assert.match(nativeMain, /if e2e_css_capture \{[\s\S]*match main_window\.ns_wind assert.match(nativeMain, /NIXMAC_E2E_CAPTURE native window diagnostics:[\s\S]*sharingTypeBefore=\{\}[\s\S]*sharingTypeAfter=\{\}[\s\S]*isOpaque[\s\S]*alphaValue[\s\S]*hasShadow/, 'Native app must log native sharing diagnostics for MacInCloud capture debugging'); assert.match(nativeMain, /fn e2e_request_native_webview_capture_probe[\s\S]*with_webview\(move \|webview\|[\s\S]*respondsToSelector: sel!\(drawsBackground\)[\s\S]*setNeedsDisplay: YES[\s\S]*displayIfNeeded[\s\S]*NIXMAC_E2E_CAPTURE webview diagnostics:[\s\S]*fn e2e_schedule_native_webview_capture_probe/, 'Native app must provide guarded WebView diagnostics and a best-effort AppKit display hint for MacInCloud black-capture diagnosis'); assert.match(nativeMain, /e2e_page_load_native_capture_probe[\s\S]*native-page-load-finished-plus-5s[\s\S]*e2e_schedule_native_webview_capture_probe[\s\S]*native-post-build-plus-2s/, 'Native app must schedule native WebView capture diagnostics after build and after page-load'); +assert.match(cargoToml, /\[target\.'cfg\(target_os = "macos"\)'\.dependencies\][\s\S]*block = "0\.1"[\s\S]*objc = "0\.2"/, 'Native WKWebView snapshot completion blocks must declare the small macOS-only block crate beside objc/cocoa'); +assert.match(nativeMain, /fn e2e_request_native_webview_snapshot[\s\S]*ConcreteBlock::new[\s\S]*NSBitmapImageRep[\s\S]*representationUsingType[\s\S]*takeSnapshotWithConfiguration[\s\S]*std::mem::forget\(completion\)/, 'Native app must provide an E2E-only WKWebView snapshot writer using WebKit snapshot API and explicit async block lifetime handling'); +assert.match(nativeMain, /let tmp_status_path = status_path\.with_extension\("json\.tmp"\)[\s\S]*std::fs::write\([\s\S]*&tmp_status_path[\s\S]*std::fs::rename\(&tmp_status_path, status_path\)/, 'Native WKWebView snapshot status writer must publish JSON atomically so shell polling cannot read partial status'); +assert.match(nativeMain, /let tmp_output = output_for_block\.with_extension\("png\.tmp"\)[\s\S]*std::fs::write\(&tmp_output, slice\)[\s\S]*std::fs::rename\(&tmp_output, &output_for_block\)/, 'Native WKWebView snapshot writer must publish PNGs atomically so shell polling cannot read partial files'); +assert.match(nativeMain, /fn e2e_native_snapshot_root_dir[\s\S]*NIXMAC_E2E_DIAGNOSTICS_DIR[\s\S]*native-webview-snapshots/, 'Native snapshots must reuse the existing E2E diagnostics directory'); +assert.match(nativeMain, /fn e2e_start_native_webview_snapshot_request_poller[\s\S]*request_dir[\s\S]*requests[\s\S]*\.request[\s\S]*e2e_request_native_webview_snapshot/, 'Native app must poll diagnostics-dir snapshot requests so Scott’s shell driver can demand fresh native visual evidence'); +assert.match(nativeMain, /let output_path = root\.join\(format!\("\{request_id\}\.png"\)\);[\s\S]*let status_path = root\.join\(format!\("\{request_id\}\.json"\)\);/, 'Native snapshot poller must use the shell request id verbatim so shell and Rust wait/write paths cannot desync on long labels'); +assert.match(nativeMain, /e2e_schedule_native_webview_snapshot[\s\S]*page-load-finished-plus-5s[\s\S]*e2e_start_native_webview_snapshot_request_poller[\s\S]*post-build-plus-2s/, 'Native app must take bounded readiness snapshots and start the on-demand snapshot request poller only in E2E capture mode'); +assert.doesNotMatch(nativeMain, /setInterval|Duration::from_millis\((?:1000|2000|3000)\)[\s\S]*e2e_request_native_webview_snapshot/, 'Native app must not create noisy background interval snapshots while scenarios run'); assert.match(nativeSolidCaptureBranch, /NIXMAC_E2E_SOLID_CAPTURE enabled/, 'Native app must keep an explicit solid-capture branch for diagnostics'); assert.doesNotMatch(nativeSolidCaptureBranch, /background_color\(tauri::utils::config::Color\(10, 10, 10, 255\)\)/, 'Native app must not apply native background_color from the default solid-capture path'); assert.match(nativeMain, /fn e2e_webview_watchdog_enabled\(\) -> bool \{\n\s+cfg!\(debug_assertions\) && crate::e2e_runtime::enabled\("NIXMAC_E2E_WEBVIEW_WATCHDOG"\)/, 'Native app must expose an E2E-only WebView watchdog gate independent of opaque capture'); @@ -283,7 +294,13 @@ assert.match(productProof, /nixmac_pp_wait_for_ready_app_shell\(\)/, 'Product Pr assert.match(productProof, /nixmac_pp_elements_show_ready_shell\(\)[\s\S]*NIXMAC_PP_READY_SHELL_MIN_ELEMENTS[\s\S]*NIXMAC_PP_READY_SHELL_PATTERN/, 'ready-shell gate must require both element breadth and product markers'); assert.match(productProof, /nixmac_pp_screenshot_has_visual_signal\(\)[\s\S]*visual-proof\.mjs[\s\S]*pngSignalStats[\s\S]*probeCropForImage/, 'ready-shell gate must use the same screenshot signal helpers as the report scanner'); assert.match(productProof, /maxDarkChromeYAvg: 42/, 'ready-shell visual gate must enforce the same nixmac dark-capture upper bound as the report scanner'); +assert.match(productProof, /nixmac_pp_request_native_webview_snapshot\(\)[\s\S]*NIXMAC_E2E_DIAGNOSTICS_DIR[\s\S]*native-webview-snapshots[\s\S]*\.request[\s\S]*status_path/, 'Product Proof must request fresh native WKWebView snapshots through the existing diagnostics directory'); +assert.match(productProof, /nixmac_pp_capture_native_visual_signal\(\)[\s\S]*nixmac_pp_request_native_webview_snapshot[\s\S]*nixmac_pp_screenshot_has_visual_signal/, 'Native WKWebView fallback must run the exact same visual signal probe before it can satisfy readiness'); +assert.match(productProof, /nixmac_pp_capture_ready_visual_signal\(\)[\s\S]*peekaboo_run see --app "\$NIXMAC_APP_NAME" --path "\$path"[\s\S]*nixmac_pp_screenshot_has_visual_signal "\$path"[\s\S]*nixmac_pp_capture_native_visual_signal/, 'Ready-shell gate must try system pixels first, then use a fresh native WKWebView snapshot only as a strict fallback'); assert.match(productProof, /nixmac_pp_wait_for_ready_app_shell\(\)[\s\S]*nixmac_pp_capture_ready_visual_signal/, 'ready-shell gate must require screenshot visual signal before launch passes'); +assert.match(nixmacAdapter, /nixmac_screenshot\(\)[\s\S]*screenshot "\$label" "\$NIXMAC_APP_NAME"[\s\S]*nixmac_pp_capture_native_visual_signal[\s\S]*webkit-snapshot[\s\S]*system-captures[\s\S]*Promoted native WKWebView snapshot/, 'nixmac screenshots must promote passing WKWebView snapshots into screenshot evidence while retaining black system captures as diagnostics'); +assert.match(peekabooRunner, /webkit-snapshot[\s\S]*WKWebView internal snapshot captured from the running nixmac WebContent surface/, 'Peekaboo runner must label WKWebView snapshot screenshot provenance'); +assert.match(runLocal, /webkit-snapshot[\s\S]*WKWebView internal snapshot[\s\S]*running WKWebView WebContent surface/, 'HTML report must visibly distinguish WKWebView internal snapshots from Peekaboo screen captures'); const setKeys = [...launchEnv.matchAll(/nixmac_pp_set_launch_env ([A-Z0-9_]+)/g)].map((match) => match[1]); const unsetKeys = new Set([...cleanup.matchAll(/nixmac_pp_unset_launch_env ([A-Z0-9_]+)/g)].map((match) => match[1])); diff --git a/tools/computer-use-e2e/run-local.mjs b/tools/computer-use-e2e/run-local.mjs index 131b919ac..3bdc09812 100644 --- a/tools/computer-use-e2e/run-local.mjs +++ b/tools/computer-use-e2e/run-local.mjs @@ -1873,7 +1873,8 @@ function screenshotFamily(labelOrPath) { return path .basename(String(labelOrPath ?? 'screenshot')) .replace(/\.[^.]+$/, '') - .replace(/_annotated$/, ''); + .replace(/_annotated$/, '') + .replace(/-webkit-snapshot-\d+$/i, ''); } function proofGalleryItems(state) { @@ -2205,6 +2206,7 @@ function renderGallery(state) { const { label } = splitScreenshotLabel(item.primary.label); const displayLabel = humanizeScreenshotLabel(label); const callouts = galleryCallouts(item); + const isWebkitSnapshot = /webkit-snapshot/i.test(`${item.primary.label ?? ''} ${item.primary.path ?? ''}`); return `
${escapeHtml(item.primary.label)} @@ -2216,10 +2218,10 @@ function renderGallery(state) {
${escapeHtml(displayLabel || item.primary.label)} - review highlight + ${escapeHtml(isWebkitSnapshot ? 'WKWebView internal snapshot' : 'review highlight')} ${escapeHtml(galleryScenarioLabel(item))} - ${escapeHtml(item.primary.note || 'Screenshot proof')} - Captured ${escapeHtml(item.primary.capturedAt || 'time unavailable')} from the raw screenshot; video/storyboard frames remain raw. + Captured ${escapeHtml(item.primary.capturedAt || 'time unavailable')} from ${escapeHtml(isWebkitSnapshot ? 'the running WKWebView WebContent surface' : 'the raw screenshot')}; video/storyboard frames remain raw. ${ item.primary.windowTitle ? `Window title at capture: ${escapeHtml(item.primary.windowTitle)}` From 6569df448904dde411972a1e0c05e6646d0a09d2 Mon Sep 17 00:00:00 2001 From: fkb032 <249513614+fkb032@users.noreply.github.com> Date: Thu, 7 May 2026 15:20:34 -0700 Subject: [PATCH 06/16] fix: force e2e webview mount on macincloud --- apps/native/src-tauri/src/main.rs | 80 +++++++++++++++++-- apps/native/src/main.tsx | 16 +++- .../peekaboo-workflow-contract-self-test.mjs | 5 +- 3 files changed, 91 insertions(+), 10 deletions(-) diff --git a/apps/native/src-tauri/src/main.rs b/apps/native/src-tauri/src/main.rs index fc0170610..530553d4d 100644 --- a/apps/native/src-tauri/src/main.rs +++ b/apps/native/src-tauri/src/main.rs @@ -354,6 +354,49 @@ fn e2e_write_native_snapshot_status( } } +#[cfg(all(debug_assertions, target_os = "macos"))] +unsafe fn e2e_nsstring_to_string(value: *mut objc::runtime::Object) -> Option { + use objc::{msg_send, sel, sel_impl}; + + if value.is_null() { + return None; + } + let utf8: *const std::os::raw::c_char = msg_send![value, UTF8String]; + if utf8.is_null() { + return None; + } + Some( + std::ffi::CStr::from_ptr(utf8) + .to_string_lossy() + .into_owned(), + ) +} + +#[cfg(all(debug_assertions, target_os = "macos"))] +unsafe fn e2e_ns_error_summary(error: *mut objc::runtime::Object) -> String { + use objc::{msg_send, sel, sel_impl}; + + if error.is_null() { + return "WebKit returned an error".to_string(); + } + let domain: *mut objc::runtime::Object = msg_send![error, domain]; + let description: *mut objc::runtime::Object = msg_send![error, localizedDescription]; + let user_info: *mut objc::runtime::Object = msg_send![error, userInfo]; + let user_info_description: *mut objc::runtime::Object = if user_info.is_null() { + std::ptr::null_mut() + } else { + msg_send![user_info, description] + }; + let code: i64 = msg_send![error, code]; + format!( + "WebKit returned an error: domain={} code={} description={} userInfo={}", + e2e_nsstring_to_string(domain).unwrap_or_else(|| "unknown".to_string()), + code, + e2e_nsstring_to_string(description).unwrap_or_else(|| "unknown".to_string()), + e2e_nsstring_to_string(user_info_description).unwrap_or_else(|| "unknown".to_string()) + ) +} + #[cfg(all(debug_assertions, target_os = "macos"))] fn e2e_request_native_webview_snapshot( window: &WebviewWindow, @@ -400,16 +443,18 @@ fn e2e_request_native_webview_snapshot( let status_for_block = status_path.clone(); let completion = ConcreteBlock::new(move |image: *mut Object, error: *mut Object| { if !error.is_null() { + let error_message = e2e_ns_error_summary(error); log::warn!( - "NIXMAC_E2E_NATIVE_SNAPSHOT failed for {}: WebKit returned an error", - label_for_status + "NIXMAC_E2E_NATIVE_SNAPSHOT failed for {}: {}", + label_for_status, + error_message ); e2e_write_native_snapshot_status( &status_for_block, "failed", &label_for_status, &output_for_block, - Some("WebKit returned an error"), + Some(&error_message), ); return; } @@ -1483,10 +1528,10 @@ fn run_gui_mode( })?; log::debug!("Main nixmac window created"); - #[cfg(target_os = "macos")] + #[cfg(all(debug_assertions, target_os = "macos"))] if e2e_css_capture { use objc::msg_send; - use objc::runtime::Object; + use objc::runtime::{Object, NO}; use objc::sel; use objc::sel_impl; @@ -1494,6 +1539,24 @@ fn run_gui_mode( Ok(ns_window) => unsafe { let ns_window = ns_window as *mut Object; let sharing_type_before: usize = msg_send![ns_window, sharingType]; + let can_disable_occlusion: bool = msg_send![ + ns_window, + respondsToSelector: sel!(_setWindowOcclusionDetectionEnabled:) + ]; + if can_disable_occlusion { + let _: () = + msg_send![ns_window, _setWindowOcclusionDetectionEnabled: NO]; + } + let can_disable_hide_on_deactivate: bool = + msg_send![ns_window, respondsToSelector: sel!(setHidesOnDeactivate:)]; + if can_disable_hide_on_deactivate { + let _: () = msg_send![ns_window, setHidesOnDeactivate: NO]; + } + let can_disable_can_hide: bool = + msg_send![ns_window, respondsToSelector: sel!(setCanHide:)]; + if can_disable_can_hide { + let _: () = msg_send![ns_window, setCanHide: NO]; + } if let Err(error) = main_window.set_content_protected(false) { log::warn!( "NIXMAC_E2E_CAPTURE native window could not disable content protection: {}", @@ -1506,7 +1569,7 @@ fn run_gui_mode( let level: i64 = msg_send![ns_window, level]; let has_shadow: bool = msg_send![ns_window, hasShadow]; log::info!( - "NIXMAC_E2E_CAPTURE native window diagnostics: mode={} sharingTypeBefore={} sharingTypeAfter={} isOpaque={} alphaValue={:.3} level={} hasShadow={}", + "NIXMAC_E2E_CAPTURE native window diagnostics: mode={} sharingTypeBefore={} sharingTypeAfter={} isOpaque={} alphaValue={:.3} level={} hasShadow={} occlusionDetectionDisabled={} hidesOnDeactivateDisabled={} canHideDisabled={}", if e2e_opaque_window { "opaque" } else { @@ -1517,7 +1580,10 @@ fn run_gui_mode( is_opaque, alpha_value, level, - has_shadow + has_shadow, + can_disable_occlusion, + can_disable_hide_on_deactivate, + can_disable_can_hide ); }, Err(error) => { diff --git a/apps/native/src/main.tsx b/apps/native/src/main.tsx index c27a7ccf9..939bbbcf8 100644 --- a/apps/native/src/main.tsx +++ b/apps/native/src/main.tsx @@ -1,4 +1,5 @@ import React from "react"; +import { flushSync } from "react-dom"; import ReactDOM from "react-dom/client"; import * as Sentry from "@sentry/react"; import App from "./App"; @@ -414,7 +415,7 @@ if (E2E_BOOT_PREFS_DISABLED) { const renderApp = () => { bootBreadcrumb("React render start"); markBootStage("react-render-start"); - root.render( + const app = ( } @@ -426,8 +427,19 @@ const renderApp = () => { > - , + ); + if (E2E_BOOT_PREFS_DISABLED) { + bootBreadcrumb("React render flushSync start"); + flushSync(() => { + root.render(app); + }); + bootBreadcrumb("React render flushSync complete", { + rootChildCount: rootElement.childElementCount, + }); + } else { + root.render(app); + } bootBreadcrumb("React render scheduled"); markBootStage("react-render-scheduled"); }; diff --git a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs index a5fa2906c..c407837b0 100644 --- a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs +++ b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs @@ -217,10 +217,12 @@ assert.match(nativeMain, /const rectFor = \(element\)[\s\S]*document\.elementFro assert.match(nativeCaptureWindowSetup, /let e2e_solid_capture = e2e_solid_capture_enabled\(\);[\s\S]*let e2e_css_capture = e2e_solid_capture \|\| e2e_opaque_window[\s\S]*transparent\(!e2e_opaque_window\)/, 'Native app must keep default solid capture CSS-backed and transparent while limiting native opacity to opaque debug mode'); assert.match(nativeCaptureWindowSetup, /if e2e_opaque_window \{\n\s+main_window_builder = main_window_builder\s+\.background_color\(tauri::utils::config::Color\(10, 10, 10, 255\)\);[\s\S]*if e2e_css_capture \{\n\s+main_window_builder =\s+main_window_builder\.initialization_script\(E2E_CAPTURE_DARK_BACKGROUND_SCRIPT\);/, 'Native app must keep native dark background only in the opaque debug path while applying CSS capture for solid and opaque modes'); assert.match(nativeMain, /if e2e_css_capture \{[\s\S]*match main_window\.ns_window\(\) \{[\s\S]*set_content_protected\(false\)/, 'Native app must explicitly make E2E capture windows readable from the native diagnostics branch'); -assert.match(nativeMain, /NIXMAC_E2E_CAPTURE native window diagnostics:[\s\S]*sharingTypeBefore=\{\}[\s\S]*sharingTypeAfter=\{\}[\s\S]*isOpaque[\s\S]*alphaValue[\s\S]*hasShadow/, 'Native app must log native sharing diagnostics for MacInCloud capture debugging'); +assert.match(nativeMain, /respondsToSelector: sel!\(_setWindowOcclusionDetectionEnabled:\)[\s\S]*_setWindowOcclusionDetectionEnabled: NO[\s\S]*setHidesOnDeactivate: NO[\s\S]*setCanHide: NO/, 'Native app must disable macOS occlusion/hiding behavior in the debug-only opaque E2E capture branch'); +assert.match(nativeMain, /NIXMAC_E2E_CAPTURE native window diagnostics:[\s\S]*sharingTypeBefore=\{\}[\s\S]*sharingTypeAfter=\{\}[\s\S]*isOpaque[\s\S]*alphaValue[\s\S]*hasShadow[\s\S]*occlusionDetectionDisabled/, 'Native app must log native sharing and occlusion diagnostics for MacInCloud capture debugging'); assert.match(nativeMain, /fn e2e_request_native_webview_capture_probe[\s\S]*with_webview\(move \|webview\|[\s\S]*respondsToSelector: sel!\(drawsBackground\)[\s\S]*setNeedsDisplay: YES[\s\S]*displayIfNeeded[\s\S]*NIXMAC_E2E_CAPTURE webview diagnostics:[\s\S]*fn e2e_schedule_native_webview_capture_probe/, 'Native app must provide guarded WebView diagnostics and a best-effort AppKit display hint for MacInCloud black-capture diagnosis'); assert.match(nativeMain, /e2e_page_load_native_capture_probe[\s\S]*native-page-load-finished-plus-5s[\s\S]*e2e_schedule_native_webview_capture_probe[\s\S]*native-post-build-plus-2s/, 'Native app must schedule native WebView capture diagnostics after build and after page-load'); assert.match(cargoToml, /\[target\.'cfg\(target_os = "macos"\)'\.dependencies\][\s\S]*block = "0\.1"[\s\S]*objc = "0\.2"/, 'Native WKWebView snapshot completion blocks must declare the small macOS-only block crate beside objc/cocoa'); +assert.match(nativeMain, /fn e2e_ns_error_summary[\s\S]*localizedDescription[\s\S]*domain[\s\S]*code[\s\S]*userInfo/, 'Native WKWebView snapshot failures must persist the actual NSError domain, code, description, and userInfo'); assert.match(nativeMain, /fn e2e_request_native_webview_snapshot[\s\S]*ConcreteBlock::new[\s\S]*NSBitmapImageRep[\s\S]*representationUsingType[\s\S]*takeSnapshotWithConfiguration[\s\S]*std::mem::forget\(completion\)/, 'Native app must provide an E2E-only WKWebView snapshot writer using WebKit snapshot API and explicit async block lifetime handling'); assert.match(nativeMain, /let tmp_status_path = status_path\.with_extension\("json\.tmp"\)[\s\S]*std::fs::write\([\s\S]*&tmp_status_path[\s\S]*std::fs::rename\(&tmp_status_path, status_path\)/, 'Native WKWebView snapshot status writer must publish JSON atomically so shell polling cannot read partial status'); assert.match(nativeMain, /let tmp_output = output_for_block\.with_extension\("png\.tmp"\)[\s\S]*std::fs::write\(&tmp_output, slice\)[\s\S]*std::fs::rename\(&tmp_output, &output_for_block\)/, 'Native WKWebView snapshot writer must publish PNGs atomically so shell polling cannot read partial files'); @@ -251,6 +253,7 @@ assert.match(frontendBootDiagnostics, /export function recordE2eDomSnapshot[\s\S assert.match(frontendBootDiagnostics, /export function scheduleE2eDomSnapshots[\s\S]*count = 5[\s\S]*intervalMs = 2_000[\s\S]*emitted < count/, 'E2E DOM diagnostics must schedule a bounded post-mount snapshot series and self-stop'); assert.match(frontendMain, /PREFS_BOOT_TIMEOUT_MS = 8000[\s\S]*ui_get_prefs invoke start[\s\S]*success after timeout[\s\S]*Promise\.race\(\[prefsPromise, timeoutPromise\]\)/, 'Frontend boot must log prefs IPC progress with clear after-timeout labels'); assert.match(frontendMain, /markBootStage\("main-loaded"\)[\s\S]*markBootStage\("root-found"\)[\s\S]*markBootStage\("react-render-start"\)[\s\S]*markBootStage\("react-render-scheduled"\)/, 'Frontend boot must synchronously mark module, root, and render-scheduling stages'); +assert.match(frontendMain, /import \{ flushSync \} from "react-dom";[\s\S]*if \(E2E_BOOT_PREFS_DISABLED\) \{[\s\S]*React render flushSync start[\s\S]*flushSync\(\(\) => \{[\s\S]*root\.render\(app\);[\s\S]*React render flushSync complete/, 'Frontend must force the initial React render synchronously in E2E mode so MacInCloud cannot stall at react-render-scheduled'); assert.match(frontendApp, /markBootStage\("app-render"\)[\s\S]*markBootStage\("app-effect"\)[\s\S]*clearBootStage\(\)/, 'App must synchronously mark render/effect stages and clear the E2E title marker after mount'); assert.match(frontendWidget, /markBootStage\("darwin-widget-render"\)/, 'DarwinWidget must mark when the product widget render body is reached'); assert.match(frontendEditorPanel, /const LazyNixEditor = lazy\(async \(\) => \{[\s\S]*import\("@\/components\/kibo-ui\/nix-editor"\)[\s\S]*default: module\.NixEditor/, 'EditorPanel must lazy-load the Monaco-backed Nix editor only when a file is opened'); From b7ca5848eae445f731d15d2b9aee14ca26af601d Mon Sep 17 00:00:00 2001 From: fkb032 <249513614+fkb032@users.noreply.github.com> Date: Thu, 7 May 2026 15:55:56 -0700 Subject: [PATCH 07/16] chore: add peekaboo webview proof diagnostics --- apps/native/src-tauri/src/commands/debug.rs | 12 +- apps/native/src-tauri/src/main.rs | 99 ++++++++++++++ tools/computer-use-e2e/peekaboo-runner.mjs | 129 +++++++++++++++++- .../peekaboo-workflow-contract-self-test.mjs | 4 + 4 files changed, 236 insertions(+), 8 deletions(-) diff --git a/apps/native/src-tauri/src/commands/debug.rs b/apps/native/src-tauri/src/commands/debug.rs index 2d6e00910..405b17f8d 100644 --- a/apps/native/src-tauri/src/commands/debug.rs +++ b/apps/native/src-tauri/src/commands/debug.rs @@ -93,6 +93,15 @@ fn clean_field(value: &str, max_len: usize) -> String { .to_string() } +#[cfg(debug_assertions)] +fn e2e_breadcrumb_detail_limit(label: &str) -> usize { + if label.starts_with("native webview boot probe ") || label.starts_with("e2e-asset-fetch-") { + 12_000 + } else { + 1_000 + } +} + /// Records frontend boot breadcrumbs for E2E diagnostics. /// /// This is debug-only and writes only when the E2E runtime file/env provides a @@ -164,8 +173,9 @@ fn write_e2e_breadcrumb( if label.is_empty() { return Ok(()); } + let detail_limit = e2e_breadcrumb_detail_limit(&label); let detail = detail - .map(|value| clean_field(value, 1_000)) + .map(|value| clean_field(value, detail_limit)) .filter(|value| !value.is_empty()); std::fs::create_dir_all(diagnostics_dir) diff --git a/apps/native/src-tauri/src/main.rs b/apps/native/src-tauri/src/main.rs index 530553d4d..707ba8292 100644 --- a/apps/native/src-tauri/src/main.rs +++ b/apps/native/src-tauri/src/main.rs @@ -95,6 +95,8 @@ fn e2e_request_webview_boot_probe(window: &WebviewWindow, label: &'static str) { return values; }}; const detail = JSON.stringify({{ + url: document.URL || "", + baseURI: document.baseURI || "", title: document.title || "", readyState: document.readyState || "", bootStage: document.documentElement?.dataset?.nixmacBootStage || "", @@ -102,6 +104,21 @@ fn e2e_request_webview_boot_probe(window: &WebviewWindow, label: &'static str) { capturePaint: document.documentElement?.dataset?.nixmacE2eCapturePaint || "", rootChildren: document.getElementById("root")?.childElementCount ?? null, bodyTextLength: document.body?.innerText?.length ?? null, + htmlLength: document.documentElement?.outerHTML?.length ?? null, + htmlExcerpt: (document.documentElement?.outerHTML || "").slice(0, 1600), + scripts: [...document.scripts].map((script) => ({{ + src: script.src || "", + type: script.type || "", + crossOrigin: script.crossOrigin || "", + defer: script.defer, + async: script.async, + }})), + stylesheets: [...document.querySelectorAll("link[rel~='stylesheet']")].map((link) => ({{ + href: link.href || "", + crossOrigin: link.crossOrigin || "", + media: link.media || "", + }})), + tauriInvokeAvailable: typeof (window.__TAURI__?.core?.invoke || window.__TAURI_INTERNALS__?.invoke) === "function", storage: summarizeStorage(), }}); const invoke = window.__TAURI__?.core?.invoke || window.__TAURI_INTERNALS__?.invoke; @@ -785,6 +802,69 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" return { ok: false, error: String(error) }; } }; + const assetUrls = () => { + const entries = []; + for (const script of [...document.scripts]) { + if (script.src) entries.push({ kind: "script", url: script.src, crossOrigin: script.crossOrigin || "" }); + } + for (const link of [...document.querySelectorAll("link[rel~='stylesheet']")]) { + if (link.href) entries.push({ kind: "stylesheet", url: link.href, crossOrigin: link.crossOrigin || "" }); + } + return entries; + }; + const assetFetchProbe = async (label) => { + const assets = assetUrls(); + const results = []; + for (const asset of assets) { + const startedAt = performance.now(); + const controller = new AbortController(); + const timeout = window.setTimeout(() => controller.abort(), 5000); + try { + const response = await fetch(asset.url, { cache: "no-store", signal: controller.signal }); + const text = await response.clone().text().catch(() => ""); + results.push({ + ...asset, + ok: response.ok, + status: response.status, + statusText: response.statusText, + redirected: response.redirected, + contentType: response.headers.get("content-type") || "", + byteLength: text.length, + elapsedMs: Math.round(performance.now() - startedAt), + }); + } catch (error) { + results.push({ + ...asset, + ok: false, + status: null, + statusText: "", + redirected: false, + contentType: "", + byteLength: 0, + elapsedMs: Math.round(performance.now() - startedAt), + errorName: error?.name || "Error", + errorMessage: String(error?.message || error), + }); + } finally { + window.clearTimeout(timeout); + } + } + const payload = { + label, + url: document.URL || "", + baseURI: document.baseURI || "", + readyState: document.readyState || "", + rootChildren: document.getElementById("root")?.childElementCount ?? null, + bodyTextLength: document.body?.innerText?.length ?? null, + htmlLength: document.documentElement?.outerHTML?.length ?? null, + assets: results, + }; + const serialized = JSON.stringify(payload); + try { + window.localStorage.setItem(`nixmac:e2e-asset-probe:${label}`, serialized.slice(0, 12000)); + } catch {} + logCaptureBreadcrumb(`e2e-asset-fetch-${label}`, serialized); + }; const captureStyleProbe = (label) => { try { const root = document.getElementById("root"); @@ -794,6 +874,7 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" const button = document.querySelector("button"); const svg = document.querySelector("svg"); const textElement = firstTextElement(); + const assets = assetUrls(); const htmlStyle = window.getComputedStyle(document.documentElement); const bodyStyle = document.body ? window.getComputedStyle(document.body) : null; const shellStyle = shell ? window.getComputedStyle(shell) : null; @@ -807,6 +888,8 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" viewport: { width: window.innerWidth, height: window.innerHeight }, rootChildren: root?.childElementCount ?? null, bodyChildren: document.body?.childElementCount ?? null, + assetCount: assets.length, + assetUrls: assets, shellClassName: typeof shell?.className === "string" ? shell.className : null, rootRect: rectFor(root), shellRect: rectFor(shell), @@ -916,6 +999,9 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" }), ); captureStyleProbe("initial-raf"); + assetFetchProbe("initial-raf").catch((error) => { + logCaptureBreadcrumb("e2e-asset-fetch-error", String(error)); + }); requestAnimationFrame(() => { document.documentElement.dataset.nixmacE2eCaptureReady = "1"; logCaptureBreadcrumb( @@ -927,6 +1013,9 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" }), ); captureStyleProbe("capture-ready-2raf"); + assetFetchProbe("capture-ready-2raf").catch((error) => { + logCaptureBreadcrumb("e2e-asset-fetch-error", String(error)); + }); }); }); window.addEventListener( @@ -947,6 +1036,16 @@ const E2E_CAPTURE_DARK_BACKGROUND_SCRIPT: &str = r#" } else { document.addEventListener("DOMContentLoaded", applyCaptureBackground, { once: true }); } + document.addEventListener("DOMContentLoaded", () => { + assetFetchProbe("dom-content-loaded").catch((error) => { + logCaptureBreadcrumb("e2e-asset-fetch-error", String(error)); + }); + }, { once: true }); + window.setTimeout(() => { + assetFetchProbe("post-injection-plus-2s").catch((error) => { + logCaptureBreadcrumb("e2e-asset-fetch-error", String(error)); + }); + }, 2000); })(); "#; diff --git a/tools/computer-use-e2e/peekaboo-runner.mjs b/tools/computer-use-e2e/peekaboo-runner.mjs index 37c33a014..fb05d06ad 100644 --- a/tools/computer-use-e2e/peekaboo-runner.mjs +++ b/tools/computer-use-e2e/peekaboo-runner.mjs @@ -331,7 +331,102 @@ function screenshotRequiresDarkChromeProbe(screenshot) { return /\b(?:launch|app shell|suggestion|descriptor|settings|history|console)\b/.test(text); } -function peekabooScreenshotSignalIssue(runDir, screenshot) { +function parseBreadcrumbDetail(detail) { + if (detail == null) return null; + if (typeof detail === 'object') return detail; + if (typeof detail !== 'string') return null; + try { + return JSON.parse(detail); + } catch { + return null; + } +} + +function readWebviewProof(runDir) { + const breadcrumbPath = path.join(runDir, 'diagnostics', 'nixmac-frontend-breadcrumbs.jsonl'); + if (!existsSync(breadcrumbPath)) { + return { + status: 'missing', + domRendered: false, + captureReady: false, + assetProbeCount: 0, + mountedBreadcrumbs: 0, + note: 'No frontend breadcrumb diagnostics were captured.', + }; + } + + const proof = { + status: 'captured', + domRendered: false, + captureReady: false, + assetProbeCount: 0, + mountedBreadcrumbs: 0, + maxRootChildren: 0, + maxBodyTextLength: 0, + assetFailures: [], + latestDomText: '', + note: '', + }; + const lines = readFileSync(breadcrumbPath, 'utf8').split(/\r?\n/).filter(Boolean); + for (const line of lines) { + let record = null; + try { + record = JSON.parse(line); + } catch { + continue; + } + const label = String(record.label ?? ''); + const detail = parseBreadcrumbDetail(record.detail); + if ( + /app mounted/i.test(label) || + (/native boot stage marker/i.test(label) && /mounted|app-render|app-effect/i.test(String(record.detail ?? ''))) + ) { + proof.mountedBreadcrumbs += 1; + } + if (/E2E DOM snapshot .* text/i.test(label) && typeof record.detail === 'string') { + proof.latestDomText = record.detail.slice(0, 220); + } + if (!detail) continue; + const rootChildren = Number(detail.rootChildren ?? detail.rootChildCount ?? 0); + const bodyTextLength = Number(detail.bodyTextLength ?? 0); + if (Number.isFinite(rootChildren)) proof.maxRootChildren = Math.max(proof.maxRootChildren, rootChildren); + if (Number.isFinite(bodyTextLength)) proof.maxBodyTextLength = Math.max(proof.maxBodyTextLength, bodyTextLength); + if (detail.captureReady || detail.capturePaint) proof.captureReady = true; + if (Array.isArray(detail.assets)) { + proof.assetProbeCount += 1; + for (const asset of detail.assets) { + if (asset?.ok === false) { + proof.assetFailures.push({ + kind: asset.kind ?? '', + url: asset.url ?? '', + status: asset.status ?? null, + errorName: asset.errorName ?? '', + errorMessage: asset.errorMessage ?? '', + }); + } + } + } + } + proof.domRendered = proof.maxRootChildren > 0 && proof.maxBodyTextLength > 0; + proof.note = proof.domRendered + ? `WebView DOM rendered (${proof.maxRootChildren} root child node(s), ${proof.maxBodyTextLength} text chars).` + : 'WebView DOM did not prove rendered app content.'; + return proof; +} + +function writeWebviewProof(runDir, proof) { + const proofPath = path.join(runDir, 'webview-proof.json'); + writeFileSync(proofPath, `${JSON.stringify(proof, null, 2)}\n`, 'utf8'); + return proofPath; +} + +function hostCaptureContext(issue, webviewProof) { + return webviewProof?.domRendered + ? `${issue}; WebView DOM rendered, so host pixel capture is likely black/occluded` + : issue; +} + +function peekabooScreenshotSignalIssue(runDir, screenshot, webviewProof = null) { const baseIssue = imageArtifactIssue({ runDir }, screenshot.path); if (baseIssue) return baseIssue; @@ -352,13 +447,22 @@ function peekabooScreenshotSignalIssue(runDir, screenshot) { const yAvg = cropStats.stats.YAVG; const yRange = Number.isFinite(yMin) && Number.isFinite(yMax) ? yMax - yMin : NaN; if (!Number.isFinite(yAvg) || yAvg < PEEKABOO_SCREENSHOT_CONTENT_PROBE.minYAvg) { - return `central app content is too dark (YAVG ${Number.isFinite(yAvg) ? yAvg : 'unknown'} below ${PEEKABOO_SCREENSHOT_CONTENT_PROBE.minYAvg})`; + return hostCaptureContext( + `central app content is too dark (YAVG ${Number.isFinite(yAvg) ? yAvg : 'unknown'} below ${PEEKABOO_SCREENSHOT_CONTENT_PROBE.minYAvg})`, + webviewProof, + ); } if (!Number.isFinite(yMax) || yMax < PEEKABOO_SCREENSHOT_CONTENT_PROBE.minYMax) { - return `central app content appears blank or occluded (YMAX ${Number.isFinite(yMax) ? yMax : 'unknown'} below ${PEEKABOO_SCREENSHOT_CONTENT_PROBE.minYMax})`; + return hostCaptureContext( + `central app content appears blank or occluded (YMAX ${Number.isFinite(yMax) ? yMax : 'unknown'} below ${PEEKABOO_SCREENSHOT_CONTENT_PROBE.minYMax})`, + webviewProof, + ); } if (!Number.isFinite(yRange) || yRange < PEEKABOO_SCREENSHOT_CONTENT_PROBE.minYRange) { - return `central app content has too little visual contrast (Y range ${Number.isFinite(yRange) ? yRange : 'unknown'} below ${PEEKABOO_SCREENSHOT_CONTENT_PROBE.minYRange})`; + return hostCaptureContext( + `central app content has too little visual contrast (Y range ${Number.isFinite(yRange) ? yRange : 'unknown'} below ${PEEKABOO_SCREENSHOT_CONTENT_PROBE.minYRange})`, + webviewProof, + ); } if ( screenshotRequiresDarkChromeProbe(screenshot) && @@ -370,12 +474,12 @@ function peekabooScreenshotSignalIssue(runDir, screenshot) { return ''; } -function scanPeekabooScreenshotSignal(runDir, screenshots) { +function scanPeekabooScreenshotSignal(runDir, screenshots, webviewProof = null) { const pngScreenshots = screenshots.filter((screenshot) => /\.png$/i.test(screenshot?.path ?? '')); const results = []; for (const screenshot of pngScreenshots) { if (!screenshot?.path || !/\.png$/i.test(screenshot.path)) continue; - const issue = peekabooScreenshotSignalIssue(runDir, screenshot); + const issue = peekabooScreenshotSignalIssue(runDir, screenshot, webviewProof); results.push({ screenshot, issue, @@ -780,7 +884,9 @@ export async function runPeekabooScenario(plan) { const secretScan = scanRunDirForUnmaskedSecrets(path.dirname(plan.logFile)); const secretScanPath = path.join(path.dirname(plan.logFile), 'secret-scan.json'); await writeFile(secretScanPath, `${JSON.stringify(secretScan, null, 2)}\n`, 'utf8'); - const screenshotSignal = scanPeekabooScreenshotSignal(path.dirname(plan.logFile), screenshots); + const webviewProof = readWebviewProof(path.dirname(plan.logFile)); + const webviewProofPath = writeWebviewProof(path.dirname(plan.logFile), webviewProof); + const screenshotSignal = scanPeekabooScreenshotSignal(path.dirname(plan.logFile), screenshots, webviewProof); const screenshotSignalPath = path.join(path.dirname(plan.logFile), 'screenshot-signal.json'); await writeFile(screenshotSignalPath, `${JSON.stringify(screenshotSignal, null, 2)}\n`, 'utf8'); const supplementalDiagnostics = [ @@ -809,6 +915,13 @@ export async function runPeekabooScenario(plan) { note: 'Peekaboo screenshot visual-signal scan.', bytes: statSync(screenshotSignalPath).size, }, + { + label: 'webview-proof.json', + path: path.relative(path.dirname(plan.logFile), webviewProofPath), + capturedAt: new Date().toISOString(), + note: 'WebView DOM, paint, and asset-probe summary used to distinguish app render from host screenshot capture.', + bytes: statSync(webviewProofPath).size, + }, ]; return { @@ -828,6 +941,7 @@ export async function runPeekabooScenario(plan) { infraFailure, secretScan, screenshotSignal, + webviewProof, coverageMap, results, report, @@ -1028,6 +1142,7 @@ export function applyPeekabooResultToState(state, peekabooResult) { if (peekabooResult.coverageMap) state.peekaboo.coverageMap = peekabooResult.coverageMap; if (peekabooResult.secretScan) state.peekaboo.secretScan = peekabooResult.secretScan; if (peekabooResult.screenshotSignal) state.peekaboo.screenshotSignal = peekabooResult.screenshotSignal; + if (peekabooResult.webviewProof) state.peekaboo.webviewProof = peekabooResult.webviewProof; return state; } diff --git a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs index c407837b0..dafc2f0d3 100644 --- a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs +++ b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs @@ -214,6 +214,9 @@ assert.match(nativeMain, /--nixmac-e2e-capture-background: hsl\(var\(--backgroun assert.doesNotMatch(nativeMain, /\.bg-card\\\/50,[\s\S]*\.bg-card\\\/95,[\s\S]*background-color: \$\{captureBackground\}/, 'Native app capture script must not flatten card surfaces to the root capture background'); assert.match(nativeMain, /const captureStyleProbe = \(label\) => \{[\s\S]*window\.getComputedStyle\(shell\)[\s\S]*cardStyle: sampleStyle\(card\)[\s\S]*headerStyle: sampleStyle\(header\)[\s\S]*buttonStyle: sampleStyle\(button\)[\s\S]*svgStyle: sampleStyle\(svg\)[\s\S]*textStyle: sampleStyle\(textElement\)[\s\S]*captureStyleProbe\("app-mounted-plus-2raf"\)/, 'Native app capture script must log post-mount computed foreground, card, border, button, and icon styles that distinguish CSS failures from compositor failures'); assert.match(nativeMain, /const rectFor = \(element\)[\s\S]*document\.elementFromPoint\(x, y\)[\s\S]*const canvasReadbackProbe = \(\)[\s\S]*getImageData\(0, 0, 1, 1\)[\s\S]*devicePixelRatio: window\.devicePixelRatio[\s\S]*textElementAtCenter: elementAtCenter\(textElement\)[\s\S]*canvasReadback: canvasReadbackProbe\(\)/, 'Native app capture script must log layout, hit-test, and canvas readback evidence for black-capture diagnosis'); +assert(nativeMain.includes('const assetFetchProbe = async (label) => {') && nativeMain.includes('document.scripts') && nativeMain.includes("link[rel~='stylesheet']") && nativeMain.includes('const controller = new AbortController()') && nativeMain.includes('await fetch(asset.url, { cache: "no-store", signal: controller.signal })') && nativeMain.includes('e2e-asset-fetch-${label}'), 'Native app capture script must fetch script and stylesheet assets from inside the WebView with a timeout so MacInCloud bundle/CORS hangs are directly diagnosed'); +assert.match(nativeMain, /htmlExcerpt: \(document\.documentElement\?\.outerHTML \|\| ""\)\.slice\(0, 1600\)[\s\S]*scripts: \[\.\.\.document\.scripts\][\s\S]*tauriInvokeAvailable/, 'Native boot probe must include URL, HTML, script/link, and Tauri invoke availability evidence'); +assert.match(debugCommands, /fn e2e_breadcrumb_detail_limit\(label: &str\) -> usize \{[\s\S]*native webview boot probe[\s\S]*e2e-asset-fetch-[\s\S]*12_000[\s\S]*write_e2e_breadcrumb[\s\S]*e2e_breadcrumb_detail_limit\(&label\)/, 'Debug breadcrumb writer must preserve parseable JSON details for native boot and asset probes instead of truncating them at the default short breadcrumb limit'); assert.match(nativeCaptureWindowSetup, /let e2e_solid_capture = e2e_solid_capture_enabled\(\);[\s\S]*let e2e_css_capture = e2e_solid_capture \|\| e2e_opaque_window[\s\S]*transparent\(!e2e_opaque_window\)/, 'Native app must keep default solid capture CSS-backed and transparent while limiting native opacity to opaque debug mode'); assert.match(nativeCaptureWindowSetup, /if e2e_opaque_window \{\n\s+main_window_builder = main_window_builder\s+\.background_color\(tauri::utils::config::Color\(10, 10, 10, 255\)\);[\s\S]*if e2e_css_capture \{\n\s+main_window_builder =\s+main_window_builder\.initialization_script\(E2E_CAPTURE_DARK_BACKGROUND_SCRIPT\);/, 'Native app must keep native dark background only in the opaque debug path while applying CSS capture for solid and opaque modes'); assert.match(nativeMain, /if e2e_css_capture \{[\s\S]*match main_window\.ns_window\(\) \{[\s\S]*set_content_protected\(false\)/, 'Native app must explicitly make E2E capture windows readable from the native diagnostics branch'); @@ -301,6 +304,7 @@ assert.match(productProof, /nixmac_pp_request_native_webview_snapshot\(\)[\s\S]* assert.match(productProof, /nixmac_pp_capture_native_visual_signal\(\)[\s\S]*nixmac_pp_request_native_webview_snapshot[\s\S]*nixmac_pp_screenshot_has_visual_signal/, 'Native WKWebView fallback must run the exact same visual signal probe before it can satisfy readiness'); assert.match(productProof, /nixmac_pp_capture_ready_visual_signal\(\)[\s\S]*peekaboo_run see --app "\$NIXMAC_APP_NAME" --path "\$path"[\s\S]*nixmac_pp_screenshot_has_visual_signal "\$path"[\s\S]*nixmac_pp_capture_native_visual_signal/, 'Ready-shell gate must try system pixels first, then use a fresh native WKWebView snapshot only as a strict fallback'); assert.match(productProof, /nixmac_pp_wait_for_ready_app_shell\(\)[\s\S]*nixmac_pp_capture_ready_visual_signal/, 'ready-shell gate must require screenshot visual signal before launch passes'); +assert.match(peekabooRunner, /function readWebviewProof\(runDir\)[\s\S]*nixmac-frontend-breadcrumbs\.jsonl[\s\S]*assetFailures[\s\S]*webview-proof\.json[\s\S]*host pixel capture is likely black\/occluded/, 'Peekaboo runner must write WebView proof diagnostics and distinguish DOM-rendered failures from host pixel-capture failures without passing them'); assert.match(nixmacAdapter, /nixmac_screenshot\(\)[\s\S]*screenshot "\$label" "\$NIXMAC_APP_NAME"[\s\S]*nixmac_pp_capture_native_visual_signal[\s\S]*webkit-snapshot[\s\S]*system-captures[\s\S]*Promoted native WKWebView snapshot/, 'nixmac screenshots must promote passing WKWebView snapshots into screenshot evidence while retaining black system captures as diagnostics'); assert.match(peekabooRunner, /webkit-snapshot[\s\S]*WKWebView internal snapshot captured from the running nixmac WebContent surface/, 'Peekaboo runner must label WKWebView snapshot screenshot provenance'); assert.match(runLocal, /webkit-snapshot[\s\S]*WKWebView internal snapshot[\s\S]*running WKWebView WebContent surface/, 'HTML report must visibly distinguish WKWebView internal snapshots from Peekaboo screen captures'); From 47badedbf77826c12e7df74d05a43f568b345208 Mon Sep 17 00:00:00 2001 From: fkb032 <249513614+fkb032@users.noreply.github.com> Date: Thu, 7 May 2026 16:41:45 -0700 Subject: [PATCH 08/16] fix: separate peekaboo readiness from visual proof --- .github/workflows/peekaboo-e2e.yml | 10 + apps/native/src-tauri/src/main.rs | 238 +++++++++++++++--- tests/e2e/adapters/nixmac.sh | 5 +- tests/e2e/lib/nixmac_product_proof.sh | 52 +++- tests/e2e/scenarios/macos_console_smoke.sh | 4 +- .../e2e/scenarios/macos_core_product_proof.sh | 2 +- .../macos_descriptor_prompt_smoke.sh | 2 +- .../scenarios/macos_provider_discard_smoke.sh | 2 +- .../macos_provider_evolve_full_smoke.sh | 2 +- .../scenarios/macos_support_dialogs_smoke.sh | 2 +- .../peekaboo-workflow-contract-self-test.mjs | 14 +- 11 files changed, 273 insertions(+), 60 deletions(-) diff --git a/.github/workflows/peekaboo-e2e.yml b/.github/workflows/peekaboo-e2e.yml index 03189c534..76bc65842 100644 --- a/.github/workflows/peekaboo-e2e.yml +++ b/.github/workflows/peekaboo-e2e.yml @@ -209,6 +209,16 @@ jobs: export PATH="$HOME/.cargo/bin:$HOME/.bun/bin:/opt/homebrew/bin:/usr/local/bin:$PATH" mkdir -p "$REPO_DIR/artifacts/computer-use-local" stale_run="$(cat "$REPO_DIR/artifacts/computer-use-local/.current-run" 2>/dev/null || true)" + stale_single_frame_recorders="$( + ps -axo pid=,command= | + awk '/ffmpeg/ && /-frames:v 1/ && !/-framerate/ { print $1 }' + )" + if [[ -n "$stale_single_frame_recorders" ]]; then + echo "Clearing stale one-frame ffmpeg capture process(es): $stale_single_frame_recorders" + kill $stale_single_frame_recorders >/dev/null 2>&1 || true + sleep 1 + kill -9 $stale_single_frame_recorders >/dev/null 2>&1 || true + fi pkill -TERM -f 'tools/computer-use-e2e/run-local\.mjs run-peekaboo-suite|tests/e2e/run\.sh|nixmac-e2e-provider|nixmac\.app/Contents/MacOS/nixmac|ffmpeg .*peekaboo-e2e' >/dev/null 2>&1 || true sleep 2 pkill -KILL -f 'tools/computer-use-e2e/run-local\.mjs run-peekaboo-suite|tests/e2e/run\.sh|nixmac-e2e-provider|nixmac\.app/Contents/MacOS/nixmac|ffmpeg .*peekaboo-e2e' >/dev/null 2>&1 || true diff --git a/apps/native/src-tauri/src/main.rs b/apps/native/src-tauri/src/main.rs index 707ba8292..ee4a0995e 100644 --- a/apps/native/src-tauri/src/main.rs +++ b/apps/native/src-tauri/src/main.rs @@ -344,6 +344,25 @@ fn e2e_write_native_snapshot_status( label: &str, output_path: &std::path::Path, message: Option<&str>, +) { + e2e_write_native_snapshot_status_with_source( + status_path, + status, + label, + output_path, + message, + "WKWebView.takeSnapshotWithConfiguration", + ); +} + +#[cfg(all(debug_assertions, target_os = "macos"))] +fn e2e_write_native_snapshot_status_with_source( + status_path: &std::path::Path, + status: &str, + label: &str, + output_path: &std::path::Path, + message: Option<&str>, + source: &str, ) { if let Some(parent) = status_path.parent() { let _ = std::fs::create_dir_all(parent); @@ -357,7 +376,7 @@ fn e2e_write_native_snapshot_status( .duration_since(std::time::UNIX_EPOCH) .map(|duration| duration.as_millis()) .unwrap_or(0), - "source": "WKWebView.takeSnapshotWithConfiguration" + "source": source }); let tmp_status_path = status_path.with_extension("json.tmp"); if std::fs::write( @@ -371,6 +390,143 @@ fn e2e_write_native_snapshot_status( } } +#[cfg(all(debug_assertions, target_os = "macos"))] +unsafe fn e2e_write_bitmap_rep_png( + bitmap_rep: *mut objc::runtime::Object, + output_path: &std::path::Path, +) -> Result<(), String> { + use objc::class; + use objc::msg_send; + use objc::runtime::Object; + use objc::sel; + use objc::sel_impl; + + if bitmap_rep.is_null() { + return Err("NSBitmapImageRep was nil".to_string()); + } + + let properties: *mut Object = msg_send![class!(NSDictionary), dictionary]; + const NS_BITMAP_IMAGE_FILE_TYPE_PNG: usize = 4; + let png_data: *mut Object = msg_send![ + bitmap_rep, + representationUsingType: NS_BITMAP_IMAGE_FILE_TYPE_PNG + properties: properties + ]; + if png_data.is_null() { + return Err("NSBitmapImageRep PNG representation was nil".to_string()); + } + + let bytes: *const std::ffi::c_void = msg_send![png_data, bytes]; + let len: usize = msg_send![png_data, length]; + if bytes.is_null() || len == 0 { + return Err("PNG data was empty".to_string()); + } + + let slice = std::slice::from_raw_parts(bytes.cast::(), len); + let tmp_output = output_path.with_extension("png.tmp"); + std::fs::write(&tmp_output, slice) + .and_then(|()| std::fs::rename(&tmp_output, output_path)) + .map_err(|error| { + let _ = std::fs::remove_file(&tmp_output); + format!("failed to write PNG atomically: {error}") + }) +} + +#[cfg(all(debug_assertions, target_os = "macos"))] +unsafe fn e2e_try_cached_display_snapshot( + webview: *mut objc::runtime::Object, + status_path: &std::path::Path, + label: &str, + output_path: &std::path::Path, + prior_error: &str, +) -> bool { + use objc::msg_send; + use objc::runtime::Object; + use objc::sel; + use objc::sel_impl; + + #[repr(C)] + #[derive(Clone, Copy)] + struct E2eNativePoint { + x: f64, + y: f64, + } + + #[repr(C)] + #[derive(Clone, Copy)] + struct E2eNativeSize { + width: f64, + height: f64, + } + + #[repr(C)] + #[derive(Clone, Copy)] + struct E2eNativeRect { + origin: E2eNativePoint, + size: E2eNativeSize, + } + + if webview.is_null() { + return false; + } + let bounds: E2eNativeRect = msg_send![webview, bounds]; + if bounds.size.width <= 0.0 || bounds.size.height <= 0.0 { + e2e_write_native_snapshot_status_with_source( + status_path, + "failed", + label, + output_path, + Some(&format!( + "{prior_error}; AppKit fallback had invalid bounds {:.0}x{:.0}", + bounds.size.width, bounds.size.height + )), + "NSView.cacheDisplayInRect", + ); + return false; + } + let bitmap_rep: *mut Object = msg_send![webview, bitmapImageRepForCachingDisplayInRect: bounds]; + if bitmap_rep.is_null() { + e2e_write_native_snapshot_status_with_source( + status_path, + "failed", + label, + output_path, + Some(&format!( + "{prior_error}; AppKit fallback returned nil bitmap rep" + )), + "NSView.cacheDisplayInRect", + ); + return false; + } + let _: () = msg_send![webview, cacheDisplayInRect: bounds toBitmapImageRep: bitmap_rep]; + match e2e_write_bitmap_rep_png(bitmap_rep, output_path) { + Ok(()) => { + e2e_write_native_snapshot_status_with_source( + status_path, + "passed", + label, + output_path, + Some(&format!( + "WKWebView snapshot failed first ({prior_error}); AppKit cached-display fallback wrote PNG" + )), + "NSView.cacheDisplayInRect", + ); + true + } + Err(error) => { + e2e_write_native_snapshot_status_with_source( + status_path, + "failed", + label, + output_path, + Some(&format!("{prior_error}; AppKit fallback failed: {error}")), + "NSView.cacheDisplayInRect", + ); + false + } + } +} + #[cfg(all(debug_assertions, target_os = "macos"))] unsafe fn e2e_nsstring_to_string(value: *mut objc::runtime::Object) -> Option { use objc::{msg_send, sel, sel_impl}; @@ -458,9 +614,23 @@ fn e2e_request_native_webview_snapshot( let label_for_status = label.clone(); let output_for_block = output_path.clone(); let status_for_block = status_path.clone(); + let webview_for_fallback = webview.inner() as *mut Object; let completion = ConcreteBlock::new(move |image: *mut Object, error: *mut Object| { if !error.is_null() { let error_message = e2e_ns_error_summary(error); + if e2e_try_cached_display_snapshot( + webview_for_fallback, + &status_for_block, + &label_for_status, + &output_for_block, + &error_message, + ) { + log::info!( + "NIXMAC_E2E_NATIVE_SNAPSHOT used AppKit cached-display fallback for {}", + label_for_status + ); + return; + } log::warn!( "NIXMAC_E2E_NATIVE_SNAPSHOT failed for {}: {}", label_for_status, @@ -513,40 +683,7 @@ fn e2e_request_native_webview_snapshot( ); return; } - let properties: *mut Object = msg_send![class!(NSDictionary), dictionary]; - const NS_BITMAP_IMAGE_FILE_TYPE_PNG: usize = 4; - let png_data: *mut Object = msg_send![ - bitmap_rep, - representationUsingType: NS_BITMAP_IMAGE_FILE_TYPE_PNG - properties: properties - ]; - if png_data.is_null() { - e2e_write_native_snapshot_status( - &status_for_block, - "failed", - &label_for_status, - &output_for_block, - Some("NSBitmapImageRep PNG representation was nil"), - ); - return; - } - let bytes: *const std::ffi::c_void = msg_send![png_data, bytes]; - let len: usize = msg_send![png_data, length]; - if bytes.is_null() || len == 0 { - e2e_write_native_snapshot_status( - &status_for_block, - "failed", - &label_for_status, - &output_for_block, - Some("PNG data was empty"), - ); - return; - } - let slice = std::slice::from_raw_parts(bytes.cast::(), len); - let tmp_output = output_for_block.with_extension("png.tmp"); - let write_result = std::fs::write(&tmp_output, slice) - .and_then(|()| std::fs::rename(&tmp_output, &output_for_block)); - match write_result { + match e2e_write_bitmap_rep_png(bitmap_rep, &output_for_block) { Ok(()) => { log::info!( "NIXMAC_E2E_NATIVE_SNAPSHOT wrote {} for {}", @@ -562,7 +699,6 @@ fn e2e_request_native_webview_snapshot( ); } Err(error) => { - let _ = std::fs::remove_file(&tmp_output); log::warn!( "NIXMAC_E2E_NATIVE_SNAPSHOT failed to write {} for {}: {}", output_for_block.display(), @@ -574,19 +710,45 @@ fn e2e_request_native_webview_snapshot( "failed", &label_for_status, &output_for_block, - Some(&format!("failed to write PNG atomically: {error}")), + Some(&error), ); } } }) .copy(); let webview = webview.inner() as *mut Object; - let configuration: *mut Object = std::ptr::null_mut(); + let configuration: *mut Object = msg_send![class!(WKSnapshotConfiguration), new]; + if !configuration.is_null() { + #[repr(C)] + struct E2eNativePoint { + x: f64, + y: f64, + } + + #[repr(C)] + struct E2eNativeSize { + width: f64, + height: f64, + } + + #[repr(C)] + struct E2eNativeRect { + origin: E2eNativePoint, + size: E2eNativeSize, + } + + let bounds: E2eNativeRect = msg_send![webview, bounds]; + let _: () = msg_send![configuration, setRect: bounds]; + let _: () = msg_send![configuration, setAfterScreenUpdates: false]; + } let _: () = msg_send![ webview, takeSnapshotWithConfiguration: configuration completionHandler: &*completion ]; + if !configuration.is_null() { + let _: () = msg_send![configuration, release]; + } // The completion is async and owned by WebKit after dispatch. Leaking the // copied block is acceptable in debug-only E2E runs and avoids use-after-free // while the virtualized host is under load. diff --git a/tests/e2e/adapters/nixmac.sh b/tests/e2e/adapters/nixmac.sh index b78347527..0e4376b5f 100644 --- a/tests/e2e/adapters/nixmac.sh +++ b/tests/e2e/adapters/nixmac.sh @@ -171,7 +171,7 @@ nixmac_text() { nixmac_screenshot() { local label="${1:-nixmac}" - local system_path native_path native_dest system_diag_dir system_diag_path + local system_path native_output native_path native_dest system_diag_dir system_diag_path system_path=$(screenshot "$label" "$NIXMAC_APP_NAME" | tail -n 1) if ! declare -f nixmac_pp_capture_native_visual_signal >/dev/null 2>&1; then @@ -179,10 +179,11 @@ nixmac_screenshot() { return 0 fi - native_path=$(nixmac_pp_capture_native_visual_signal "$label") || { + native_output=$(nixmac_pp_capture_native_visual_signal "$label") || { printf '%s\n' "$system_path" return 0 } + native_path=$(printf '%s\n' "$native_output" | tail -n 1) mkdir -p "$E2E_SCREENSHOT_DIR" 2>/dev/null || true native_dest="$E2E_SCREENSHOT_DIR/${label//[^a-zA-Z0-9._-]/_}-webkit-snapshot-$(date +%s)-$$-$RANDOM.png" diff --git a/tests/e2e/lib/nixmac_product_proof.sh b/tests/e2e/lib/nixmac_product_proof.sh index 256e663ad..67fdafffd 100644 --- a/tests/e2e/lib/nixmac_product_proof.sh +++ b/tests/e2e/lib/nixmac_product_proof.sh @@ -434,11 +434,23 @@ nixmac_pp_request_native_webview_snapshot() { nixmac_pp_capture_native_visual_signal() { local label="${1:-ready-shell}" - local path result - - path=$(nixmac_pp_request_native_webview_snapshot "$label" 5) || return 1 + local path result disable_marker disable_marker_dir + + disable_marker="${NIXMAC_E2E_DIAGNOSTICS_DIR:-$E2E_DIAGNOSTIC_DIR}/native-webview-snapshots/.disabled" + [ ! -f "$disable_marker" ] || return 1 + path=$(nixmac_pp_request_native_webview_snapshot "$label" 5) || { + disable_marker_dir="$(dirname "$disable_marker")" + mkdir -p "$disable_marker_dir" + printf 'request-failed label=%s ts=%s\n' "$label" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$disable_marker" + debug "Native WKWebView snapshot disabled for the rest of this scenario after request failure" + return 1 + } result=$(nixmac_pp_screenshot_has_visual_signal "$path" 2>&1) || { debug "Native WKWebView visual signal not established for $path: $result" + disable_marker_dir="$(dirname "$disable_marker")" + mkdir -p "$disable_marker_dir" + printf 'visual-signal-failed label=%s ts=%s path=%s detail=%s\n' "$label" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$path" "$result" >"$disable_marker" + debug "Native WKWebView snapshot promotion disabled for the rest of this scenario after visual-signal failure" return 1 } debug "Native WKWebView snapshot visual signal ready for $path: $result" @@ -447,7 +459,7 @@ nixmac_pp_capture_native_visual_signal() { nixmac_pp_capture_ready_visual_signal() { local label="${1:-ready-shell}" - local dir path result native_path + local dir path result native_output native_path dir="$E2E_DIAGNOSTIC_DIR/visual-readiness" mkdir -p "$dir" @@ -464,13 +476,36 @@ nixmac_pp_capture_ready_visual_signal() { debug "Ready-shell system visual signal not established for $path: $result" fi - native_path=$(nixmac_pp_capture_native_visual_signal "$label") || { + native_output=$(nixmac_pp_capture_native_visual_signal "$label") || { debug "Ready-shell visual signal not established for $path: $result" return 1 } + native_path=$(printf '%s\n' "$native_output" | tail -n 1) debug "Ready-shell visual signal established from native WKWebView snapshot: $native_path" } +nixmac_pp_record_ready_visual_signal() { + local label="${1:-ready-shell}" + local dir result_path result + + dir="$E2E_DIAGNOSTIC_DIR/visual-readiness" + mkdir -p "$dir" + result_path="$dir/${label//[^a-zA-Z0-9._-]/_}-status.txt" + if result=$(nixmac_pp_capture_ready_visual_signal "$label" 2>&1); then + { + printf 'status=passed\n' + printf 'detail=%s\n' "$result" + } >"$result_path" + return 0 + fi + { + printf 'status=failed\n' + printf 'detail=%s\n' "$result" + } >"$result_path" + warn "Ready-shell visual signal unavailable for $label; continuing with driver-visible shell and preserving strict screenshot-signal report gate" + return 1 +} + nixmac_pp_wait_for_ready_app_shell() { local timeout="${1:-45}" local deadline json count @@ -483,10 +518,9 @@ nixmac_pp_wait_for_ready_app_shell() { json=$(E2E_PEEKABOO_SUPPRESS_EMPTY_DIAG=1 peek_elements "$NIXMAC_APP_NAME") || true count=$(peekaboo_element_count "$json") if nixmac_pp_elements_show_ready_shell "$json" "$NIXMAC_PP_READY_SHELL_MIN_ELEMENTS" "$NIXMAC_PP_READY_SHELL_PATTERN"; then - if nixmac_pp_capture_ready_visual_signal "ready-shell"; then - debug "nixmac Product Proof shell ready ($count element(s))" - return 0 - fi + nixmac_pp_record_ready_visual_signal "ready-shell" || true + debug "nixmac Product Proof shell driver-ready ($count element(s))" + return 0 else debug "nixmac Product Proof shell not ready yet ($count element(s))" fi diff --git a/tests/e2e/scenarios/macos_console_smoke.sh b/tests/e2e/scenarios/macos_console_smoke.sh index b1c72fbf8..f4e01587b 100644 --- a/tests/e2e/scenarios/macos_console_smoke.sh +++ b/tests/e2e/scenarios/macos_console_smoke.sh @@ -34,7 +34,7 @@ scenario_test() { phase "Launch nixmac app" nixmac_launch || die "App failed to launch" nixmac_pp_wait_for_ready_app_shell 60 \ - || die "App shell did not expose Console footer with visible screenshot signal" + || die "App shell did not expose Console footer" nixmac_screenshot "01-console-launch" phase_pass "peekabooCoreLaunch: App shell rendered before Console proof" @@ -48,7 +48,7 @@ scenario_test() { die "Console text surface did not render" fi nixmac_screenshot "02-console-expanded" - phase_pass "peekabooCoreConsole: Console rendered text evidence with screenshot proof" + phase_pass "peekabooCoreConsole: Console rendered text evidence" } scenario_cleanup() { diff --git a/tests/e2e/scenarios/macos_core_product_proof.sh b/tests/e2e/scenarios/macos_core_product_proof.sh index f25a275bc..eb09e5d07 100644 --- a/tests/e2e/scenarios/macos_core_product_proof.sh +++ b/tests/e2e/scenarios/macos_core_product_proof.sh @@ -66,7 +66,7 @@ scenario_test() { nixmac_launch || die "App failed to launch" if ! nixmac_pp_wait_for_ready_app_shell 60; then nixmac_screenshot "core-launch-missing" - die "App shell did not expose expected interactive controls with visible screenshot signal" + die "App shell did not expose expected interactive controls" fi nixmac_screenshot "01-core-launch" phase_pass "peekabooCoreLaunch: App shell, header controls, and initial prompt surface rendered" diff --git a/tests/e2e/scenarios/macos_descriptor_prompt_smoke.sh b/tests/e2e/scenarios/macos_descriptor_prompt_smoke.sh index 86378d04e..fcc130826 100644 --- a/tests/e2e/scenarios/macos_descriptor_prompt_smoke.sh +++ b/tests/e2e/scenarios/macos_descriptor_prompt_smoke.sh @@ -29,7 +29,7 @@ scenario_test() { phase "Launch nixmac app" nixmac_launch || die "App failed to launch" nixmac_pp_wait_for_ready_app_shell 60 \ - || die "App shell did not expose descriptor prompt with visible screenshot signal" + || die "App shell did not expose descriptor prompt" nixmac_screenshot "01-launched" phase_pass "App launched" diff --git a/tests/e2e/scenarios/macos_provider_discard_smoke.sh b/tests/e2e/scenarios/macos_provider_discard_smoke.sh index 9464cfe09..d372bac3a 100644 --- a/tests/e2e/scenarios/macos_provider_discard_smoke.sh +++ b/tests/e2e/scenarios/macos_provider_discard_smoke.sh @@ -156,7 +156,7 @@ scenario_test() { phase "Launch nixmac app for discard proof" nixmac_launch || die "App failed to launch" nixmac_pp_wait_for_ready_app_shell 60 \ - || die "App shell did not expose discard prompt with visible screenshot signal" + || die "App shell did not expose discard prompt" nixmac_screenshot "01-discard-launched" phase_pass "peekabooProviderLaunch: App launched for discard proof" diff --git a/tests/e2e/scenarios/macos_provider_evolve_full_smoke.sh b/tests/e2e/scenarios/macos_provider_evolve_full_smoke.sh index f27b5d675..d430fd338 100644 --- a/tests/e2e/scenarios/macos_provider_evolve_full_smoke.sh +++ b/tests/e2e/scenarios/macos_provider_evolve_full_smoke.sh @@ -654,7 +654,7 @@ scenario_test() { phase "Launch nixmac app" nixmac_launch || die "App failed to launch" nixmac_pp_wait_for_ready_app_shell 60 \ - || die "App shell did not expose provider prompt with visible screenshot signal" + || die "App shell did not expose provider prompt" nixmac_screenshot "01-launched" phase_pass "peekabooProviderLaunch: App launched" diff --git a/tests/e2e/scenarios/macos_support_dialogs_smoke.sh b/tests/e2e/scenarios/macos_support_dialogs_smoke.sh index 762e8066c..b1d42275b 100644 --- a/tests/e2e/scenarios/macos_support_dialogs_smoke.sh +++ b/tests/e2e/scenarios/macos_support_dialogs_smoke.sh @@ -42,7 +42,7 @@ scenario_test() { phase "Launch nixmac app" nixmac_launch || die "App failed to launch" nixmac_pp_wait_for_ready_app_shell 60 \ - || die "App shell did not expose support controls with visible screenshot signal" + || die "App shell did not expose support controls" nixmac_screenshot "01-support-launch" phase_pass "peekabooCoreLaunch: App shell rendered before support dialog proof" diff --git a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs index dafc2f0d3..74903f7ef 100644 --- a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs +++ b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs @@ -226,9 +226,9 @@ assert.match(nativeMain, /fn e2e_request_native_webview_capture_probe[\s\S]*with assert.match(nativeMain, /e2e_page_load_native_capture_probe[\s\S]*native-page-load-finished-plus-5s[\s\S]*e2e_schedule_native_webview_capture_probe[\s\S]*native-post-build-plus-2s/, 'Native app must schedule native WebView capture diagnostics after build and after page-load'); assert.match(cargoToml, /\[target\.'cfg\(target_os = "macos"\)'\.dependencies\][\s\S]*block = "0\.1"[\s\S]*objc = "0\.2"/, 'Native WKWebView snapshot completion blocks must declare the small macOS-only block crate beside objc/cocoa'); assert.match(nativeMain, /fn e2e_ns_error_summary[\s\S]*localizedDescription[\s\S]*domain[\s\S]*code[\s\S]*userInfo/, 'Native WKWebView snapshot failures must persist the actual NSError domain, code, description, and userInfo'); -assert.match(nativeMain, /fn e2e_request_native_webview_snapshot[\s\S]*ConcreteBlock::new[\s\S]*NSBitmapImageRep[\s\S]*representationUsingType[\s\S]*takeSnapshotWithConfiguration[\s\S]*std::mem::forget\(completion\)/, 'Native app must provide an E2E-only WKWebView snapshot writer using WebKit snapshot API and explicit async block lifetime handling'); +assert.match(nativeMain, /fn e2e_write_bitmap_rep_png[\s\S]*NSBitmapImageRep[\s\S]*representationUsingType[\s\S]*fn e2e_request_native_webview_snapshot[\s\S]*ConcreteBlock::new[\s\S]*takeSnapshotWithConfiguration[\s\S]*std::mem::forget\(completion\)/, 'Native app must provide an E2E-only WKWebView snapshot writer using WebKit snapshot API and explicit async block lifetime handling'); assert.match(nativeMain, /let tmp_status_path = status_path\.with_extension\("json\.tmp"\)[\s\S]*std::fs::write\([\s\S]*&tmp_status_path[\s\S]*std::fs::rename\(&tmp_status_path, status_path\)/, 'Native WKWebView snapshot status writer must publish JSON atomically so shell polling cannot read partial status'); -assert.match(nativeMain, /let tmp_output = output_for_block\.with_extension\("png\.tmp"\)[\s\S]*std::fs::write\(&tmp_output, slice\)[\s\S]*std::fs::rename\(&tmp_output, &output_for_block\)/, 'Native WKWebView snapshot writer must publish PNGs atomically so shell polling cannot read partial files'); +assert.match(nativeMain, /let tmp_output = output_path\.with_extension\("png\.tmp"\)[\s\S]*std::fs::write\(&tmp_output, slice\)[\s\S]*std::fs::rename\(&tmp_output, output_path\)/, 'Native WKWebView snapshot writer must publish PNGs atomically so shell polling cannot read partial files'); assert.match(nativeMain, /fn e2e_native_snapshot_root_dir[\s\S]*NIXMAC_E2E_DIAGNOSTICS_DIR[\s\S]*native-webview-snapshots/, 'Native snapshots must reuse the existing E2E diagnostics directory'); assert.match(nativeMain, /fn e2e_start_native_webview_snapshot_request_poller[\s\S]*request_dir[\s\S]*requests[\s\S]*\.request[\s\S]*e2e_request_native_webview_snapshot/, 'Native app must poll diagnostics-dir snapshot requests so Scott’s shell driver can demand fresh native visual evidence'); assert.match(nativeMain, /let output_path = root\.join\(format!\("\{request_id\}\.png"\)\);[\s\S]*let status_path = root\.join\(format!\("\{request_id\}\.json"\)\);/, 'Native snapshot poller must use the shell request id verbatim so shell and Rust wait/write paths cannot desync on long labels'); @@ -302,12 +302,18 @@ assert.match(productProof, /nixmac_pp_screenshot_has_visual_signal\(\)[\s\S]*vis assert.match(productProof, /maxDarkChromeYAvg: 42/, 'ready-shell visual gate must enforce the same nixmac dark-capture upper bound as the report scanner'); assert.match(productProof, /nixmac_pp_request_native_webview_snapshot\(\)[\s\S]*NIXMAC_E2E_DIAGNOSTICS_DIR[\s\S]*native-webview-snapshots[\s\S]*\.request[\s\S]*status_path/, 'Product Proof must request fresh native WKWebView snapshots through the existing diagnostics directory'); assert.match(productProof, /nixmac_pp_capture_native_visual_signal\(\)[\s\S]*nixmac_pp_request_native_webview_snapshot[\s\S]*nixmac_pp_screenshot_has_visual_signal/, 'Native WKWebView fallback must run the exact same visual signal probe before it can satisfy readiness'); -assert.match(productProof, /nixmac_pp_capture_ready_visual_signal\(\)[\s\S]*peekaboo_run see --app "\$NIXMAC_APP_NAME" --path "\$path"[\s\S]*nixmac_pp_screenshot_has_visual_signal "\$path"[\s\S]*nixmac_pp_capture_native_visual_signal/, 'Ready-shell gate must try system pixels first, then use a fresh native WKWebView snapshot only as a strict fallback'); -assert.match(productProof, /nixmac_pp_wait_for_ready_app_shell\(\)[\s\S]*nixmac_pp_capture_ready_visual_signal/, 'ready-shell gate must require screenshot visual signal before launch passes'); +assert.match(productProof, /disable_marker="\$\{NIXMAC_E2E_DIAGNOSTICS_DIR:-\$E2E_DIAGNOSTIC_DIR\}\/native-webview-snapshots\/\.disabled"[\s\S]*\[ ! -f "\$disable_marker" \][\s\S]*visual-signal-failed/, 'Native WKWebView snapshot fallback must persist scenario-level disable state outside command-substitution subshells'); +assert.match(nativeMain, /WKSnapshotConfiguration[\s\S]*setRect:[\s\S]*setAfterScreenUpdates: false[\s\S]*takeSnapshotWithConfiguration: configuration/, 'Native snapshot requests must use an explicit WKSnapshotConfiguration instead of relying on the virtualized host visible-rect default'); +assert.match(nativeMain, /e2e_try_cached_display_snapshot[\s\S]*bitmapImageRepForCachingDisplayInRect[\s\S]*cacheDisplayInRect[\s\S]*NSView\.cacheDisplayInRect/, 'Native snapshot requests must fall back to AppKit cached-display capture when WebKit snapshotting fails'); +assert.match(productProof, /nixmac_pp_capture_ready_visual_signal\(\)[\s\S]*peekaboo_run see --app "\$NIXMAC_APP_NAME" --path "\$path"[\s\S]*nixmac_pp_screenshot_has_visual_signal "\$path"[\s\S]*nixmac_pp_capture_native_visual_signal/, 'Ready-shell visual probe must try system pixels first, then use a fresh native WKWebView snapshot only as a strict fallback'); +assert.match(productProof, /nixmac_pp_record_ready_visual_signal\(\)[\s\S]*status=failed[\s\S]*preserving strict screenshot-signal report gate/, 'ready-shell must record visual proof quality without weakening the runner screenshot-signal gate'); +assert.match(productProof, /nixmac_pp_wait_for_ready_app_shell\(\)[\s\S]*nixmac_pp_elements_show_ready_shell[\s\S]*nixmac_pp_record_ready_visual_signal "ready-shell" \|\| true[\s\S]*return 0/, 'ready-shell readiness must succeed on driver-visible AX/product evidence instead of requiring host pixels before launch passes'); +assert.doesNotMatch(productProof, /nixmac_pp_wait_for_ready_app_shell\(\)[\s\S]*if nixmac_pp_capture_ready_visual_signal "ready-shell"; then[\s\S]*return 0/, 'ready-shell readiness must not block scenario execution on host pixel capture'); assert.match(peekabooRunner, /function readWebviewProof\(runDir\)[\s\S]*nixmac-frontend-breadcrumbs\.jsonl[\s\S]*assetFailures[\s\S]*webview-proof\.json[\s\S]*host pixel capture is likely black\/occluded/, 'Peekaboo runner must write WebView proof diagnostics and distinguish DOM-rendered failures from host pixel-capture failures without passing them'); assert.match(nixmacAdapter, /nixmac_screenshot\(\)[\s\S]*screenshot "\$label" "\$NIXMAC_APP_NAME"[\s\S]*nixmac_pp_capture_native_visual_signal[\s\S]*webkit-snapshot[\s\S]*system-captures[\s\S]*Promoted native WKWebView snapshot/, 'nixmac screenshots must promote passing WKWebView snapshots into screenshot evidence while retaining black system captures as diagnostics'); assert.match(peekabooRunner, /webkit-snapshot[\s\S]*WKWebView internal snapshot captured from the running nixmac WebContent surface/, 'Peekaboo runner must label WKWebView snapshot screenshot provenance'); assert.match(runLocal, /webkit-snapshot[\s\S]*WKWebView internal snapshot[\s\S]*running WKWebView WebContent surface/, 'HTML report must visibly distinguish WKWebView internal snapshots from Peekaboo screen captures'); +assert.match(proof, /stale_single_frame_recorders="\$\([\s\S]*ps -axo pid=,command=[\s\S]*-frames:v 1[\s\S]*!\/-framerate\/[\s\S]*Clearing stale one-frame ffmpeg capture process/, 'remote setup must clean stale one-frame ffmpeg captures without matching active scenario recorders'); const setKeys = [...launchEnv.matchAll(/nixmac_pp_set_launch_env ([A-Z0-9_]+)/g)].map((match) => match[1]); const unsetKeys = new Set([...cleanup.matchAll(/nixmac_pp_unset_launch_env ([A-Z0-9_]+)/g)].map((match) => match[1])); From e2293f5047523118481a07b56078a970f2c5d892 Mon Sep 17 00:00:00 2001 From: fkb032 <249513614+fkb032@users.noreply.github.com> Date: Thu, 7 May 2026 17:28:53 -0700 Subject: [PATCH 09/16] fix: mock nix setup checks for peekaboo product proof --- apps/native/src-tauri/src/commands/apply.rs | 85 ++++++++++++++++++- .../peekaboo-workflow-contract-self-test.mjs | 18 ++++ 2 files changed, 99 insertions(+), 4 deletions(-) diff --git a/apps/native/src-tauri/src/commands/apply.rs b/apps/native/src-tauri/src/commands/apply.rs index 09049eda6..3eeccf281 100644 --- a/apps/native/src-tauri/src/commands/apply.rs +++ b/apps/native/src-tauri/src/commands/apply.rs @@ -6,6 +6,14 @@ use crate::{rebuild, shared_types}; use std::process::Command; use tauri::AppHandle; +fn e2e_mock_system_enabled() -> bool { + cfg!(debug_assertions) && crate::e2e_runtime::enabled("NIXMAC_E2E_MOCK_SYSTEM") +} + +fn e2e_mock_host_attr() -> String { + crate::e2e_runtime::value("NIXMAC_E2E_HOST_ATTR").unwrap_or_else(|| "e2e-host".to_string()) +} + /// Starts a streaming darwin-rebuild switch operation. /// Progress is emitted via `darwin:apply:data` events, completion via `darwin:apply:end`. #[tauri::command] @@ -141,6 +149,14 @@ pub async fn finalize_rollback( #[tauri::command] pub async fn flake_installed_apps(app: AppHandle) -> Result, String> { + if e2e_mock_system_enabled() { + log::info!( + "[apply] NIXMAC_E2E_MOCK_SYSTEM enabled; returning empty installed apps fixture" + ); + // No current Product Proof surface depends on installed-app shape; avoid real Nix here. + return Ok(Vec::new()); + } + let dir = store::ensure_config_dir_exists(&app) .map_err(|e| capture_err("flake_installed_apps", e))?; @@ -160,8 +176,18 @@ pub async fn flake_installed_apps(app: AppHandle) -> Result Result { +fn nix_check_result() -> shared_types::NixCheckResult { + if e2e_mock_system_enabled() { + log::info!( + "[apply] NIXMAC_E2E_MOCK_SYSTEM enabled; reporting mocked Nix/darwin-rebuild availability" + ); + return shared_types::NixCheckResult { + installed: true, + version: Some("NIXMAC_E2E_MOCK_SYSTEM mocked nix".to_string()), + darwin_rebuild_available: true, + }; + } + let installed = nix::is_nix_installed(); let version = if installed { nix::get_nix_version() @@ -173,11 +199,16 @@ pub async fn nix_check() -> Result { } else { false }; - Ok(shared_types::NixCheckResult { + shared_types::NixCheckResult { installed, version, darwin_rebuild_available, - }) + } +} + +#[tauri::command] +pub async fn nix_check() -> Result { + Ok(nix_check_result()) } #[tauri::command] @@ -202,8 +233,54 @@ pub async fn finalize_flake_lock(app: AppHandle) -> Result Result, String> { + if e2e_mock_system_enabled() { + let host = e2e_mock_host_attr(); + log::info!( + "[apply] NIXMAC_E2E_MOCK_SYSTEM enabled; returning mocked flake host {}", + host + ); + return Ok(vec![host]); + } + let dir = store::ensure_config_dir_exists(&app).map_err(|e| capture_err("flake_list_hosts", e))?; let hosts = nix::list_darwin_hosts(&dir).map_err(|e| capture_err("flake_list_hosts", e))?; Ok(hosts) } + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(debug_assertions)] + #[test] + fn e2e_mock_nix_check_reports_available_system() { + let _env_lock = crate::test_support::e2e_env_lock(); + let _env_restore = crate::test_support::EnvVarRestore::capture(&["NIXMAC_E2E_MOCK_SYSTEM"]); + + std::env::set_var("NIXMAC_E2E_MOCK_SYSTEM", "1"); + let result = nix_check_result(); + + assert!(result.installed); + assert!(result.darwin_rebuild_available); + assert_eq!( + result.version.as_deref(), + Some("NIXMAC_E2E_MOCK_SYSTEM mocked nix") + ); + } + + #[test] + fn e2e_mock_host_attr_uses_runtime_value_or_default() { + let _env_lock = crate::test_support::e2e_env_lock(); + let _env_restore = + crate::test_support::EnvVarRestore::capture(&["HOME", "NIXMAC_E2E_HOST_ATTR"]); + let temp_home = tempfile::tempdir().unwrap(); + + std::env::set_var("HOME", temp_home.path()); + std::env::remove_var("NIXMAC_E2E_HOST_ATTR"); + assert_eq!(e2e_mock_host_attr(), "e2e-host"); + + std::env::set_var("NIXMAC_E2E_HOST_ATTR", "ci-host"); + assert_eq!(e2e_mock_host_attr(), "ci-host"); + } +} diff --git a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs index 74903f7ef..af3e64600 100644 --- a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs +++ b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs @@ -16,6 +16,7 @@ const peekabooRunnerPath = path.join(repoRoot, 'tools/computer-use-e2e/peekaboo- const permissionsPath = path.join(repoRoot, 'apps/native/src-tauri/src/system/permissions.rs'); const e2eRuntimePath = path.join(repoRoot, 'apps/native/src-tauri/src/e2e_runtime.rs'); const nativeMainPath = path.join(repoRoot, 'apps/native/src-tauri/src/main.rs'); +const applyCommandsPath = path.join(repoRoot, 'apps/native/src-tauri/src/commands/apply.rs'); const debugCommandsPath = path.join(repoRoot, 'apps/native/src-tauri/src/commands/debug.rs'); const nativeStorePath = path.join(repoRoot, 'apps/native/src-tauri/src/storage/store.rs'); const frontendMainPath = path.join(repoRoot, 'apps/native/src/main.tsx'); @@ -35,6 +36,7 @@ const peekabooRunner = readFileSync(peekabooRunnerPath, 'utf8'); const permissions = readFileSync(permissionsPath, 'utf8'); const e2eRuntime = readFileSync(e2eRuntimePath, 'utf8'); const nativeMain = readFileSync(nativeMainPath, 'utf8'); +const applyCommands = readFileSync(applyCommandsPath, 'utf8'); const debugCommands = readFileSync(debugCommandsPath, 'utf8'); const nativeStore = readFileSync(nativeStorePath, 'utf8'); const frontendMain = readFileSync(frontendMainPath, 'utf8'); @@ -72,6 +74,18 @@ const result = section(/^ peekaboo-result:$/m); const launchEnv = section({ sourceText: productProof, pattern: /^nixmac_pp_set_e2e_launch_env\(\) \{$/m }, /^}$/m); const cleanup = section({ sourceText: productProof, pattern: /^nixmac_pp_cleanup_common\(\) \{$/m }, /^}$/m); const frontendRenderApp = section({ sourceText: frontendMain, pattern: /^const renderApp = \(\) => \{$/m }, /^};$/m); +const flakeInstalledAppsCommand = section( + { sourceText: applyCommands, pattern: /^pub async fn flake_installed_apps\(app: AppHandle\)/m }, + /^\}\n\nfn nix_check_result/m, +); +const nixCheckResultCommand = section( + { sourceText: applyCommands, pattern: /^fn nix_check_result\(\) -> shared_types::NixCheckResult \{$/m }, + /^\}\n\n#\[tauri::command\]\npub async fn nix_check/m, +); +const flakeListHostsCommand = section( + { sourceText: applyCommands, pattern: /^pub async fn flake_list_hosts\(app: AppHandle\)/m }, + /^\}\n\n#\[cfg\(test\)\]/m, +); const nativeCaptureWindowSetup = section( { sourceText: nativeMain, pattern: /let e2e_opaque_window = e2e_opaque_window_enabled\(\);/ }, /let main_window = main_window_builder\.build/, @@ -278,6 +292,10 @@ assert.match(runnerShell, /E2E_TERMINAL_CLEANUP_MODE=kill recording_close_termin assert.match(peekabooRunner, /for key in NIXMAC_E2E_MOCK_SYSTEM NIXMAC_E2E_SOLID_CAPTURE NIXMAC_E2E_OPAQUE_WINDOW NIXMAC_E2E_WEBVIEW_WATCHDOG NIXMAC_SKIP_PERMISSIONS/, 'Runner preflight must clear stale Peekaboo launchctl flags, including solid capture, opaque capture, and the independent WebView watchdog'); assert.match(e2eRuntime, /#\[cfg\(debug_assertions\)\][\s\S]*fn file_value[\s\S]*runtime\.schema_version != 1[\s\S]*runtime\.session_id\.trim\(\)\.is_empty\(\)[\s\S]*now_unix\(\)\? > runtime\.expires_at_unix/, 'Rust E2E runtime file reader must be debug-only and reject stale, malformed, or expired runtime files'); assert.match(e2eRuntime, /#\[cfg\(not\(debug_assertions\)\)\][\s\S]*fn file_value\(_name: &str\) -> Option[\s\S]*None/, 'Release builds must ignore E2E runtime files'); +assert.match(applyCommands, /fn e2e_mock_system_enabled\(\) -> bool \{\s*cfg!\(debug_assertions\) && crate::e2e_runtime::enabled\("NIXMAC_E2E_MOCK_SYSTEM"\)\s*\}/, 'apply commands must use the same debug-only NIXMAC_E2E_MOCK_SYSTEM gate as rebuild/scanner/store paths'); +assert.match(nixCheckResultCommand, /NIXMAC_E2E_MOCK_SYSTEM enabled[\s\S]{0,240}?installed: true[\s\S]{0,180}?darwin_rebuild_available: true[\s\S]{0,180}?let installed = nix::is_nix_installed/, 'nix_check must report mocked Nix and darwin-rebuild availability before calling real Nix on MacInCloud Product Proof runs'); +assert.match(flakeListHostsCommand, /if e2e_mock_system_enabled\(\) \{[\s\S]{0,260}?return Ok\(vec!\[host\]\);[\s\S]{0,180}?nix::list_darwin_hosts/, 'flake_list_hosts must use the mocked E2E host before shelling out to real nix'); +assert.match(flakeInstalledAppsCommand, /if e2e_mock_system_enabled\(\) \{[\s\S]{0,320}?return Ok\(Vec::new\(\)\);[\s\S]{0,700}?nix::evaluate_installed_apps/, 'flake_installed_apps must avoid real nix under the E2E mock system gate'); assert.match(permissions, /fn check_desktop_access\(\) -> PermissionStatus \{\n\s+if e2e_skip_permissions_enabled\(\)[\s\S]{0,180}?dirs::home_dir/, 'Desktop permission check must honor E2E skip before touching the Desktop folder'); assert.match(permissions, /fn check_documents_access\(\) -> PermissionStatus \{\n\s+if e2e_skip_permissions_enabled\(\)[\s\S]{0,180}?dirs::home_dir/, 'Documents permission check must honor E2E skip before touching the Documents folder'); assert.match(permissions, /"desktop" => \{\n\s+if e2e_skip_permissions_enabled\(\)[\s\S]{0,520}?let home = dirs::home_dir/, 'Desktop permission request must return before filesystem writes when E2E skip is enabled'); From c35f400c1e0b38b4c05d0ea69bdafc3c72795c07 Mon Sep 17 00:00:00 2001 From: fkb032 <249513614+fkb032@users.noreply.github.com> Date: Thu, 7 May 2026 18:17:15 -0700 Subject: [PATCH 10/16] fix: main-thread peekaboo webview snapshots --- apps/native/src-tauri/src/main.rs | 71 ++++++++++++++++--- apps/native/src/components/widget/widget.tsx | 38 +++++++++- apps/native/src/lib/e2e-boot-diagnostics.ts | 48 +++++++++++++ tests/e2e/lib/nixmac_product_proof.sh | 25 ++++--- .../peekaboo-workflow-contract-self-test.mjs | 23 +++++- 5 files changed, 184 insertions(+), 21 deletions(-) diff --git a/apps/native/src-tauri/src/main.rs b/apps/native/src-tauri/src/main.rs index ee4a0995e..af84b8987 100644 --- a/apps/native/src-tauri/src/main.rs +++ b/apps/native/src-tauri/src/main.rs @@ -277,7 +277,16 @@ fn e2e_schedule_native_webview_capture_probe( ) { std::thread::spawn(move || { std::thread::sleep(delay); - e2e_request_native_webview_capture_probe(&window, label); + let probe_window = window.clone(); + if let Err(error) = window.run_on_main_thread(move || { + e2e_request_native_webview_capture_probe(&probe_window, label); + }) { + log::warn!( + "NIXMAC_E2E_CAPTURE could not schedule native WebView probe {} on main thread: {}", + label, + error + ); + } }); } @@ -503,7 +512,7 @@ unsafe fn e2e_try_cached_display_snapshot( Ok(()) => { e2e_write_native_snapshot_status_with_source( status_path, - "passed", + "degraded", label, output_path, Some(&format!( @@ -739,7 +748,7 @@ fn e2e_request_native_webview_snapshot( let bounds: E2eNativeRect = msg_send![webview, bounds]; let _: () = msg_send![configuration, setRect: bounds]; - let _: () = msg_send![configuration, setAfterScreenUpdates: false]; + let _: () = msg_send![configuration, setAfterScreenUpdates: true]; } let _: () = msg_send![ webview, @@ -783,12 +792,30 @@ fn e2e_schedule_native_webview_snapshot( std::thread::spawn(move || { std::thread::sleep(delay); if let Some((output_path, status_path)) = e2e_native_snapshot_paths(label) { - e2e_request_native_webview_snapshot( - &window, - label.to_string(), - output_path, - status_path, - ); + let snapshot_window = window.clone(); + let output_path_for_error = output_path.clone(); + let status_path_for_error = status_path.clone(); + if let Err(error) = window.run_on_main_thread(move || { + e2e_request_native_webview_snapshot( + &snapshot_window, + label.to_string(), + output_path, + status_path, + ); + }) { + log::warn!( + "NIXMAC_E2E_NATIVE_SNAPSHOT could not schedule {} on main thread: {}", + label, + error + ); + e2e_write_native_snapshot_status( + &status_path_for_error, + "failed", + label, + &output_path_for_error, + Some(&format!("main-thread scheduling failed: {error}")), + ); + } } }); } @@ -836,7 +863,31 @@ fn e2e_start_native_webview_snapshot_request_poller(window: WebviewWindow) { let _ = std::fs::remove_file(&request_path); let output_path = root.join(format!("{request_id}.png")); let status_path = root.join(format!("{request_id}.json")); - e2e_request_native_webview_snapshot(&window, label, output_path, status_path); + let snapshot_window = window.clone(); + let output_path_for_error = output_path.clone(); + let status_path_for_error = status_path.clone(); + let label_for_error = label.clone(); + if let Err(error) = window.run_on_main_thread(move || { + e2e_request_native_webview_snapshot( + &snapshot_window, + label, + output_path, + status_path, + ); + }) { + log::warn!( + "NIXMAC_E2E_NATIVE_SNAPSHOT could not schedule request {} on main thread: {}", + label_for_error, + error + ); + e2e_write_native_snapshot_status( + &status_path_for_error, + "failed", + &label_for_error, + &output_path_for_error, + Some(&format!("main-thread scheduling failed: {error}")), + ); + } } std::thread::sleep(Duration::from_millis(250)); } diff --git a/apps/native/src/components/widget/widget.tsx b/apps/native/src/components/widget/widget.tsx index 6965fc883..6857b2a61 100644 --- a/apps/native/src/components/widget/widget.tsx +++ b/apps/native/src/components/widget/widget.tsx @@ -35,7 +35,13 @@ import { useQueueSummarizer } from "@/hooks/use-queue-summarizer"; import { useWatcher } from "@/hooks/use-watcher"; import { loadConfig, loadHosts, loadEvolveState } from "@/hooks/use-widget-initialization"; import { useSummary } from "@/hooks/use-summary"; -import { markBootStage } from "@/lib/e2e-boot-diagnostics"; +import { + e2eBootDiagnosticsActive, + e2eErrorClass, + markBootStage, + recordE2eStartupState, +} from "@/lib/e2e-boot-diagnostics"; +import { computeCurrentStep } from "@/components/widget/utils"; import { useCurrentStep, useWidgetStore } from "@/stores/widget-store"; import { UpdateBanner } from "@/components/widget/layout/update-banner"; import { setupErrorTestHelpers } from "@/utils/error-test-helpers"; @@ -59,6 +65,25 @@ export function DarwinWidget() { const { queueForSummaries } = useQueueSummarizer(); const { findChangeMap } = useSummary(); + const recordStartupState = (label: string, error?: unknown) => { + if (!e2eBootDiagnosticsActive()) return; + + const state = useWidgetStore.getState(); + recordE2eStartupState(label, { + currentStep: computeCurrentStep(state), + configDirPresent: state.configDir.length > 0, + hostPresent: state.host.length > 0, + hostKnown: state.host.length > 0 && state.hosts.includes(state.host), + hostsCount: state.hosts.length, + nixInstalled: state.nixInstalled, + darwinRebuildAvailable: state.darwinRebuildAvailable, + permissionsChecked: state.permissionsChecked, + permissionsComplete: Boolean(state.permissionsState?.allRequiredGranted), + prefsLoaded: state.prefsLoaded, + errorClass: e2eErrorClass(error) ?? (state.error ? "StoreError" : null), + }); + }; + // Set up panic handler to catch Rust crashes and show feedback dialog usePanicHandler(); @@ -108,18 +133,29 @@ export function DarwinWidget() { useEffect(() => { (async () => { try { + recordStartupState("init-start"); await checkPermissions(); + recordStartupState("after-check-permissions"); await loadConfig(); + recordStartupState("after-load-config"); await checkNix(); + recordStartupState("after-check-nix"); await loadHosts(); + recordStartupState("after-load-hosts"); await loadEvolveState(); + recordStartupState("after-load-evolve-state"); await getInitialStatus(); + recordStartupState("after-git-status"); await loadPrefs(); + recordStartupState("after-load-prefs"); await findChangeMap(); + recordStartupState("after-find-change-map"); refreshPromptHistory(); } catch (e: unknown) { useWidgetStore.getState().setError((e as Error)?.message || String(e)); + recordStartupState("startup-error", e); } + recordStartupState("startup-complete"); // Start watching for git changes and summarizer updates after initial load startWatching(); diff --git a/apps/native/src/lib/e2e-boot-diagnostics.ts b/apps/native/src/lib/e2e-boot-diagnostics.ts index b35aacb39..f778d74a1 100644 --- a/apps/native/src/lib/e2e-boot-diagnostics.ts +++ b/apps/native/src/lib/e2e-boot-diagnostics.ts @@ -17,6 +17,10 @@ const NIX_SECRET_ASSIGNMENT_PATTERN = const e2eBootDiagnosticsEnabled = import.meta.env.VITE_NIXMAC_SKIP_PERMISSIONS === "true"; let bootStageCleared = false; +export function e2eBootDiagnosticsActive() { + return e2eBootDiagnosticsEnabled; +} + function setStorageValue(key: string, value: string) { try { window.localStorage.setItem(key, value); @@ -38,6 +42,14 @@ function simpleHash(value: string) { return (hash >>> 0).toString(16).padStart(8, "0"); } +export function e2eErrorClass(error: unknown): string | null { + if (error == null) return null; + if (error instanceof Error) { + return error.constructor?.name || error.name || "Error"; + } + return typeof error; +} + function sanitizeUrl(value: string): string { if (!(value.startsWith("http://") || value.startsWith("https://"))) { return value; @@ -121,6 +133,42 @@ export function bootBreadcrumb(label: string, detail?: unknown) { void darwinAPI.debug.logBreadcrumb(label, summarized, clientTimestampUnixMs).catch(() => {}); } +type StartupStateDiagnostic = { + currentStep: string; + configDirPresent: boolean; + hostPresent: boolean; + hostKnown: boolean; + hostsCount: number; + nixInstalled: boolean | null; + darwinRebuildAvailable: boolean | null; + permissionsChecked: boolean; + permissionsComplete: boolean; + prefsLoaded: boolean; + errorClass: string | null; +}; + +export function recordE2eStartupState(label: string, state: StartupStateDiagnostic) { + if (!e2eBootDiagnosticsEnabled) return; + + const bodyText = sanitizeE2eDiagnosticText(document.body?.innerText ?? ""); + bootBreadcrumb(`E2E startup state ${label}`, { + label, + currentStep: state.currentStep, + configDirPresent: state.configDirPresent, + hostPresent: state.hostPresent, + hostKnown: state.hostKnown, + hostsCount: state.hostsCount, + nixInstalled: state.nixInstalled, + darwinRebuildAvailable: state.darwinRebuildAvailable, + permissionsChecked: state.permissionsChecked, + permissionsComplete: state.permissionsComplete, + prefsLoaded: state.prefsLoaded, + errorClass: state.errorClass, + bodyTextLength: bodyText.length, + bodyTextHash: simpleHash(bodyText), + }); +} + type DomSnapshotOptions = { storagePrefix?: string; }; diff --git a/tests/e2e/lib/nixmac_product_proof.sh b/tests/e2e/lib/nixmac_product_proof.sh index 67fdafffd..92cf86071 100644 --- a/tests/e2e/lib/nixmac_product_proof.sh +++ b/tests/e2e/lib/nixmac_product_proof.sh @@ -414,16 +414,25 @@ nixmac_pp_request_native_webview_snapshot() { printf '%s\n' "$label" > "$request_path" || return 1 deadline=$(($(date +%s) + timeout)) while [ "$(date +%s)" -le "$deadline" ]; do - if [ -s "$output_path" ]; then - printf '%s\n' "$output_path" - return 0 - fi if [ -s "$status_path" ]; then status=$(jq -r '.status // ""' "$status_path" 2>/dev/null || true) - if [ "$status" = "failed" ]; then - debug "Native WKWebView snapshot failed for $label: $(jq -r '.message // "unknown"' "$status_path" 2>/dev/null || echo unknown)" - return 1 - fi + case "$status" in + passed) + if [ -s "$output_path" ]; then + printf '%s\n' "$output_path" + return 0 + fi + debug "Native WKWebView snapshot status passed for $label but PNG is not ready yet" + ;; + degraded) + debug "Native WKWebView snapshot degraded for $label: $(jq -r '.message // "unknown"' "$status_path" 2>/dev/null || echo unknown)" + return 1 + ;; + failed) + debug "Native WKWebView snapshot failed for $label: $(jq -r '.message // "unknown"' "$status_path" 2>/dev/null || echo unknown)" + return 1 + ;; + esac fi sleep 0.2 done diff --git a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs index af3e64600..eb43e660a 100644 --- a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs +++ b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs @@ -74,6 +74,18 @@ const result = section(/^ peekaboo-result:$/m); const launchEnv = section({ sourceText: productProof, pattern: /^nixmac_pp_set_e2e_launch_env\(\) \{$/m }, /^}$/m); const cleanup = section({ sourceText: productProof, pattern: /^nixmac_pp_cleanup_common\(\) \{$/m }, /^}$/m); const frontendRenderApp = section({ sourceText: frontendMain, pattern: /^const renderApp = \(\) => \{$/m }, /^};$/m); +const nativeWebviewCaptureProbeScheduler = section( + { sourceText: nativeMain, pattern: /^fn e2e_schedule_native_webview_capture_probe\(/m }, + /^}\n\n#\[cfg/m, +); +const nativeWebviewSnapshotScheduler = section( + { sourceText: nativeMain, pattern: /^fn e2e_schedule_native_webview_snapshot\(/m }, + /^}\n\n#\[cfg/m, +); +const nativeWebviewSnapshotPoller = section( + { sourceText: nativeMain, pattern: /^fn e2e_start_native_webview_snapshot_request_poller\(/m }, + /^}\n\n#\[cfg/m, +); const flakeInstalledAppsCommand = section( { sourceText: applyCommands, pattern: /^pub async fn flake_installed_apps\(app: AppHandle\)/m }, /^\}\n\nfn nix_check_result/m, @@ -268,11 +280,14 @@ assert.match(frontendBootDiagnostics, /export function clearBootStage[\s\S]*mark assert.match(frontendBootDiagnostics, /export function sanitizeE2eDiagnosticText[\s\S]*EMAIL_PATTERN[\s\S]*BEARER_TOKEN_PATTERN[\s\S]*OPENAI_TOKEN_PATTERN[\s\S]*HOME_DIR_PATH_PATTERN/, 'E2E DOM diagnostics must sanitize secret-shaped text before persisting report artifacts'); assert.match(frontendBootDiagnostics, /export function recordE2eDomSnapshot[\s\S]*storagePrefix[\s\S]*nixmac:e2e-dom-snapshot[\s\S]*document\.documentElement\.dataset\.nixmacE2eDomSnapshot[\s\S]*`\$\{storagePrefix\}:last`[\s\S]*E2E DOM snapshot \$\{label\} summary[\s\S]*E2E DOM snapshot \$\{label\} text[\s\S]*E2E DOM snapshot \$\{label\} html/, 'E2E DOM diagnostics must persist bounded snapshots through both out-of-band DOM/localStorage state and breadcrumb artifacts'); assert.match(frontendBootDiagnostics, /export function scheduleE2eDomSnapshots[\s\S]*count = 5[\s\S]*intervalMs = 2_000[\s\S]*emitted < count/, 'E2E DOM diagnostics must schedule a bounded post-mount snapshot series and self-stop'); +assert.match(frontendBootDiagnostics, /export function recordE2eStartupState[\s\S]*bootBreadcrumb\(`E2E startup state[\s\S]*configDirPresent[\s\S]*hostPresent[\s\S]*hostKnown[\s\S]*hostsCount[\s\S]*errorClass[\s\S]*bodyTextHash/, 'E2E startup diagnostics must persist only allowlisted routing booleans/counts/enums plus text fingerprint'); +assert.match(frontendBootDiagnostics, /export function e2eErrorClass\(error: unknown\)[\s\S]*error instanceof Error[\s\S]*error\.constructor\?\.name[\s\S]*return typeof error/, 'E2E startup diagnostics must classify errors by type/constructor instead of persisting error messages'); assert.match(frontendMain, /PREFS_BOOT_TIMEOUT_MS = 8000[\s\S]*ui_get_prefs invoke start[\s\S]*success after timeout[\s\S]*Promise\.race\(\[prefsPromise, timeoutPromise\]\)/, 'Frontend boot must log prefs IPC progress with clear after-timeout labels'); assert.match(frontendMain, /markBootStage\("main-loaded"\)[\s\S]*markBootStage\("root-found"\)[\s\S]*markBootStage\("react-render-start"\)[\s\S]*markBootStage\("react-render-scheduled"\)/, 'Frontend boot must synchronously mark module, root, and render-scheduling stages'); assert.match(frontendMain, /import \{ flushSync \} from "react-dom";[\s\S]*if \(E2E_BOOT_PREFS_DISABLED\) \{[\s\S]*React render flushSync start[\s\S]*flushSync\(\(\) => \{[\s\S]*root\.render\(app\);[\s\S]*React render flushSync complete/, 'Frontend must force the initial React render synchronously in E2E mode so MacInCloud cannot stall at react-render-scheduled'); assert.match(frontendApp, /markBootStage\("app-render"\)[\s\S]*markBootStage\("app-effect"\)[\s\S]*clearBootStage\(\)/, 'App must synchronously mark render/effect stages and clear the E2E title marker after mount'); assert.match(frontendWidget, /markBootStage\("darwin-widget-render"\)/, 'DarwinWidget must mark when the product widget render body is reached'); +assert.match(frontendWidget, /recordStartupState\("init-start"\)[\s\S]*recordStartupState\("after-load-config"\)[\s\S]*recordStartupState\("after-load-hosts"\)[\s\S]*recordStartupState\("startup-error", e\)[\s\S]*recordStartupState\("startup-complete"\)/, 'DarwinWidget startup must breadcrumb async initialization state through success and failure without raw config values'); assert.match(frontendEditorPanel, /const LazyNixEditor = lazy\(async \(\) => \{[\s\S]*import\("@\/components\/kibo-ui\/nix-editor"\)[\s\S]*default: module\.NixEditor/, 'EditorPanel must lazy-load the Monaco-backed Nix editor only when a file is opened'); assert.doesNotMatch(frontendEditorPanel, /import \{ NixEditor \}/, 'EditorPanel must not import the Monaco-backed editor in the first app boot bundle'); assert.match(frontendMain, /if \(E2E_BOOT_PREFS_DISABLED\) \{[\s\S]*setInterval\(\(\) => \{[\s\S]*boot heartbeat[\s\S]*boot heartbeat upper bound reached[\s\S]*stopBootHeartbeat[\s\S]*boot heartbeat stopped[\s\S]*nixmac:app-mounted/, 'Frontend boot must emit bounded E2E-only heartbeat breadcrumbs until App mounted and record when the bound is reached'); @@ -321,8 +336,12 @@ assert.match(productProof, /maxDarkChromeYAvg: 42/, 'ready-shell visual gate mus assert.match(productProof, /nixmac_pp_request_native_webview_snapshot\(\)[\s\S]*NIXMAC_E2E_DIAGNOSTICS_DIR[\s\S]*native-webview-snapshots[\s\S]*\.request[\s\S]*status_path/, 'Product Proof must request fresh native WKWebView snapshots through the existing diagnostics directory'); assert.match(productProof, /nixmac_pp_capture_native_visual_signal\(\)[\s\S]*nixmac_pp_request_native_webview_snapshot[\s\S]*nixmac_pp_screenshot_has_visual_signal/, 'Native WKWebView fallback must run the exact same visual signal probe before it can satisfy readiness'); assert.match(productProof, /disable_marker="\$\{NIXMAC_E2E_DIAGNOSTICS_DIR:-\$E2E_DIAGNOSTIC_DIR\}\/native-webview-snapshots\/\.disabled"[\s\S]*\[ ! -f "\$disable_marker" \][\s\S]*visual-signal-failed/, 'Native WKWebView snapshot fallback must persist scenario-level disable state outside command-substitution subshells'); -assert.match(nativeMain, /WKSnapshotConfiguration[\s\S]*setRect:[\s\S]*setAfterScreenUpdates: false[\s\S]*takeSnapshotWithConfiguration: configuration/, 'Native snapshot requests must use an explicit WKSnapshotConfiguration instead of relying on the virtualized host visible-rect default'); -assert.match(nativeMain, /e2e_try_cached_display_snapshot[\s\S]*bitmapImageRepForCachingDisplayInRect[\s\S]*cacheDisplayInRect[\s\S]*NSView\.cacheDisplayInRect/, 'Native snapshot requests must fall back to AppKit cached-display capture when WebKit snapshotting fails'); +assert.match(nativeWebviewCaptureProbeScheduler, /run_on_main_thread[\s\S]*e2e_request_native_webview_capture_probe/, 'Native WebView capture probes must touch AppKit/WebKit from the main thread'); +assert.match(nativeWebviewSnapshotScheduler, /run_on_main_thread[\s\S]*e2e_request_native_webview_snapshot/, 'Scheduled native WebView snapshots must touch WKWebView from the main thread'); +assert.match(nativeWebviewSnapshotPoller, /run_on_main_thread[\s\S]*e2e_request_native_webview_snapshot/, 'On-demand native WebView snapshot requests must touch WKWebView from the main thread'); +assert.match(nativeMain, /WKSnapshotConfiguration[\s\S]*setRect:[\s\S]*setAfterScreenUpdates: true[\s\S]*takeSnapshotWithConfiguration: configuration/, 'Native snapshot requests must wait for painted WKWebView content instead of capturing a pre-update virtualized host surface'); +assert.match(nativeMain, /e2e_try_cached_display_snapshot[\s\S]*bitmapImageRepForCachingDisplayInRect[\s\S]*cacheDisplayInRect[\s\S]*"degraded"[\s\S]*NSView\.cacheDisplayInRect/, 'Native snapshot requests must retain AppKit cached-display fallback as degraded diagnostics, not passing visual proof'); +assert.match(productProof, /status=\$\(jq -r '\.status \/\/ ""' "\$status_path"[\s\S]*passed\)[\s\S]*\[ -s "\$output_path" \][\s\S]*degraded\)[\s\S]*return 1[\s\S]*failed\)[\s\S]*return 1/, 'Product Proof native snapshot reader must wait for terminal status and accept only status=passed before returning a PNG path'); assert.match(productProof, /nixmac_pp_capture_ready_visual_signal\(\)[\s\S]*peekaboo_run see --app "\$NIXMAC_APP_NAME" --path "\$path"[\s\S]*nixmac_pp_screenshot_has_visual_signal "\$path"[\s\S]*nixmac_pp_capture_native_visual_signal/, 'Ready-shell visual probe must try system pixels first, then use a fresh native WKWebView snapshot only as a strict fallback'); assert.match(productProof, /nixmac_pp_record_ready_visual_signal\(\)[\s\S]*status=failed[\s\S]*preserving strict screenshot-signal report gate/, 'ready-shell must record visual proof quality without weakening the runner screenshot-signal gate'); assert.match(productProof, /nixmac_pp_wait_for_ready_app_shell\(\)[\s\S]*nixmac_pp_elements_show_ready_shell[\s\S]*nixmac_pp_record_ready_visual_signal "ready-shell" \|\| true[\s\S]*return 0/, 'ready-shell readiness must succeed on driver-visible AX/product evidence instead of requiring host pixels before launch passes'); From 6f42d76563aecd5c6cecd96c68706d7c720f626b Mon Sep 17 00:00:00 2001 From: fkb032 <249513614+fkb032@users.noreply.github.com> Date: Thu, 7 May 2026 23:10:03 -0700 Subject: [PATCH 11/16] fix: keep e2e logs off stdout --- tests/e2e/lib/core.sh | 27 +++++++++++--------- tests/e2e/lib/peekaboo.test.sh | 45 ++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/tests/e2e/lib/core.sh b/tests/e2e/lib/core.sh index c17ac4ae7..9043ead69 100644 --- a/tests/e2e/lib/core.sh +++ b/tests/e2e/lib/core.sh @@ -89,28 +89,33 @@ _NC='\033[0m' # --- Logging --- +_e2e_log_stderr() { + local line="$1" + printf '%b\n' "$line" | tee -a "$E2E_LOG_FILE" >&2 +} + log() { - echo -e "${_BLUE}[$(date +%H:%M:%S)]${_NC} $*" | tee -a "$E2E_LOG_FILE" + _e2e_log_stderr "${_BLUE}[$(date +%H:%M:%S)]${_NC} $*" } debug() { if [ "${E2E_VERBOSE:-0}" = "1" ]; then - echo -e "${_DIM}[$(date +%H:%M:%S)] [debug] $*${_NC}" | tee -a "$E2E_LOG_FILE" + _e2e_log_stderr "${_DIM}[$(date +%H:%M:%S)] [debug] $*${_NC}" fi } warn() { - echo -e "${_YELLOW}[WARN]${_NC} $*" | tee -a "$E2E_LOG_FILE" + _e2e_log_stderr "${_YELLOW}[WARN]${_NC} $*" } pass() { _E2E_PASS_COUNT=$((_E2E_PASS_COUNT + 1)) - echo -e "${_GREEN}[PASS]${_NC} $*" | tee -a "$E2E_LOG_FILE" + _e2e_log_stderr "${_GREEN}[PASS]${_NC} $*" } fail() { _E2E_FAIL_COUNT=$((_E2E_FAIL_COUNT + 1)) - echo -e "${_RED}[FAIL]${_NC} $*" | tee -a "$E2E_LOG_FILE" + _e2e_log_stderr "${_RED}[FAIL]${_NC} $*" } die() { @@ -239,7 +244,7 @@ assert_command() { # --- Results --- print_results() { - echo "" + _e2e_log_stderr "" log "==========================================" log " Test Results" log "==========================================" @@ -251,17 +256,17 @@ print_results() { local num=$(echo "$result" | cut -d'|' -f2) local msg=$(echo "$result" | cut -d'|' -f3-) if [ "$status" = "PASS" ]; then - echo -e " ${_GREEN}✅${_NC} Phase $num: $msg" | tee -a "$E2E_LOG_FILE" + _e2e_log_stderr " ${_GREEN}✅${_NC} Phase $num: $msg" else - echo -e " ${_RED}❌${_NC} Phase $num: $msg" | tee -a "$E2E_LOG_FILE" + _e2e_log_stderr " ${_RED}❌${_NC} Phase $num: $msg" fi done - echo "" | tee -a "$E2E_LOG_FILE" + _e2e_log_stderr "" if [ "$_E2E_FAIL_COUNT" -eq 0 ]; then - echo -e " ${_GREEN}All $total checks passed${_NC}" | tee -a "$E2E_LOG_FILE" + _e2e_log_stderr " ${_GREEN}All $total checks passed${_NC}" else - echo -e " ${_RED}$_E2E_FAIL_COUNT/$total checks failed${_NC}" | tee -a "$E2E_LOG_FILE" + _e2e_log_stderr " ${_RED}$_E2E_FAIL_COUNT/$total checks failed${_NC}" fi log "==========================================" } diff --git a/tests/e2e/lib/peekaboo.test.sh b/tests/e2e/lib/peekaboo.test.sh index b0144aa3b..657f517b9 100644 --- a/tests/e2e/lib/peekaboo.test.sh +++ b/tests/e2e/lib/peekaboo.test.sh @@ -20,6 +20,31 @@ source "$E2E_LIB/core.sh" source "$E2E_LIB/peekaboo.sh" source "$E2E_LIB/nixmac_product_proof.sh" +assert_empty_stdout() { + local label="$1" + local output="$2" + if [ -n "$output" ]; then + echo "expected $label to write terminal output to stderr only, got stdout: $output" >&2 + exit 1 + fi +} + +export E2E_VERBOSE=1 +assert_empty_stdout "log" "$(log "stdout isolation log probe" 2>"$TEST_DIR/log.stderr")" +assert_empty_stdout "debug" "$(debug "stdout isolation debug probe" 2>"$TEST_DIR/debug.stderr")" +assert_empty_stdout "warn" "$(warn "stdout isolation warn probe" 2>"$TEST_DIR/warn.stderr")" +assert_empty_stdout "pass" "$(pass "stdout isolation pass probe" 2>"$TEST_DIR/pass.stderr")" +assert_empty_stdout "fail" "$(fail "stdout isolation fail probe" 2>"$TEST_DIR/fail.stderr")" +for expected in "stdout isolation log probe" "stdout isolation debug probe" "stdout isolation warn probe" "stdout isolation pass probe" "stdout isolation fail probe"; do + grep -q "$expected" "$E2E_LOG_FILE" || { + echo "expected log file to contain: $expected" >&2 + exit 1 + } +done +_E2E_PASS_COUNT=0 +_E2E_FAIL_COUNT=0 +export E2E_VERBOSE=0 + peekaboo_run() { case "$*" in "app switch --to nixmac") @@ -117,4 +142,24 @@ nixmac_pp_elements_show_ready_shell "$ready_shell_json" 20 || { exit 1 } +native_snapshot_path="$TEST_DIR/native-proof.png" +printf 'png' > "$native_snapshot_path" +nixmac_pp_request_native_webview_snapshot() { + printf '%s\n' "$native_snapshot_path" +} +nixmac_pp_screenshot_has_visual_signal() { + printf 'visual signal probe passed\n' +} +export E2E_VERBOSE=1 +native_stdout="$(nixmac_pp_capture_native_visual_signal "stdout-isolation" 2>"$TEST_DIR/native-stderr.log")" +if [ "$native_stdout" != "$native_snapshot_path" ]; then + echo "expected native visual signal stdout to contain only the snapshot path, got: $native_stdout" >&2 + exit 1 +fi +grep -q "Native WKWebView snapshot visual signal ready" "$TEST_DIR/native-stderr.log" || { + echo "expected native visual signal debug detail on stderr" >&2 + exit 1 +} +export E2E_VERBOSE=0 + echo "Peekaboo shell fallback self-test passed." From 6271388de9a97e83e70846ee8faaa1b528a123ea Mon Sep 17 00:00:00 2001 From: fkb032 <249513614+fkb032@users.noreply.github.com> Date: Thu, 7 May 2026 23:57:06 -0700 Subject: [PATCH 12/16] fix: use full-view webkit snapshots for peekaboo --- apps/native/src-tauri/src/main.rs | 33 +++---------------- .../peekaboo-workflow-contract-self-test.mjs | 2 +- 2 files changed, 6 insertions(+), 29 deletions(-) diff --git a/apps/native/src-tauri/src/main.rs b/apps/native/src-tauri/src/main.rs index af84b8987..17cd7e50b 100644 --- a/apps/native/src-tauri/src/main.rs +++ b/apps/native/src-tauri/src/main.rs @@ -726,44 +726,21 @@ fn e2e_request_native_webview_snapshot( }) .copy(); let webview = webview.inner() as *mut Object; - let configuration: *mut Object = msg_send![class!(WKSnapshotConfiguration), new]; - if !configuration.is_null() { - #[repr(C)] - struct E2eNativePoint { - x: f64, - y: f64, - } - - #[repr(C)] - struct E2eNativeSize { - width: f64, - height: f64, - } - - #[repr(C)] - struct E2eNativeRect { - origin: E2eNativePoint, - size: E2eNativeSize, - } - - let bounds: E2eNativeRect = msg_send![webview, bounds]; - let _: () = msg_send![configuration, setRect: bounds]; - let _: () = msg_send![configuration, setAfterScreenUpdates: true]; - } + // Use WebKit's full-view snapshot path instead of passing a rect from + // AppKit bounds. On virtualized hosts the bounds can be valid while the + // configured snapshot still fails with WKErrorDomain code=1. + let configuration: *mut Object = std::ptr::null_mut(); let _: () = msg_send![ webview, takeSnapshotWithConfiguration: configuration completionHandler: &*completion ]; - if !configuration.is_null() { - let _: () = msg_send![configuration, release]; - } // The completion is async and owned by WebKit after dispatch. Leaking the // copied block is acceptable in debug-only E2E runs and avoids use-after-free // while the virtualized host is under load. std::mem::forget(completion); log::debug!( - "NIXMAC_E2E_NATIVE_SNAPSHOT requested {} for {}", + "NIXMAC_E2E_NATIVE_SNAPSHOT requested {} for {} using nil WKSnapshotConfiguration", output_path.display(), label_for_log ); diff --git a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs index eb43e660a..6c0b4e50d 100644 --- a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs +++ b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs @@ -339,7 +339,7 @@ assert.match(productProof, /disable_marker="\$\{NIXMAC_E2E_DIAGNOSTICS_DIR:-\$E2 assert.match(nativeWebviewCaptureProbeScheduler, /run_on_main_thread[\s\S]*e2e_request_native_webview_capture_probe/, 'Native WebView capture probes must touch AppKit/WebKit from the main thread'); assert.match(nativeWebviewSnapshotScheduler, /run_on_main_thread[\s\S]*e2e_request_native_webview_snapshot/, 'Scheduled native WebView snapshots must touch WKWebView from the main thread'); assert.match(nativeWebviewSnapshotPoller, /run_on_main_thread[\s\S]*e2e_request_native_webview_snapshot/, 'On-demand native WebView snapshot requests must touch WKWebView from the main thread'); -assert.match(nativeMain, /WKSnapshotConfiguration[\s\S]*setRect:[\s\S]*setAfterScreenUpdates: true[\s\S]*takeSnapshotWithConfiguration: configuration/, 'Native snapshot requests must wait for painted WKWebView content instead of capturing a pre-update virtualized host surface'); +assert.match(nativeMain, /let configuration: \*mut Object = std::ptr::null_mut\(\);[\s\S]*takeSnapshotWithConfiguration: configuration/, 'Native snapshot requests must use WebKit full-view snapshots so virtualized hosts do not fail on stale AppKit rect configuration'); assert.match(nativeMain, /e2e_try_cached_display_snapshot[\s\S]*bitmapImageRepForCachingDisplayInRect[\s\S]*cacheDisplayInRect[\s\S]*"degraded"[\s\S]*NSView\.cacheDisplayInRect/, 'Native snapshot requests must retain AppKit cached-display fallback as degraded diagnostics, not passing visual proof'); assert.match(productProof, /status=\$\(jq -r '\.status \/\/ ""' "\$status_path"[\s\S]*passed\)[\s\S]*\[ -s "\$output_path" \][\s\S]*degraded\)[\s\S]*return 1[\s\S]*failed\)[\s\S]*return 1/, 'Product Proof native snapshot reader must wait for terminal status and accept only status=passed before returning a PNG path'); assert.match(productProof, /nixmac_pp_capture_ready_visual_signal\(\)[\s\S]*peekaboo_run see --app "\$NIXMAC_APP_NAME" --path "\$path"[\s\S]*nixmac_pp_screenshot_has_visual_signal "\$path"[\s\S]*nixmac_pp_capture_native_visual_signal/, 'Ready-shell visual probe must try system pixels first, then use a fresh native WKWebView snapshot only as a strict fallback'); From 759589853619ae181e811160183b13e9bab936ca Mon Sep 17 00:00:00 2001 From: fkb032 <249513614+fkb032@users.noreply.github.com> Date: Fri, 8 May 2026 00:22:31 -0700 Subject: [PATCH 13/16] fix: add webkit pdf fallback for peekaboo snapshots --- apps/native/src-tauri/src/main.rs | 240 ++++++++++++++++++ .../peekaboo-workflow-contract-self-test.mjs | 2 + 2 files changed, 242 insertions(+) diff --git a/apps/native/src-tauri/src/main.rs b/apps/native/src-tauri/src/main.rs index 17cd7e50b..dd7b8ec94 100644 --- a/apps/native/src-tauri/src/main.rs +++ b/apps/native/src-tauri/src/main.rs @@ -441,6 +441,77 @@ unsafe fn e2e_write_bitmap_rep_png( }) } +#[cfg(all(debug_assertions, target_os = "macos"))] +unsafe fn e2e_write_nsdata( + data: *mut objc::runtime::Object, + output_path: &std::path::Path, +) -> Result<(), String> { + use objc::msg_send; + use objc::sel; + use objc::sel_impl; + + if data.is_null() { + return Err("NSData was nil".to_string()); + } + + let bytes: *const std::ffi::c_void = msg_send![data, bytes]; + let len: usize = msg_send![data, length]; + if bytes.is_null() || len == 0 { + return Err("NSData was empty".to_string()); + } + + let slice = std::slice::from_raw_parts(bytes.cast::(), len); + let tmp_output = output_path.with_extension( + output_path + .extension() + .and_then(|extension| extension.to_str()) + .map(|extension| format!("{extension}.tmp")) + .unwrap_or_else(|| "tmp".to_string()), + ); + std::fs::write(&tmp_output, slice) + .and_then(|()| std::fs::rename(&tmp_output, output_path)) + .map_err(|error| { + let _ = std::fs::remove_file(&tmp_output); + format!("failed to write NSData atomically: {error}") + }) +} + +#[cfg(all(debug_assertions, target_os = "macos"))] +unsafe fn e2e_write_pdf_data_as_png( + pdf_data: *mut objc::runtime::Object, + output_path: &std::path::Path, +) -> Result<(), String> { + use objc::class; + use objc::msg_send; + use objc::runtime::Object; + use objc::sel; + use objc::sel_impl; + + let ns_image_alloc: *mut Object = msg_send![class!(NSImage), alloc]; + if ns_image_alloc.is_null() { + return Err("NSImage alloc returned nil".to_string()); + } + let ns_image: *mut Object = msg_send![ns_image_alloc, initWithData: pdf_data]; + if ns_image.is_null() { + return Err("NSImage could not initialize from PDF data".to_string()); + } + + let tiff_data: *mut Object = msg_send![ns_image, TIFFRepresentation]; + if tiff_data.is_null() { + return Err("NSImage TIFFRepresentation for PDF data was nil".to_string()); + } + + let bitmap_rep: *mut Object = msg_send![class!(NSBitmapImageRep), imageRepWithData: tiff_data]; + if bitmap_rep.is_null() { + let _: () = msg_send![ns_image, release]; + return Err("NSBitmapImageRep could not read rendered PDF image data".to_string()); + } + + let result = e2e_write_bitmap_rep_png(bitmap_rep, output_path); + let _: () = msg_send![ns_image, release]; + result +} + #[cfg(all(debug_assertions, target_os = "macos"))] unsafe fn e2e_try_cached_display_snapshot( webview: *mut objc::runtime::Object, @@ -536,6 +607,162 @@ unsafe fn e2e_try_cached_display_snapshot( } } +#[cfg(all(debug_assertions, target_os = "macos"))] +unsafe fn e2e_try_pdf_snapshot( + webview: *mut objc::runtime::Object, + status_path: &std::path::Path, + label: &str, + output_path: &std::path::Path, + prior_error: &str, +) -> bool { + use block::ConcreteBlock; + use objc::class; + use objc::msg_send; + use objc::runtime::Object; + use objc::sel; + use objc::sel_impl; + + if webview.is_null() { + return false; + } + let can_create_pdf: bool = + msg_send![webview, respondsToSelector: sel!(createPDFWithConfiguration:completionHandler:)]; + if !can_create_pdf { + return false; + } + + #[repr(C)] + #[derive(Clone, Copy)] + struct E2eNativePoint { + x: f64, + y: f64, + } + + #[repr(C)] + #[derive(Clone, Copy)] + struct E2eNativeSize { + width: f64, + height: f64, + } + + #[repr(C)] + #[derive(Clone, Copy)] + struct E2eNativeRect { + origin: E2eNativePoint, + size: E2eNativeSize, + } + + let bounds: E2eNativeRect = msg_send![webview, bounds]; + if bounds.size.width <= 0.0 || bounds.size.height <= 0.0 { + return false; + } + + let configuration: *mut Object = msg_send![class!(WKPDFConfiguration), new]; + if configuration.is_null() { + return false; + } + let _: () = msg_send![configuration, setRect: bounds]; + if msg_send![configuration, respondsToSelector: sel!(setAllowTransparentBackground:)] { + let _: () = msg_send![configuration, setAllowTransparentBackground: false]; + } + + let label_for_status = label.to_string(); + let output_for_block = output_path.to_path_buf(); + let status_for_block = status_path.to_path_buf(); + let prior_error_for_block = prior_error.to_string(); + let pdf_output = output_path.with_extension("pdf"); + let webview_for_fallback = webview; + let completion = ConcreteBlock::new(move |pdf_data: *mut Object, error: *mut Object| { + if !error.is_null() { + let error_message = e2e_ns_error_summary(error); + if e2e_try_cached_display_snapshot( + webview_for_fallback, + &status_for_block, + &label_for_status, + &output_for_block, + &format!( + "{}; WKWebView PDF fallback failed ({})", + prior_error_for_block, error_message + ), + ) { + log::info!( + "NIXMAC_E2E_NATIVE_SNAPSHOT used AppKit cached-display fallback for {} after PDF failure", + label_for_status + ); + return; + } + e2e_write_native_snapshot_status_with_source( + &status_for_block, + "failed", + &label_for_status, + &output_for_block, + Some(&format!( + "{}; WKWebView PDF fallback failed ({})", + prior_error_for_block, error_message + )), + "WKWebView.createPDFWithConfiguration", + ); + return; + } + + if pdf_data.is_null() { + e2e_write_native_snapshot_status_with_source( + &status_for_block, + "failed", + &label_for_status, + &output_for_block, + Some(&format!( + "{}; WKWebView PDF fallback returned no data", + prior_error_for_block + )), + "WKWebView.createPDFWithConfiguration", + ); + return; + } + + match e2e_write_nsdata(pdf_data, &pdf_output) + .and_then(|()| e2e_write_pdf_data_as_png(pdf_data, &output_for_block)) + { + Ok(()) => { + e2e_write_native_snapshot_status_with_source( + &status_for_block, + "passed", + &label_for_status, + &output_for_block, + Some(&format!( + "WKWebView snapshot failed first ({}); WKWebView PDF fallback wrote PNG and PDF", + prior_error_for_block + )), + "WKWebView.createPDFWithConfiguration", + ); + } + Err(error) => { + e2e_write_native_snapshot_status_with_source( + &status_for_block, + "failed", + &label_for_status, + &output_for_block, + Some(&format!( + "{}; WKWebView PDF fallback could not render PNG ({})", + prior_error_for_block, error + )), + "WKWebView.createPDFWithConfiguration", + ); + } + } + }) + .copy(); + + let _: () = msg_send![ + webview, + createPDFWithConfiguration: configuration + completionHandler: &*completion + ]; + let _: () = msg_send![configuration, release]; + std::mem::forget(completion); + true +} + #[cfg(all(debug_assertions, target_os = "macos"))] unsafe fn e2e_nsstring_to_string(value: *mut objc::runtime::Object) -> Option { use objc::{msg_send, sel, sel_impl}; @@ -627,6 +854,19 @@ fn e2e_request_native_webview_snapshot( let completion = ConcreteBlock::new(move |image: *mut Object, error: *mut Object| { if !error.is_null() { let error_message = e2e_ns_error_summary(error); + if e2e_try_pdf_snapshot( + webview_for_fallback, + &status_for_block, + &label_for_status, + &output_for_block, + &error_message, + ) { + log::info!( + "NIXMAC_E2E_NATIVE_SNAPSHOT requested WKWebView PDF fallback for {}", + label_for_status + ); + return; + } if e2e_try_cached_display_snapshot( webview_for_fallback, &status_for_block, diff --git a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs index 6c0b4e50d..81f0f31bf 100644 --- a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs +++ b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs @@ -340,6 +340,8 @@ assert.match(nativeWebviewCaptureProbeScheduler, /run_on_main_thread[\s\S]*e2e_r assert.match(nativeWebviewSnapshotScheduler, /run_on_main_thread[\s\S]*e2e_request_native_webview_snapshot/, 'Scheduled native WebView snapshots must touch WKWebView from the main thread'); assert.match(nativeWebviewSnapshotPoller, /run_on_main_thread[\s\S]*e2e_request_native_webview_snapshot/, 'On-demand native WebView snapshot requests must touch WKWebView from the main thread'); assert.match(nativeMain, /let configuration: \*mut Object = std::ptr::null_mut\(\);[\s\S]*takeSnapshotWithConfiguration: configuration/, 'Native snapshot requests must use WebKit full-view snapshots so virtualized hosts do not fail on stale AppKit rect configuration'); +assert.match(nativeMain, /fn e2e_try_pdf_snapshot[\s\S]*WKPDFConfiguration[\s\S]*createPDFWithConfiguration: configuration/, 'Native snapshot requests must try WebKit PDF rendering before falling back to black-prone AppKit cached display on virtualized hosts'); +assert.match(nativeMain, /WKWebView\.createPDFWithConfiguration/, 'Native snapshot PDF fallback status must preserve WebKit PDF provenance'); assert.match(nativeMain, /e2e_try_cached_display_snapshot[\s\S]*bitmapImageRepForCachingDisplayInRect[\s\S]*cacheDisplayInRect[\s\S]*"degraded"[\s\S]*NSView\.cacheDisplayInRect/, 'Native snapshot requests must retain AppKit cached-display fallback as degraded diagnostics, not passing visual proof'); assert.match(productProof, /status=\$\(jq -r '\.status \/\/ ""' "\$status_path"[\s\S]*passed\)[\s\S]*\[ -s "\$output_path" \][\s\S]*degraded\)[\s\S]*return 1[\s\S]*failed\)[\s\S]*return 1/, 'Product Proof native snapshot reader must wait for terminal status and accept only status=passed before returning a PNG path'); assert.match(productProof, /nixmac_pp_capture_ready_visual_signal\(\)[\s\S]*peekaboo_run see --app "\$NIXMAC_APP_NAME" --path "\$path"[\s\S]*nixmac_pp_screenshot_has_visual_signal "\$path"[\s\S]*nixmac_pp_capture_native_visual_signal/, 'Ready-shell visual probe must try system pixels first, then use a fresh native WKWebView snapshot only as a strict fallback'); From 2bf8c90a775450587faf7dfa397c201899a30a29 Mon Sep 17 00:00:00 2001 From: fkb032 <249513614+fkb032@users.noreply.github.com> Date: Fri, 8 May 2026 01:05:09 -0700 Subject: [PATCH 14/16] fix: defer peekaboo native captures until webview load --- apps/native/src-tauri/src/main.rs | 34 ++++++++++++------- tests/e2e/lib/nixmac_product_proof.sh | 6 ++-- .../peekaboo-workflow-contract-self-test.mjs | 11 +++--- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/apps/native/src-tauri/src/main.rs b/apps/native/src-tauri/src/main.rs index dd7b8ec94..f85ef175b 100644 --- a/apps/native/src-tauri/src/main.rs +++ b/apps/native/src-tauri/src/main.rs @@ -65,6 +65,10 @@ fn e2e_webview_watchdog_enabled() -> bool { cfg!(debug_assertions) && crate::e2e_runtime::enabled("NIXMAC_E2E_WEBVIEW_WATCHDOG") } +fn e2e_preload_native_capture_enabled() -> bool { + cfg!(debug_assertions) && crate::e2e_runtime::enabled("NIXMAC_E2E_PRELOAD_NATIVE_CAPTURE") +} + #[cfg(all(debug_assertions, target_os = "macos"))] static E2E_NATIVE_SNAPSHOT_COUNTER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); @@ -726,11 +730,11 @@ unsafe fn e2e_try_pdf_snapshot( Ok(()) => { e2e_write_native_snapshot_status_with_source( &status_for_block, - "passed", + "rendered", &label_for_status, &output_for_block, Some(&format!( - "WKWebView snapshot failed first ({}); WKWebView PDF fallback wrote PNG and PDF", + "WKWebView snapshot failed first ({}); WKWebView PDF fallback wrote PNG and PDF; caller must validate visual signal before using it as proof", prior_error_for_block )), "WKWebView.createPDFWithConfiguration", @@ -2126,16 +2130,22 @@ fn run_gui_mode( if e2e_css_capture { e2e_start_native_webview_snapshot_request_poller(main_window.clone()); - e2e_schedule_native_webview_capture_probe( - main_window.clone(), - "native-post-build-plus-2s", - Duration::from_secs(2), - ); - e2e_schedule_native_webview_snapshot( - main_window.clone(), - "post-build-plus-2s", - Duration::from_secs(2), - ); + if e2e_preload_native_capture_enabled() { + e2e_schedule_native_webview_capture_probe( + main_window.clone(), + "native-post-build-plus-2s", + Duration::from_secs(2), + ); + e2e_schedule_native_webview_snapshot( + main_window.clone(), + "post-build-plus-2s", + Duration::from_secs(2), + ); + } else { + log::info!( + "NIXMAC_E2E_PRELOAD_NATIVE_CAPTURE disabled; deferring native WebView captures until page-load or explicit shell request" + ); + } } if e2e_webview_watchdog { diff --git a/tests/e2e/lib/nixmac_product_proof.sh b/tests/e2e/lib/nixmac_product_proof.sh index 92cf86071..153e8357b 100644 --- a/tests/e2e/lib/nixmac_product_proof.sh +++ b/tests/e2e/lib/nixmac_product_proof.sh @@ -417,12 +417,14 @@ nixmac_pp_request_native_webview_snapshot() { if [ -s "$status_path" ]; then status=$(jq -r '.status // ""' "$status_path" 2>/dev/null || true) case "$status" in - passed) + # "rendered" means a fallback produced PNG bytes, not pass-grade proof. + # Callers must run nixmac_pp_screenshot_has_visual_signal before trusting it. + passed|rendered) if [ -s "$output_path" ]; then printf '%s\n' "$output_path" return 0 fi - debug "Native WKWebView snapshot status passed for $label but PNG is not ready yet" + debug "Native WKWebView snapshot status $status for $label but PNG is not ready yet" ;; degraded) debug "Native WKWebView snapshot degraded for $label: $(jq -r '.message // "unknown"' "$status_path" 2>/dev/null || echo unknown)" diff --git a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs index 81f0f31bf..99a3de045 100644 --- a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs +++ b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs @@ -249,7 +249,9 @@ assert.match(nativeMain, /if e2e_css_capture \{[\s\S]*match main_window\.ns_wind assert.match(nativeMain, /respondsToSelector: sel!\(_setWindowOcclusionDetectionEnabled:\)[\s\S]*_setWindowOcclusionDetectionEnabled: NO[\s\S]*setHidesOnDeactivate: NO[\s\S]*setCanHide: NO/, 'Native app must disable macOS occlusion/hiding behavior in the debug-only opaque E2E capture branch'); assert.match(nativeMain, /NIXMAC_E2E_CAPTURE native window diagnostics:[\s\S]*sharingTypeBefore=\{\}[\s\S]*sharingTypeAfter=\{\}[\s\S]*isOpaque[\s\S]*alphaValue[\s\S]*hasShadow[\s\S]*occlusionDetectionDisabled/, 'Native app must log native sharing and occlusion diagnostics for MacInCloud capture debugging'); assert.match(nativeMain, /fn e2e_request_native_webview_capture_probe[\s\S]*with_webview\(move \|webview\|[\s\S]*respondsToSelector: sel!\(drawsBackground\)[\s\S]*setNeedsDisplay: YES[\s\S]*displayIfNeeded[\s\S]*NIXMAC_E2E_CAPTURE webview diagnostics:[\s\S]*fn e2e_schedule_native_webview_capture_probe/, 'Native app must provide guarded WebView diagnostics and a best-effort AppKit display hint for MacInCloud black-capture diagnosis'); -assert.match(nativeMain, /e2e_page_load_native_capture_probe[\s\S]*native-page-load-finished-plus-5s[\s\S]*e2e_schedule_native_webview_capture_probe[\s\S]*native-post-build-plus-2s/, 'Native app must schedule native WebView capture diagnostics after build and after page-load'); +assert.match(nativeMain, /fn e2e_preload_native_capture_enabled\(\) -> bool \{\n\s+cfg!\(debug_assertions\) && crate::e2e_runtime::enabled\("NIXMAC_E2E_PRELOAD_NATIVE_CAPTURE"\)/, 'Native app must keep pre-load native capture diagnostics behind an explicit E2E-only opt-in gate'); +assert.match(nativeMain, /e2e_page_load_native_capture_probe[\s\S]*native-page-load-finished-plus-5s[\s\S]*e2e_schedule_native_webview_capture_probe/, 'Native app must schedule native WebView capture diagnostics after page-load'); +assert.match(nativeMain, /if e2e_preload_native_capture_enabled\(\) \{[\s\S]*native-post-build-plus-2s[\s\S]*post-build-plus-2s[\s\S]*NIXMAC_E2E_PRELOAD_NATIVE_CAPTURE disabled/, 'Native app must not run pre-load native WebView captures by default on MacInCloud'); assert.match(cargoToml, /\[target\.'cfg\(target_os = "macos"\)'\.dependencies\][\s\S]*block = "0\.1"[\s\S]*objc = "0\.2"/, 'Native WKWebView snapshot completion blocks must declare the small macOS-only block crate beside objc/cocoa'); assert.match(nativeMain, /fn e2e_ns_error_summary[\s\S]*localizedDescription[\s\S]*domain[\s\S]*code[\s\S]*userInfo/, 'Native WKWebView snapshot failures must persist the actual NSError domain, code, description, and userInfo'); assert.match(nativeMain, /fn e2e_write_bitmap_rep_png[\s\S]*NSBitmapImageRep[\s\S]*representationUsingType[\s\S]*fn e2e_request_native_webview_snapshot[\s\S]*ConcreteBlock::new[\s\S]*takeSnapshotWithConfiguration[\s\S]*std::mem::forget\(completion\)/, 'Native app must provide an E2E-only WKWebView snapshot writer using WebKit snapshot API and explicit async block lifetime handling'); @@ -258,7 +260,7 @@ assert.match(nativeMain, /let tmp_output = output_path\.with_extension\("png\.tm assert.match(nativeMain, /fn e2e_native_snapshot_root_dir[\s\S]*NIXMAC_E2E_DIAGNOSTICS_DIR[\s\S]*native-webview-snapshots/, 'Native snapshots must reuse the existing E2E diagnostics directory'); assert.match(nativeMain, /fn e2e_start_native_webview_snapshot_request_poller[\s\S]*request_dir[\s\S]*requests[\s\S]*\.request[\s\S]*e2e_request_native_webview_snapshot/, 'Native app must poll diagnostics-dir snapshot requests so Scott’s shell driver can demand fresh native visual evidence'); assert.match(nativeMain, /let output_path = root\.join\(format!\("\{request_id\}\.png"\)\);[\s\S]*let status_path = root\.join\(format!\("\{request_id\}\.json"\)\);/, 'Native snapshot poller must use the shell request id verbatim so shell and Rust wait/write paths cannot desync on long labels'); -assert.match(nativeMain, /e2e_schedule_native_webview_snapshot[\s\S]*page-load-finished-plus-5s[\s\S]*e2e_start_native_webview_snapshot_request_poller[\s\S]*post-build-plus-2s/, 'Native app must take bounded readiness snapshots and start the on-demand snapshot request poller only in E2E capture mode'); +assert.match(nativeMain, /e2e_schedule_native_webview_snapshot[\s\S]*page-load-finished-plus-5s[\s\S]*e2e_start_native_webview_snapshot_request_poller/, 'Native app must take bounded page-load readiness snapshots and start the on-demand snapshot request poller only in E2E capture mode'); assert.doesNotMatch(nativeMain, /setInterval|Duration::from_millis\((?:1000|2000|3000)\)[\s\S]*e2e_request_native_webview_snapshot/, 'Native app must not create noisy background interval snapshots while scenarios run'); assert.match(nativeSolidCaptureBranch, /NIXMAC_E2E_SOLID_CAPTURE enabled/, 'Native app must keep an explicit solid-capture branch for diagnostics'); assert.doesNotMatch(nativeSolidCaptureBranch, /background_color\(tauri::utils::config::Color\(10, 10, 10, 255\)\)/, 'Native app must not apply native background_color from the default solid-capture path'); @@ -334,6 +336,7 @@ assert.match(productProof, /nixmac_pp_elements_show_ready_shell\(\)[\s\S]*NIXMAC assert.match(productProof, /nixmac_pp_screenshot_has_visual_signal\(\)[\s\S]*visual-proof\.mjs[\s\S]*pngSignalStats[\s\S]*probeCropForImage/, 'ready-shell gate must use the same screenshot signal helpers as the report scanner'); assert.match(productProof, /maxDarkChromeYAvg: 42/, 'ready-shell visual gate must enforce the same nixmac dark-capture upper bound as the report scanner'); assert.match(productProof, /nixmac_pp_request_native_webview_snapshot\(\)[\s\S]*NIXMAC_E2E_DIAGNOSTICS_DIR[\s\S]*native-webview-snapshots[\s\S]*\.request[\s\S]*status_path/, 'Product Proof must request fresh native WKWebView snapshots through the existing diagnostics directory'); +assert.match(productProof, /passed\|rendered\)[\s\S]*printf '%s\\n' "\$output_path"/, 'Product Proof may consume rendered PDF fallback PNGs only after the shell visual-signal probe validates their pixels'); assert.match(productProof, /nixmac_pp_capture_native_visual_signal\(\)[\s\S]*nixmac_pp_request_native_webview_snapshot[\s\S]*nixmac_pp_screenshot_has_visual_signal/, 'Native WKWebView fallback must run the exact same visual signal probe before it can satisfy readiness'); assert.match(productProof, /disable_marker="\$\{NIXMAC_E2E_DIAGNOSTICS_DIR:-\$E2E_DIAGNOSTIC_DIR\}\/native-webview-snapshots\/\.disabled"[\s\S]*\[ ! -f "\$disable_marker" \][\s\S]*visual-signal-failed/, 'Native WKWebView snapshot fallback must persist scenario-level disable state outside command-substitution subshells'); assert.match(nativeWebviewCaptureProbeScheduler, /run_on_main_thread[\s\S]*e2e_request_native_webview_capture_probe/, 'Native WebView capture probes must touch AppKit/WebKit from the main thread'); @@ -341,9 +344,9 @@ assert.match(nativeWebviewSnapshotScheduler, /run_on_main_thread[\s\S]*e2e_reque assert.match(nativeWebviewSnapshotPoller, /run_on_main_thread[\s\S]*e2e_request_native_webview_snapshot/, 'On-demand native WebView snapshot requests must touch WKWebView from the main thread'); assert.match(nativeMain, /let configuration: \*mut Object = std::ptr::null_mut\(\);[\s\S]*takeSnapshotWithConfiguration: configuration/, 'Native snapshot requests must use WebKit full-view snapshots so virtualized hosts do not fail on stale AppKit rect configuration'); assert.match(nativeMain, /fn e2e_try_pdf_snapshot[\s\S]*WKPDFConfiguration[\s\S]*createPDFWithConfiguration: configuration/, 'Native snapshot requests must try WebKit PDF rendering before falling back to black-prone AppKit cached display on virtualized hosts'); -assert.match(nativeMain, /WKWebView\.createPDFWithConfiguration/, 'Native snapshot PDF fallback status must preserve WebKit PDF provenance'); +assert.match(nativeMain, /"rendered"[\s\S]*WKWebView PDF fallback wrote PNG and PDF; caller must validate visual signal before using it as proof[\s\S]*WKWebView\.createPDFWithConfiguration/, 'Native snapshot PDF fallback status must preserve WebKit PDF provenance without marking unverified PDF pixels as pass-grade proof'); assert.match(nativeMain, /e2e_try_cached_display_snapshot[\s\S]*bitmapImageRepForCachingDisplayInRect[\s\S]*cacheDisplayInRect[\s\S]*"degraded"[\s\S]*NSView\.cacheDisplayInRect/, 'Native snapshot requests must retain AppKit cached-display fallback as degraded diagnostics, not passing visual proof'); -assert.match(productProof, /status=\$\(jq -r '\.status \/\/ ""' "\$status_path"[\s\S]*passed\)[\s\S]*\[ -s "\$output_path" \][\s\S]*degraded\)[\s\S]*return 1[\s\S]*failed\)[\s\S]*return 1/, 'Product Proof native snapshot reader must wait for terminal status and accept only status=passed before returning a PNG path'); +assert.match(productProof, /status=\$\(jq -r '\.status \/\/ ""' "\$status_path"[\s\S]*passed\|rendered\)[\s\S]*\[ -s "\$output_path" \][\s\S]*degraded\)[\s\S]*return 1[\s\S]*failed\)[\s\S]*return 1/, 'Product Proof native snapshot reader must wait for terminal status and accept only pass-grade or externally-validated rendered PNG candidates before the visual-signal probe decides proof quality'); assert.match(productProof, /nixmac_pp_capture_ready_visual_signal\(\)[\s\S]*peekaboo_run see --app "\$NIXMAC_APP_NAME" --path "\$path"[\s\S]*nixmac_pp_screenshot_has_visual_signal "\$path"[\s\S]*nixmac_pp_capture_native_visual_signal/, 'Ready-shell visual probe must try system pixels first, then use a fresh native WKWebView snapshot only as a strict fallback'); assert.match(productProof, /nixmac_pp_record_ready_visual_signal\(\)[\s\S]*status=failed[\s\S]*preserving strict screenshot-signal report gate/, 'ready-shell must record visual proof quality without weakening the runner screenshot-signal gate'); assert.match(productProof, /nixmac_pp_wait_for_ready_app_shell\(\)[\s\S]*nixmac_pp_elements_show_ready_shell[\s\S]*nixmac_pp_record_ready_visual_signal "ready-shell" \|\| true[\s\S]*return 0/, 'ready-shell readiness must succeed on driver-visible AX/product evidence instead of requiring host pixels before launch passes'); From bd746f473b18c74f632c4dbaa220c5aa76ebeb34 Mon Sep 17 00:00:00 2001 From: fkb032 <249513614+fkb032@users.noreply.github.com> Date: Fri, 8 May 2026 01:40:36 -0700 Subject: [PATCH 15/16] fix: try peekaboo window capture before native snapshots --- tests/e2e/adapters/nixmac.sh | 23 ++++++++++++- tests/e2e/lib/nixmac_product_proof.sh | 27 ++++++++++++++- tests/e2e/lib/peekaboo.sh | 34 +++++++++++++++++++ tests/e2e/lib/peekaboo.test.sh | 15 ++++++++ .../peekaboo-workflow-contract-self-test.mjs | 5 ++- 5 files changed, 101 insertions(+), 3 deletions(-) diff --git a/tests/e2e/adapters/nixmac.sh b/tests/e2e/adapters/nixmac.sh index 0e4376b5f..7c5938a6d 100644 --- a/tests/e2e/adapters/nixmac.sh +++ b/tests/e2e/adapters/nixmac.sh @@ -171,9 +171,30 @@ nixmac_text() { nixmac_screenshot() { local label="${1:-nixmac}" - local system_path native_output native_path native_dest system_diag_dir system_diag_path + local system_path window_output window_path window_dest native_output native_path native_dest system_diag_dir system_diag_path system_path=$(screenshot "$label" "$NIXMAC_APP_NAME" | tail -n 1) + if declare -f nixmac_pp_capture_window_visual_signal >/dev/null 2>&1; then + window_output=$(nixmac_pp_capture_window_visual_signal "$label") && { + window_path=$(printf '%s\n' "$window_output" | tail -n 1) + mkdir -p "$E2E_SCREENSHOT_DIR" 2>/dev/null || true + window_dest="$E2E_SCREENSHOT_DIR/${label//[^a-zA-Z0-9._-]/_}-peekaboo-window-$(date +%s)-$$-$RANDOM.png" + if cp "$window_path" "$window_dest" 2>/dev/null; then + if [ -n "$system_path" ] && [ -f "$system_path" ]; then + system_diag_dir="${NIXMAC_E2E_DIAGNOSTICS_DIR:-$E2E_DIAGNOSTIC_DIR}/system-captures" + mkdir -p "$system_diag_dir" 2>/dev/null || true + system_diag_path="$system_diag_dir/$(basename "$system_path" .png)-peekaboo-system.png" + mv "$system_path" "$system_diag_path" 2>/dev/null || true + log "Promoted Peekaboo window-id capture for $label; retained system capture diagnostic: $system_diag_path" + else + log "Promoted Peekaboo window-id capture for $label" + fi + printf '%s\n' "$window_dest" + return 0 + fi + } + fi + if ! declare -f nixmac_pp_capture_native_visual_signal >/dev/null 2>&1; then printf '%s\n' "$system_path" return 0 diff --git a/tests/e2e/lib/nixmac_product_proof.sh b/tests/e2e/lib/nixmac_product_proof.sh index 153e8357b..84e775e5d 100644 --- a/tests/e2e/lib/nixmac_product_proof.sh +++ b/tests/e2e/lib/nixmac_product_proof.sh @@ -468,9 +468,28 @@ nixmac_pp_capture_native_visual_signal() { printf '%s\n' "$path" } +nixmac_pp_capture_window_visual_signal() { + local label="${1:-window-capture}" + local dir path result + + dir="${NIXMAC_E2E_DIAGNOSTICS_DIR:-$E2E_DIAGNOSTIC_DIR}/window-captures" + mkdir -p "$dir" + path="$dir/${label//[^a-zA-Z0-9._-]/_}-window-$(date +%s)-$$-$RANDOM.png" + if ! peekaboo_capture_window_image "$NIXMAC_APP_NAME" "$path"; then + debug "Peekaboo window-id capture failed for $label" + return 1 + fi + result=$(nixmac_pp_screenshot_has_visual_signal "$path" 2>&1) || { + debug "Peekaboo window-id capture visual signal not established for $path: $result" + return 1 + } + debug "Peekaboo window-id capture visual signal ready for $path: $result" + printf '%s\n' "$path" +} + nixmac_pp_capture_ready_visual_signal() { local label="${1:-ready-shell}" - local dir path result native_output native_path + local dir path result window_output window_path native_output native_path dir="$E2E_DIAGNOSTIC_DIR/visual-readiness" mkdir -p "$dir" @@ -487,6 +506,12 @@ nixmac_pp_capture_ready_visual_signal() { debug "Ready-shell system visual signal not established for $path: $result" fi + window_output=$(nixmac_pp_capture_window_visual_signal "$label") && { + window_path=$(printf '%s\n' "$window_output" | tail -n 1) + debug "Ready-shell visual signal established from Peekaboo window-id capture: $window_path" + return 0 + } + native_output=$(nixmac_pp_capture_native_visual_signal "$label") || { debug "Ready-shell visual signal not established for $path: $result" return 1 diff --git a/tests/e2e/lib/peekaboo.sh b/tests/e2e/lib/peekaboo.sh index fa2fecead..5cbddc8cc 100644 --- a/tests/e2e/lib/peekaboo.sh +++ b/tests/e2e/lib/peekaboo.sh @@ -191,6 +191,40 @@ peekaboo_capture_app_diagnostics() { echo "[diagnostic] Captured Peekaboo diagnostics for $app ($reason): $prefix-*" >> "$E2E_LOG_FILE" } +peekaboo_window_id_for_app() { + local app="$1" + local json window_id + + [ -n "$app" ] || return 1 + json=$(peekaboo_run window list --app "$app" --json 2>/dev/null || echo '{}') + window_id=$(echo "$json" | jq -r ' + [ + .data.windows[]? | + select((.is_on_screen // true) == true) | + # Skip tooltips/popovers and prefer the real app window. + select(((.bounds.width // 0) >= 500) and ((.bounds.height // 0) >= 350)) | + .window_id + ][0] // [ + .data.windows[]? | + select((.is_on_screen // true) == true) | + .window_id + ][0] // "" + ' 2>/dev/null | head -1) + [ -n "$window_id" ] || return 1 + printf '%s\n' "$window_id" +} + +peekaboo_capture_window_image() { + local app="$1" + local path="$2" + local window_id + + [ -n "$path" ] || return 1 + window_id=$(peekaboo_window_id_for_app "$app") || return 1 + mkdir -p "$(dirname "$path")" || return 1 + peekaboo_run image --mode window --window-id "$window_id" --path "$path" >/dev/null 2>&1 +} + peekaboo_app_pid() { local app="$1" local bundle="${E2E_ACTIVE_BUNDLE_ID:-}" diff --git a/tests/e2e/lib/peekaboo.test.sh b/tests/e2e/lib/peekaboo.test.sh index 657f517b9..66e15913e 100644 --- a/tests/e2e/lib/peekaboo.test.sh +++ b/tests/e2e/lib/peekaboo.test.sh @@ -111,6 +111,21 @@ find "$E2E_DIAGNOSTIC_DIR" -type f -name '*-app-list.json' | grep -q . || { exit 1 } +window_id="$(peekaboo_window_id_for_app nixmac)" +[ "$window_id" = "1" ] || { + echo "expected window-id helper to return 1, got $window_id" >&2 + exit 1 +} +window_capture="$TEST_DIR/window-capture.png" +peekaboo_capture_window_image nixmac "$window_capture" || { + echo "expected window-id capture helper to succeed" >&2 + exit 1 +} +[ -s "$window_capture" ] || { + echo "expected window-id capture helper to write a PNG" >&2 + exit 1 +} + export E2E_ACTIVE_APP_NAME="nixmac" PEEKABOO_TEST_RESTORE_READY=0 : > "$TEST_DIR/app-switch.log" diff --git a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs index 99a3de045..16c057c2d 100644 --- a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs +++ b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs @@ -335,6 +335,9 @@ assert.match(productProof, /nixmac_pp_wait_for_ready_app_shell\(\)/, 'Product Pr assert.match(productProof, /nixmac_pp_elements_show_ready_shell\(\)[\s\S]*NIXMAC_PP_READY_SHELL_MIN_ELEMENTS[\s\S]*NIXMAC_PP_READY_SHELL_PATTERN/, 'ready-shell gate must require both element breadth and product markers'); assert.match(productProof, /nixmac_pp_screenshot_has_visual_signal\(\)[\s\S]*visual-proof\.mjs[\s\S]*pngSignalStats[\s\S]*probeCropForImage/, 'ready-shell gate must use the same screenshot signal helpers as the report scanner'); assert.match(productProof, /maxDarkChromeYAvg: 42/, 'ready-shell visual gate must enforce the same nixmac dark-capture upper bound as the report scanner'); +assert.match(peekabooShell, /peekaboo_window_id_for_app\(\)[\s\S]*window list --app "\$app" --json[\s\S]*\.data\.windows\[\]\?[\s\S]*\.window_id[\s\S]*peekaboo_capture_window_image\(\)[\s\S]*image --mode window --window-id "\$window_id" --path "\$path"/, 'Peekaboo helpers must expose a window-id capture path before falling back to full-screen pixels'); +assert.match(productProof, /nixmac_pp_capture_window_visual_signal\(\)[\s\S]*peekaboo_capture_window_image "\$NIXMAC_APP_NAME" "\$path"[\s\S]*nixmac_pp_screenshot_has_visual_signal "\$path"/, 'Product Proof must only use Peekaboo window-id captures after the strict visual-signal probe passes'); +assert.match(productProof, /nixmac_pp_capture_ready_visual_signal\(\)[\s\S]*nixmac_pp_capture_window_visual_signal "\$label"[\s\S]*nixmac_pp_capture_native_visual_signal "\$label"/, 'Ready-shell visual proof must try system pixels, then window-id pixels, then native WebView diagnostics'); assert.match(productProof, /nixmac_pp_request_native_webview_snapshot\(\)[\s\S]*NIXMAC_E2E_DIAGNOSTICS_DIR[\s\S]*native-webview-snapshots[\s\S]*\.request[\s\S]*status_path/, 'Product Proof must request fresh native WKWebView snapshots through the existing diagnostics directory'); assert.match(productProof, /passed\|rendered\)[\s\S]*printf '%s\\n' "\$output_path"/, 'Product Proof may consume rendered PDF fallback PNGs only after the shell visual-signal probe validates their pixels'); assert.match(productProof, /nixmac_pp_capture_native_visual_signal\(\)[\s\S]*nixmac_pp_request_native_webview_snapshot[\s\S]*nixmac_pp_screenshot_has_visual_signal/, 'Native WKWebView fallback must run the exact same visual signal probe before it can satisfy readiness'); @@ -352,7 +355,7 @@ assert.match(productProof, /nixmac_pp_record_ready_visual_signal\(\)[\s\S]*statu assert.match(productProof, /nixmac_pp_wait_for_ready_app_shell\(\)[\s\S]*nixmac_pp_elements_show_ready_shell[\s\S]*nixmac_pp_record_ready_visual_signal "ready-shell" \|\| true[\s\S]*return 0/, 'ready-shell readiness must succeed on driver-visible AX/product evidence instead of requiring host pixels before launch passes'); assert.doesNotMatch(productProof, /nixmac_pp_wait_for_ready_app_shell\(\)[\s\S]*if nixmac_pp_capture_ready_visual_signal "ready-shell"; then[\s\S]*return 0/, 'ready-shell readiness must not block scenario execution on host pixel capture'); assert.match(peekabooRunner, /function readWebviewProof\(runDir\)[\s\S]*nixmac-frontend-breadcrumbs\.jsonl[\s\S]*assetFailures[\s\S]*webview-proof\.json[\s\S]*host pixel capture is likely black\/occluded/, 'Peekaboo runner must write WebView proof diagnostics and distinguish DOM-rendered failures from host pixel-capture failures without passing them'); -assert.match(nixmacAdapter, /nixmac_screenshot\(\)[\s\S]*screenshot "\$label" "\$NIXMAC_APP_NAME"[\s\S]*nixmac_pp_capture_native_visual_signal[\s\S]*webkit-snapshot[\s\S]*system-captures[\s\S]*Promoted native WKWebView snapshot/, 'nixmac screenshots must promote passing WKWebView snapshots into screenshot evidence while retaining black system captures as diagnostics'); +assert.match(nixmacAdapter, /nixmac_screenshot\(\)[\s\S]*screenshot "\$label" "\$NIXMAC_APP_NAME"[\s\S]*nixmac_pp_capture_window_visual_signal[\s\S]*peekaboo-window[\s\S]*Promoted Peekaboo window-id capture[\s\S]*nixmac_pp_capture_native_visual_signal[\s\S]*webkit-snapshot[\s\S]*system-captures[\s\S]*Promoted native WKWebView snapshot/, 'nixmac screenshots must promote passing window-id or native snapshots into screenshot evidence while retaining black system captures as diagnostics'); assert.match(peekabooRunner, /webkit-snapshot[\s\S]*WKWebView internal snapshot captured from the running nixmac WebContent surface/, 'Peekaboo runner must label WKWebView snapshot screenshot provenance'); assert.match(runLocal, /webkit-snapshot[\s\S]*WKWebView internal snapshot[\s\S]*running WKWebView WebContent surface/, 'HTML report must visibly distinguish WKWebView internal snapshots from Peekaboo screen captures'); assert.match(proof, /stale_single_frame_recorders="\$\([\s\S]*ps -axo pid=,command=[\s\S]*-frames:v 1[\s\S]*!\/-framerate\/[\s\S]*Clearing stale one-frame ffmpeg capture process/, 'remote setup must clean stale one-frame ffmpeg captures without matching active scenario recorders'); From eea0eb61ced8f57c79bb72425b2d0e65433659f3 Mon Sep 17 00:00:00 2001 From: fkb032 <249513614+fkb032@users.noreply.github.com> Date: Fri, 8 May 2026 02:23:19 -0700 Subject: [PATCH 16/16] fix: defer peekaboo page-load native snapshots --- apps/native/src-tauri/src/main.rs | 11 +++++++- .../widget/controls/directory-picker.test.tsx | 1 - .../widget/promptinput/prompt-input.tsx | 3 +-- apps/native/src/lib/env.ts | 27 ++++++++++++------- .../peekaboo-workflow-contract-self-test.mjs | 7 +++-- 5 files changed, 34 insertions(+), 15 deletions(-) diff --git a/apps/native/src-tauri/src/main.rs b/apps/native/src-tauri/src/main.rs index f85ef175b..8cf0a1eb2 100644 --- a/apps/native/src-tauri/src/main.rs +++ b/apps/native/src-tauri/src/main.rs @@ -69,6 +69,10 @@ fn e2e_preload_native_capture_enabled() -> bool { cfg!(debug_assertions) && crate::e2e_runtime::enabled("NIXMAC_E2E_PRELOAD_NATIVE_CAPTURE") } +fn e2e_page_load_native_capture_enabled() -> bool { + cfg!(debug_assertions) && crate::e2e_runtime::enabled("NIXMAC_E2E_PAGE_LOAD_NATIVE_CAPTURE") +} + #[cfg(all(debug_assertions, target_os = "macos"))] static E2E_NATIVE_SNAPSHOT_COUNTER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); @@ -1985,7 +1989,8 @@ fn run_gui_mode( let main_webview_loaded = Arc::new(AtomicBool::new(false)); let main_webview_loaded_for_page_load = Arc::clone(&main_webview_loaded); let e2e_page_load_boot_probe = e2e_webview_watchdog; - let e2e_page_load_native_capture_probe = e2e_css_capture; + let e2e_page_load_native_capture_probe = + e2e_css_capture && e2e_page_load_native_capture_enabled(); let mut main_window_builder = WebviewWindowBuilder::new(app, "main", WebviewUrl::App("index.html".into())) @@ -2039,6 +2044,10 @@ fn run_gui_mode( "page-load-finished-plus-5s", Duration::from_secs(5), ); + } else if e2e_css_capture { + log::info!( + "NIXMAC_E2E_PAGE_LOAD_NATIVE_CAPTURE disabled; native WebView captures remain on-demand after shell readiness" + ); } } }); diff --git a/apps/native/src/components/widget/controls/directory-picker.test.tsx b/apps/native/src/components/widget/controls/directory-picker.test.tsx index 8ceb33220..d5ddf604e 100644 --- a/apps/native/src/components/widget/controls/directory-picker.test.tsx +++ b/apps/native/src/components/widget/controls/directory-picker.test.tsx @@ -5,7 +5,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useWidgetStore } from "@/stores/widget-store"; import type { SetDirResult } from "@/types/shared"; import { DirectoryPicker } from "@/components/widget/controls/directory-picker"; -import type { SetDirResult } from "@/types/shared"; // --------------------------------------------------------------------------- // Mocks diff --git a/apps/native/src/components/widget/promptinput/prompt-input.tsx b/apps/native/src/components/widget/promptinput/prompt-input.tsx index 195018d1e..383aa85b2 100644 --- a/apps/native/src/components/widget/promptinput/prompt-input.tsx +++ b/apps/native/src/components/widget/promptinput/prompt-input.tsx @@ -17,9 +17,8 @@ import { useEvolve } from "@/hooks/use-evolve"; import { getProviderConfigInvalidReason } from "@/lib/ai-provider-validation"; import { useWidgetStore } from "@/stores/widget-store"; import { darwinAPI } from "@/tauri-api"; -import { ArrowUpIcon, Plus } from "lucide-react"; +import { ArrowUpIcon } from "lucide-react"; import { useEffect, useState } from "react"; -import { DropdownMenu, DropdownMenuTrigger, DropdownMenuContent, DropdownMenuItem } from "@/components/ui/dropdown-menu"; import { Separator } from "@/components/ui/separator"; const MAX_CONTEXT_LENGTH = 1000; diff --git a/apps/native/src/lib/env.ts b/apps/native/src/lib/env.ts index 384339863..a8a479153 100644 --- a/apps/native/src/lib/env.ts +++ b/apps/native/src/lib/env.ts @@ -1,15 +1,24 @@ -import * as Schema from "effect/Schema"; +export type SettingsType = { + VITE_SERVER_URL?: string; + NIX_INSTALLED_OVERRIDE?: boolean; +}; -const Settings = Schema.Struct({ - VITE_SERVER_URL: Schema.optional(Schema.String), - NIX_INSTALLED_OVERRIDE: Schema.optional(Schema.BooleanFromString), -}); +function booleanFromEnv(value: unknown): boolean | undefined { + if (value == null || value === "") return undefined; + if (typeof value === "boolean") return value; + if (typeof value !== "string") return undefined; -export type SettingsType = Schema.Schema.Type; + const normalized = value.trim().toLowerCase(); + if (["1", "true", "yes", "on"].includes(normalized)) return true; + if (["0", "false", "no", "off"].includes(normalized)) return false; + return undefined; +} -export const settings: SettingsType = Schema.decodeUnknownSync(Settings)( - import.meta.env, -); +export const settings: SettingsType = { + VITE_SERVER_URL: + typeof import.meta.env?.VITE_SERVER_URL === "string" ? import.meta.env.VITE_SERVER_URL : undefined, + NIX_INSTALLED_OVERRIDE: booleanFromEnv(import.meta.env?.NIX_INSTALLED_OVERRIDE), +}; // Helper to resolve the public website URL used by the native/web apps. // Prefers the Vite env var `VITE_SERVER_URL` when available, otherwise diff --git a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs index 16c057c2d..88e069ba4 100644 --- a/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs +++ b/tools/computer-use-e2e/peekaboo-workflow-contract-self-test.mjs @@ -250,7 +250,10 @@ assert.match(nativeMain, /respondsToSelector: sel!\(_setWindowOcclusionDetection assert.match(nativeMain, /NIXMAC_E2E_CAPTURE native window diagnostics:[\s\S]*sharingTypeBefore=\{\}[\s\S]*sharingTypeAfter=\{\}[\s\S]*isOpaque[\s\S]*alphaValue[\s\S]*hasShadow[\s\S]*occlusionDetectionDisabled/, 'Native app must log native sharing and occlusion diagnostics for MacInCloud capture debugging'); assert.match(nativeMain, /fn e2e_request_native_webview_capture_probe[\s\S]*with_webview\(move \|webview\|[\s\S]*respondsToSelector: sel!\(drawsBackground\)[\s\S]*setNeedsDisplay: YES[\s\S]*displayIfNeeded[\s\S]*NIXMAC_E2E_CAPTURE webview diagnostics:[\s\S]*fn e2e_schedule_native_webview_capture_probe/, 'Native app must provide guarded WebView diagnostics and a best-effort AppKit display hint for MacInCloud black-capture diagnosis'); assert.match(nativeMain, /fn e2e_preload_native_capture_enabled\(\) -> bool \{\n\s+cfg!\(debug_assertions\) && crate::e2e_runtime::enabled\("NIXMAC_E2E_PRELOAD_NATIVE_CAPTURE"\)/, 'Native app must keep pre-load native capture diagnostics behind an explicit E2E-only opt-in gate'); -assert.match(nativeMain, /e2e_page_load_native_capture_probe[\s\S]*native-page-load-finished-plus-5s[\s\S]*e2e_schedule_native_webview_capture_probe/, 'Native app must schedule native WebView capture diagnostics after page-load'); +assert.match(nativeMain, /fn e2e_page_load_native_capture_enabled\(\) -> bool \{\n\s+cfg!\(debug_assertions\) && crate::e2e_runtime::enabled\("NIXMAC_E2E_PAGE_LOAD_NATIVE_CAPTURE"\)/, 'Native app must keep page-load native capture diagnostics behind an explicit E2E-only opt-in gate'); +assert.match(nativeMain, /let e2e_page_load_native_capture_probe =\s+e2e_css_capture && e2e_page_load_native_capture_enabled\(\);/, 'Native app must not schedule page-load native WebView captures by default on MacInCloud'); +assert.match(nativeMain, /if e2e_page_load_native_capture_probe \{[\s\S]*e2e_schedule_native_webview_capture_probe\([\s\S]*native-page-load-finished-plus-5s[\s\S]*e2e_schedule_native_webview_snapshot\([\s\S]*page-load-finished-plus-5s/, 'Native app must only schedule native WebView capture diagnostics after page-load when explicitly opted in'); +assert.match(nativeMain, /else if e2e_css_capture \{[\s\S]*NIXMAC_E2E_PAGE_LOAD_NATIVE_CAPTURE disabled/, 'Native app must log when page-load native captures are disabled by default'); assert.match(nativeMain, /if e2e_preload_native_capture_enabled\(\) \{[\s\S]*native-post-build-plus-2s[\s\S]*post-build-plus-2s[\s\S]*NIXMAC_E2E_PRELOAD_NATIVE_CAPTURE disabled/, 'Native app must not run pre-load native WebView captures by default on MacInCloud'); assert.match(cargoToml, /\[target\.'cfg\(target_os = "macos"\)'\.dependencies\][\s\S]*block = "0\.1"[\s\S]*objc = "0\.2"/, 'Native WKWebView snapshot completion blocks must declare the small macOS-only block crate beside objc/cocoa'); assert.match(nativeMain, /fn e2e_ns_error_summary[\s\S]*localizedDescription[\s\S]*domain[\s\S]*code[\s\S]*userInfo/, 'Native WKWebView snapshot failures must persist the actual NSError domain, code, description, and userInfo'); @@ -260,7 +263,7 @@ assert.match(nativeMain, /let tmp_output = output_path\.with_extension\("png\.tm assert.match(nativeMain, /fn e2e_native_snapshot_root_dir[\s\S]*NIXMAC_E2E_DIAGNOSTICS_DIR[\s\S]*native-webview-snapshots/, 'Native snapshots must reuse the existing E2E diagnostics directory'); assert.match(nativeMain, /fn e2e_start_native_webview_snapshot_request_poller[\s\S]*request_dir[\s\S]*requests[\s\S]*\.request[\s\S]*e2e_request_native_webview_snapshot/, 'Native app must poll diagnostics-dir snapshot requests so Scott’s shell driver can demand fresh native visual evidence'); assert.match(nativeMain, /let output_path = root\.join\(format!\("\{request_id\}\.png"\)\);[\s\S]*let status_path = root\.join\(format!\("\{request_id\}\.json"\)\);/, 'Native snapshot poller must use the shell request id verbatim so shell and Rust wait/write paths cannot desync on long labels'); -assert.match(nativeMain, /e2e_schedule_native_webview_snapshot[\s\S]*page-load-finished-plus-5s[\s\S]*e2e_start_native_webview_snapshot_request_poller/, 'Native app must take bounded page-load readiness snapshots and start the on-demand snapshot request poller only in E2E capture mode'); +assert.match(nativeMain, /e2e_start_native_webview_snapshot_request_poller[\s\S]*NIXMAC_E2E_PAGE_LOAD_NATIVE_CAPTURE disabled/, 'Native app must start the on-demand snapshot request poller while leaving page-load native snapshots opt-in'); assert.doesNotMatch(nativeMain, /setInterval|Duration::from_millis\((?:1000|2000|3000)\)[\s\S]*e2e_request_native_webview_snapshot/, 'Native app must not create noisy background interval snapshots while scenarios run'); assert.match(nativeSolidCaptureBranch, /NIXMAC_E2E_SOLID_CAPTURE enabled/, 'Native app must keep an explicit solid-capture branch for diagnostics'); assert.doesNotMatch(nativeSolidCaptureBranch, /background_color\(tauri::utils::config::Color\(10, 10, 10, 255\)\)/, 'Native app must not apply native background_color from the default solid-capture path');