From 91225e5fdb108ad0bb79e9be15e6a13331b6cb59 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Thu, 6 Aug 2026 12:19:45 +0800 Subject: [PATCH 01/12] feat(extension): add bsk hover and adjust vom to support --- apps/extension/src/content/record-capture.ts | 49 +++++- .../src/lib/__tests__/trace-reducer.test.ts | 24 +++ apps/extension/src/lib/describe-target.ts | 14 ++ apps/extension/src/lib/record-bridge.ts | 2 +- apps/extension/src/lib/trace-reducer.ts | 7 + .../src/tools/__tests__/dispatcher.test.ts | 78 ++++++++++ .../src/tools/__tests__/interaction.test.ts | 45 ++++++ .../src/tools/__tests__/observation.test.ts | 2 +- apps/extension/src/tools/dispatcher.ts | 143 +++++++++++++++--- apps/extension/src/tools/interaction.ts | 125 ++++++++++++++- apps/extension/src/tools/observation.ts | 5 +- apps/extension/src/transport/types.ts | 25 +++ crates/bsk-cli/skill/SKILL.md | 18 ++- crates/bsk-cli/src/cli/interaction.rs | 89 ++++++++++- crates/bsk-cli/src/cli/mod.rs | 5 +- crates/bsk-cli/src/daemon/ipc.rs | 1 + crates/bsk-cli/src/main.rs | 1 + crates/bsk-cli/tests/cli_parse.rs | 18 +++ .../schema/tool_hover_params.json | 72 +++++++++ .../schema/tool_hover_result.json | 113 ++++++++++++++ .../schema/tool_record_await_result.json | 40 +++++ .../schema/tool_record_stop_result.json | 40 +++++ crates/bsk-protocol/schema/trace.json | 40 +++++ crates/bsk-protocol/schema/trace_step.json | 40 +++++ crates/bsk-protocol/src/bin/dump-schema.rs | 2 + crates/bsk-protocol/src/method.rs | 7 +- crates/bsk-protocol/src/tools/interaction.rs | 64 ++++++++ crates/bsk-protocol/src/tools/record.rs | 5 + packages/vom/src/__tests__/render.test.ts | 2 +- packages/vom/src/render.ts | 4 +- skill/SKILL.md | 18 ++- 31 files changed, 1051 insertions(+), 47 deletions(-) create mode 100644 crates/bsk-protocol/schema/tool_hover_params.json create mode 100644 crates/bsk-protocol/schema/tool_hover_result.json diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index 12bf6af..8c9d5d0 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -1,4 +1,9 @@ -import { describeEventTarget, describeTarget, type TargetDescriptor } from "@/lib/describe-target"; +import { + describeEventTarget, + describeTarget, + resolveHoverElement, + type TargetDescriptor, +} from "@/lib/describe-target"; import { isRecordCancelMessage, isRecordStartMessage, @@ -49,8 +54,16 @@ interface FillSession { lastValue: string; } +interface PendingHover { + element: Element; + target: TargetDescriptor; + recordedAt: number; +} + type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement; +const HOVER_BEFORE_CLICK_MAX_MS = 10_000; + function eventTarget(event: Event): EventTarget | null { return event.composedPath()[0] ?? event.target; } @@ -122,6 +135,14 @@ function fillableValue(el: FillableElement): string { return el.textContent ?? ""; } +function hoverTargetFromEvent(target: EventTarget | null): PendingHover | null { + if (!(target instanceof Element)) return null; + const element = resolveHoverElement(target); + if (!element) return null; + const desc = describeTarget(element); + return { element, target: desc, recordedAt: Date.now() }; +} + /** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */ function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean { if (!session || !(target instanceof Element)) return false; @@ -172,6 +193,7 @@ export function startRecordCapture( let composing = false; let lastUrl = location.href; let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined; + let pendingHover: PendingHover | null = null; let generatedControlClick: Element | null = null; let navigationActionPending = false; let navigationActionVersion = 0; @@ -249,6 +271,28 @@ export function startRecordCapture( }); }; + const emitPendingHoverBeforeClick = (clickTarget: EventTarget | null) => { + if (!pendingHover || !(clickTarget instanceof Element)) return; + if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) { + pendingHover = null; + return; + } + if (pendingHover.element.contains(clickTarget)) return; + emitStep({ + op: "hover", + target: pendingHover.target, + }); + pendingHover = null; + }; + + const onMouseOver = (event: MouseEvent) => { + const target = eventTarget(event); + if (isOverlayTarget(target)) return; + const hover = hoverTargetFromEvent(target); + if (!hover || hover.element === pendingHover?.element) return; + pendingHover = hover; + }; + const onClick = (event: MouseEvent) => { if (event.button !== 0) return; const target = eventTarget(event); @@ -304,6 +348,7 @@ export function startRecordCapture( commitFillSession(); if (target instanceof Element && target.closest("select")) return; + emitPendingHoverBeforeClick(target); emitClick(event); }; @@ -408,6 +453,7 @@ export function startRecordCapture( }; document.addEventListener("click", onClick, true); + document.addEventListener("mouseover", onMouseOver, true); document.addEventListener("focusin", onFocusIn, true); document.addEventListener("focusout", onFocusOut, true); document.addEventListener("input", onInput, true); @@ -438,6 +484,7 @@ export function startRecordCapture( dispose() { commitFillSession(); document.removeEventListener("click", onClick, true); + document.removeEventListener("mouseover", onMouseOver, true); document.removeEventListener("focusin", onFocusIn, true); document.removeEventListener("focusout", onFocusOut, true); document.removeEventListener("input", onInput, true); diff --git a/apps/extension/src/lib/__tests__/trace-reducer.test.ts b/apps/extension/src/lib/__tests__/trace-reducer.test.ts index 777a86c..8cc773c 100644 --- a/apps/extension/src/lib/__tests__/trace-reducer.test.ts +++ b/apps/extension/src/lib/__tests__/trace-reducer.test.ts @@ -130,6 +130,30 @@ describe("reduceTraceSteps", () => { }); }); + it("keeps hover steps before menu clicks", () => { + const { steps } = reduceTraceSteps( + [ + { + op: "hover", + target: { tag: "span", role: "button", name: "Account" }, + page_url: "https://example.com/app", + }, + { + op: "click", + target: { tag: "a", role: "link", name: "Profile" }, + page_url: "https://example.com/app", + }, + ], + "https://example.com/app", + ); + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + page: "p1", + target: { name: "Account" }, + }); + }); + it("resolveTraceStartUrl prefers explicit start URL", () => { expect( resolveTraceStartUrl( diff --git a/apps/extension/src/lib/describe-target.ts b/apps/extension/src/lib/describe-target.ts index 7d3898a..0a1fcb2 100644 --- a/apps/extension/src/lib/describe-target.ts +++ b/apps/extension/src/lib/describe-target.ts @@ -228,6 +228,20 @@ export function resolveClickableElement(target: Element): Element | null { return null; } +export function resolveHoverElement(target: Element): Element | null { + const clickable = resolveClickableElement(target); + if (clickable) return clickable; + + let node: Element | null = target; + let depth = 0; + while (node && node !== document.body && node !== document.documentElement && depth < 8) { + if (looksClickable(node)) return node; + node = node.parentElement; + depth += 1; + } + return null; +} + /** * Teachable clicks: the LLM must get a label it can later find via snapshot * (visible name), or at least a form `name_attr` for checkbox/radio. diff --git a/apps/extension/src/lib/record-bridge.ts b/apps/extension/src/lib/record-bridge.ts index cb5d446..cb7eb88 100644 --- a/apps/extension/src/lib/record-bridge.ts +++ b/apps/extension/src/lib/record-bridge.ts @@ -33,7 +33,7 @@ export interface RecordStartMessage { } export interface RecordStepPayload { - op: "click" | "fill" | "press" | "select" | "navigate"; + op: "click" | "hover" | "fill" | "press" | "select" | "navigate"; target?: TargetDescriptor; value?: string; key?: string; diff --git a/apps/extension/src/lib/trace-reducer.ts b/apps/extension/src/lib/trace-reducer.ts index e3572ee..6da0c3d 100644 --- a/apps/extension/src/lib/trace-reducer.ts +++ b/apps/extension/src/lib/trace-reducer.ts @@ -139,6 +139,13 @@ function toV2Step( }, effectForNavigation(step.navigated_to, urlToId), ); + case "hover": + return { + op: "hover", + id, + page, + target: step.target, + }; case "fill": return { op: "fill", diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 060b30a..1b99653 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -311,6 +311,84 @@ describe("ToolDispatcher", () => { expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11"); }); + it("reasserts remembered hover before follow-up work and releases only after actions", async () => { + vi.stubGlobal("chrome", { + tabs: { + sendMessage: vi.fn(async () => undefined), + }, + }); + const { transport } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 1), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + const order: string[] = []; + const cdp = { + send: vi.fn(async () => { + order.push("hover"); + return {}; + }), + detachSession: vi.fn(async () => {}), + ensureNetworkCapture: vi.fn(async () => {}), + networkEntriesSince: vi.fn(() => ({ + tab_id: 7, + entries: [], + next_since: 0, + truncated: false, + })), + setDeviceMetricsOverride: vi.fn(async () => {}), + clearDeviceMetricsOverride: vi.fn(async () => {}), + setUserAgentOverride: vi.fn(async () => {}), + setTouchEmulationEnabled: vi.fn(async () => {}), + }; + const dispatcher = new ToolDispatcher({ transport, sessions, cdp }); + + (dispatcher as unknown as { rememberHover: (result: unknown) => unknown }).rememberHover({ + tab_id: 7, + x: 10, + y: 20, + }); + const helpers = dispatcher as unknown as { + withHoverReassert: ( + work: () => Promise, + options?: { releaseAfter?: boolean }, + ) => Promise; + setHoverBypass: (tabId: number, enabled: boolean) => Promise; + }; + + await helpers.withHoverReassert(async () => { + order.push("observe"); + return {}; + }); + expect(order).toEqual(["hover", "observe"]); + expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + + await helpers.setHoverBypass(7, true); + await helpers.withHoverReassert( + async () => { + order.push("click"); + return {}; + }, + { releaseAfter: true }, + ); + + expect(cdp.send).toHaveBeenLastCalledWith(7, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: 10, + y: 20, + }); + expect(chrome.tabs.sendMessage).toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + }); + it("disconnects the transport when send() fails so keepalive can rebuild", async () => { const { transport, deliver } = fakeTransport(); const sessions = new SessionManager({ diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index de56af4..6f8eeed 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -4,6 +4,7 @@ import type { CdpRunner } from "@/tools/shared"; import { handleClick, handleFill, + handleHover, handlePress, handleSelect, modifiersBitfield, @@ -435,6 +436,50 @@ describe("handleClick", () => { }); }); +describe("handleHover", () => { + it("keeps overlay bypass enabled after a successful hover when requested", async () => { + const bypassOverlay = vi.fn().mockResolvedValue(undefined); + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), + "Runtime.evaluate": (params: unknown) => { + const expr = String((params as { expression?: string })?.expression ?? ""); + if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) { + return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } }; + } + if (expr.includes("hitIndex")) { + return { + result: { + value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 }, + }, + }; + } + throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`); + }, + "Input.dispatchMouseEvent": () => ({}), + }); + + const res = await handleHover( + sm, + { session_id: "aa11", ref: "@e3", settle_ms: 0 }, + { + cdp: fake.cdp, + tabsApi: fake.tabsApi, + bypassOverlay, + keepOverlayBypassAfterHover: true, + }, + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 }); + expect(bypassOverlay).toHaveBeenCalledTimes(1); + expect(bypassOverlay).toHaveBeenCalledWith(4, true); + }); +}); + describe("handleFill", () => { it("returns not_found for unknown ref", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index d6cd621..643ad0b 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -1480,7 +1480,7 @@ describe("buildVomScene", () => { expect(scene.surfaces).toEqual([ { triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, ]); - expect(renderVom(scene).text).toContain('@e1 button "Products" [hover: Shoes | Bags]'); + expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]'); }); it("enriches names and active scope signals from AX properties", () => { diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index dd419fb..308db59 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -8,6 +8,8 @@ import type { EvaluateParams, FillParams, GetHtmlParams, + HoverParams, + HoverResult, NavigateBackParams, NavigateForwardParams, NavigateParams, @@ -33,7 +35,7 @@ import { handleConsole } from "./console"; import { type EmulateCdpRunner, handleEmulate } from "./emulate"; import { handleEvaluate } from "./evaluate"; import { handleRequestHelp } from "./human-loop"; -import { handleClick, handleFill, handlePress, handleSelect } from "./interaction"; +import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction"; import { handleNavigate, handleNavigateBack, @@ -126,6 +128,8 @@ export class ToolDispatcher { private readonly approveBorrow?: BorrowConfirmationApprover; private readonly helpNotificationCopy?: () => { title: string; body: string }; private subscription: { dispose(): void } | null = null; + private readonly hoverBypassTabs = new Set(); + private readonly hoverLatches = new Map(); /** * Per-rpc-id `AbortController` registry. Populated inside * [`dispatch`] before we await the tool handler and torn down in @@ -271,10 +275,12 @@ export class ToolDispatcher { switch (req.method) { case "tool.session_start": return handleSessionStart(this.sessions, req.params as SessionStartParams); - case "tool.session_stop": + case "tool.session_stop": { + await this.releaseHoverLatch(); return handleSessionStop(this.sessions, req.params as SessionStopParams, { cdp: this.cdp, }); + } case "tool.tab_list": return handleTabList(this.sessions, req.params as TabListParams); case "tool.tab_create": @@ -319,16 +325,26 @@ export class ToolDispatcher { this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined, ); case "tool.snapshot": - return handleSnapshot( - this.sessions, - req.params as SnapshotParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined, + return this.withHoverReassert(() => + handleSnapshot( + this.sessions, + req.params as SnapshotParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined, + ), ); case "tool.observe": - return handleObserve( - this.sessions, - req.params as ObserveParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined, + return this.withHoverReassert(() => + handleObserve( + this.sessions, + req.params as ObserveParams, + this.cdp + ? { + cdp: this.cdp, + tabsApi: chromeTabsCaptureApi, + conditionalSurfaceProbe: !this.hasHoverLatch(), + } + : undefined, + ), ); case "tool.get_html": return handleGetHtml( @@ -361,27 +377,38 @@ export class ToolDispatcher { this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, ); case "tool.click": - return handleClick( + return this.withHoverReassert( + () => + handleClick( + this.sessions, + req.params as ClickParams, + this.cdp + ? { + cdp: this.cdp, + tabsApi: chromeTabsApi, + signal, + bypassOverlay, + } + : undefined, + ), + { releaseAfter: true }, + ); + case "tool.hover": { + const result = await handleHover( this.sessions, - req.params as ClickParams, + req.params as HoverParams, this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal, - bypassOverlay: async (tabId, enabled) => { - try { - await chrome.tabs.sendMessage(tabId, { - type: OVERLAY_AUTOMATION_BYPASS, - enabled, - }); - } catch { - // Content script may be unavailable on restricted pages. - } - }, + bypassOverlay: (tabId, enabled) => this.setHoverBypass(tabId, enabled), + keepOverlayBypassAfterHover: true, } : undefined, ); + return this.rememberHover(result); + } case "tool.fill": return handleFill( this.sessions, @@ -471,6 +498,66 @@ export class ToolDispatcher { } satisfies RpcError; } } + + private async setHoverBypass(tabId: number, enabled: boolean): Promise { + if (enabled) { + if (this.hoverBypassTabs.has(tabId)) return; + await bypassOverlay(tabId, true); + this.hoverBypassTabs.add(tabId); + } else { + if (!this.hoverBypassTabs.has(tabId)) return; + await bypassOverlay(tabId, false); + this.hoverBypassTabs.delete(tabId); + } + } + + private rememberHover(result: HoverResult | RpcError): HoverResult | RpcError { + if (!isRpcError(result)) { + this.hoverLatches.set(result.tab_id, { x: result.x, y: result.y }); + } + return result; + } + + private hasHoverLatch(): boolean { + return this.hoverLatches.size > 0; + } + + private async withHoverReassert( + work: () => Promise, + options: { releaseAfter?: boolean } = {}, + ): Promise { + await this.reassertHover(); + try { + return await work(); + } finally { + if (options.releaseAfter) { + await this.releaseHoverLatch(); + } + } + } + + private async reassertHover(): Promise { + if (!this.cdp) return; + await Promise.all( + [...this.hoverLatches.entries()].map(([tabId, point]) => + this.cdp!.send(tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: point.x, + y: point.y, + }).catch((err) => { + console.debug("[bsk dispatcher] hover reassert failed", err); + this.hoverLatches.delete(tabId); + }), + ), + ); + } + + private async releaseHoverLatch(): Promise { + const tabs = new Set([...this.hoverBypassTabs, ...this.hoverLatches.keys()]); + this.hoverBypassTabs.clear(); + this.hoverLatches.clear(); + await Promise.all([...tabs].map((tabId) => bypassOverlay(tabId, false))); + } } function isRpcError(v: unknown): v is RpcError { @@ -497,6 +584,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.navigate_forward": case "tool.reload": case "tool.click": + case "tool.hover": case "tool.fill": case "tool.press": case "tool.select": @@ -512,6 +600,17 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { } } +async function bypassOverlay(tabId: number, enabled: boolean): Promise { + try { + await chrome.tabs.sendMessage(tabId, { + type: OVERLAY_AUTOMATION_BYPASS, + enabled, + }); + } catch { + // Content script may be unavailable on restricted pages. + } +} + /** * Resolves never; rejects with `AbortLikeError` as soon as the signal * fires (or immediately if it is already aborted). Used by the diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 87decea..ee66dac 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -17,6 +17,8 @@ import type { ClickResult, FillParams, FillResult, + HoverParams, + HoverResult, KeyModifier, MouseButton, PressParams, @@ -53,9 +55,12 @@ export interface InteractionDeps { defaultTimeoutMs?: number; /** Temporarily disable overlay click blocker during CDP automation. */ bypassOverlay?: (tabId: number, enabled: boolean) => Promise; + /** Keep hover hit-testing active for the caller's next observation/action. */ + keepOverlayBypassAfterHover?: boolean; } const DEFAULT_TIMEOUT_MS = 30_000; +const DEFAULT_HOVER_SETTLE_MS = 200; let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null; function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } { @@ -100,6 +105,27 @@ function throwIfAborted(signal: AbortSignal | undefined): RpcError | null { return null; } +function isAbortLikeError(err: unknown): boolean { + return err instanceof DOMException && err.name === "AbortError"; +} + +async function wait(ms: number, signal?: AbortSignal): Promise { + if (signal?.aborted) throw new DOMException("aborted", "AbortError"); + await new Promise((resolve, reject) => { + const cleanup = () => signal?.removeEventListener("abort", onAbort); + const timer = setTimeout(() => { + cleanup(); + resolve(); + }, ms); + const onAbort = () => { + clearTimeout(timer); + cleanup(); + reject(new DOMException("aborted", "AbortError")); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + /** * Resolve `{ref?, selector?}` into a `backendNodeId`. Returns an * `RpcError` if the caller supplied neither (or both), or if neither @@ -286,7 +312,104 @@ export async function handleClick( message: err instanceof Error ? err.message : String(err), }; } finally { - if (automationBypassEnabled && deps.bypassOverlay) { + if (automationBypassEnabled && deps.bypassOverlay && !deps.keepOverlayBypassAfterHover) { + try { + await deps.bypassOverlay(target.tabId, false); + } catch (err) { + console.debug("[bsk interaction] overlay bypass disable failed", err); + } + } + } + + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: node.usedRef, + used_selector: node.usedSelector, + x: centre.x, + y: centre.y, + }); +} + +// --------------------------------------------------------------------------- +// tool.hover +// --------------------------------------------------------------------------- + +export async function handleHover( + manager: SessionManager, + params: HoverParams, + deps: InteractionDeps = getDefaultDeps(), +): Promise { + const ctxOrErr = lookupSession(manager, params, "hover"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + const aborted = throwIfAborted(deps.signal); + if (aborted) return aborted; + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "hover"); + if (denied) return denied; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + + const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover"); + if (isRpcError(node)) return node; + + try { + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId); + if (scrollErr) return scrollErr; + } catch (err) { + return { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + }; + } + + const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId); + if (isRpcError(centre)) return centre; + + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "hover aborted" }; + } + + const modifiers = modifiersBitfield(params.modifiers); + const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y); + let automationBypassEnabled = false; + let hoverCompleted = false; + if (overlayBlocking && deps.bypassOverlay) { + try { + await deps.bypassOverlay(target.tabId, true); + automationBypassEnabled = true; + } catch (err) { + console.debug("[bsk interaction] overlay bypass enable failed", err); + } + } + + try { + await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: centre.x, + y: centre.y, + modifiers, + }); + const settleMs = params.settle_ms ?? DEFAULT_HOVER_SETTLE_MS; + if (settleMs > 0) { + await wait(settleMs, deps.signal); + } + hoverCompleted = true; + } catch (err) { + if (isAbortLikeError(err)) { + return { code: "cancelled", message: "hover aborted" }; + } + return { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + }; + } finally { + if ( + automationBypassEnabled && + deps.bypassOverlay && + (!deps.keepOverlayBypassAfterHover || !hoverCompleted) + ) { try { await deps.bypassOverlay(target.tabId, false); } catch (err) { diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 48fe799..11ad305 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -1072,6 +1072,7 @@ export interface SnapshotDeps { get(tabId: number): Promise; query(q: chrome.tabs.QueryInfo): Promise; }; + conditionalSurfaceProbe?: boolean; } let defaultDeps: SnapshotDeps | null = null; @@ -1258,7 +1259,9 @@ async function handleVomObservation( {}, ); const axNodes = result.nodes ?? []; - const captured = await captureForVom(deps.cdp, target.tabId, conditionalSurfaceProbe); + const effectiveConditionalSurfaceProbe = + deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe; + const captured = await captureForVom(deps.cdp, target.tabId, effectiveConditionalSurfaceProbe); const scene = buildVomScene(axNodes, captured, { pageUrl: target.url }); const rendered = renderVom(scene, { maxDepth: params.max_depth, diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index a335747..c3ebdff 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -410,6 +410,25 @@ export interface ClickResult { dialogs?: JavaScriptDialogInfo[]; } +export interface HoverParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + modifiers?: KeyModifier[]; + settle_ms?: number; + timeout_ms?: number; +} + +export interface HoverResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + x: number; + y: number; + dialogs?: JavaScriptDialogInfo[]; +} + export interface FillParams { session_id: string; value: string; @@ -668,6 +687,11 @@ export type DraftTraceStep = navigated_to?: string; page_url?: string; } + | { + op: "hover"; + target: TargetDescriptor; + page_url?: string; + } | { op: "fill"; target: TargetDescriptor; @@ -701,6 +725,7 @@ export type DraftTraceStep = export type Step = | ({ op: "navigate" } & StepCommon & { to: string }) | ({ op: "click" } & StepCommon & { target: TargetDescriptor }) + | ({ op: "hover" } & StepCommon & { target: TargetDescriptor }) | ({ op: "fill" } & StepCommon & { target: TargetDescriptor; value: string; diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 3c36961..9a1b6a8 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -67,22 +67,25 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe ``` bsk navigate --session -bsk snapshot --session → aria tree with @e1, @e2, … refs -bsk observe --session → semantic VOM view; may reveal hover/focus surfaces -bsk click @e3 --session → or bsk fill, bsk select, bsk press -bsk snapshot --session → again after navigation / DOM change +bsk observe --session → primary semantic VOM view; reveals hover/focus surfaces +bsk snapshot --session → static aria tree fallback when VOM is insufficient +bsk hover @e3 --session → reveal hover-triggered menus before re-observing/clicking +bsk click @e4 --session → or bsk fill, bsk select, bsk press +bsk observe --session → again after navigation / DOM change ``` **Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page. Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`). +When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover --session `, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action. + ## Observation priority -Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question: +Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question: -1. `bsk snapshot` — strict static page understanding and interaction planning -2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery +1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery +2. `bsk snapshot` — strict static accessibility tree fallback 3. `bsk get-html` — when hidden DOM, metadata, or markup details are required 4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab. @@ -186,6 +189,7 @@ bsk emulate --session --off | Command | Summary | |---------|---------| | `bsk click ` | Click element (`--button`, `--click-count`, `--modifiers`) | +| `bsk hover ` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) | | `bsk fill --value ` | Clear and type into input | | `bsk select --value ` | Set `` option(s) by `value` (repeat `--value` for multi-select) | | `bsk press ` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) | From dbc4845c0d47e0745f3b5fe51d0a5eb44b178930 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Thu, 6 Aug 2026 15:30:11 +0800 Subject: [PATCH 02/12] feat(extension): vom hover description bug fix --- .../src/tools/__tests__/observation.test.ts | 196 ++++++- apps/extension/src/tools/dispatcher.ts | 1 + apps/extension/src/tools/observation.ts | 135 ++++- .../src/tools/vom/__tests__/capture.test.ts | 282 +++++++++- apps/extension/src/tools/vom/capture.ts | 485 ++++++++++++++---- apps/extension/src/transport/types.ts | 17 +- crates/bsk-cli/src/cli/observe.rs | 5 + crates/bsk-cli/tests/tools_ipc.rs | 2 + .../schema/tool_observe_params.json | 5 + .../schema/tool_observe_result.json | 77 +++ crates/bsk-protocol/src/tools/observation.rs | 29 ++ packages/vom/src/index.ts | 2 +- packages/vom/src/render.ts | 20 +- 13 files changed, 1081 insertions(+), 175 deletions(-) diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 643ad0b..9d9e003 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -1430,7 +1430,7 @@ describe("buildVomScene", () => { expect(renderVom(scene).text).toContain("[§ active: Reviews (12)]"); }); - it("maps hover surface probes to matching node labels", () => { + it("maps hover surface probes to matching backend node ids", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", @@ -1451,7 +1451,7 @@ describe("buildVomScene", () => { iframeNodes: new Map(), excludedBackendNodeIds: new Set(), surfaceProbes: [ - { triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] }, + { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, ], nodes: [ { @@ -1483,6 +1483,195 @@ describe("buildVomScene", () => { expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]'); }); + it("maps hover surface probes to rendered descendants in the same trigger subtree", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 10, + childIds: ["2"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "image" }, + backendDOMNodeId: 21, + }, + ]; + const scene = buildVomScene(axNodes, { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + surfaceProbes: [ + { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] }, + ], + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "div", + attrs: { class: "tg-avatar" }, + rect: { x: 900, y: 10, w: 30, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 21, + parentBackendNodeId: 20, + tag: "div", + attrs: { class: "tg-avatar__inner" }, + rect: { x: 902, y: 12, w: 26, h: 26 }, + paintOrder: 2, + position: "static", + pointerEvents: "auto", + }, + ], + }); + + expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]'); + }); + + it("maps hover surface probes to DOM-recovered custom controls", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 10, + childIds: ["2"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "img" }, + name: { type: "computedString", value: "image" }, + backendDOMNodeId: 21, + }, + ]; + const scene = buildVomScene(axNodes, { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + surfaceProbes: [ + { + triggerBackendNodeId: 20, + triggerPoint: { x: 915, y: 25 }, + triggerAction: "hover", + subItems: ["My profile", "Sign out"], + }, + ], + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + cursor: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "div", + attrs: { class: "tg-avatar" }, + rect: { x: 900, y: 10, w: 30, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + cursor: "pointer", + }, + { + backendNodeId: 21, + parentBackendNodeId: 20, + tag: "div", + attrs: { class: "tg-avatar__inner" }, + rect: { x: 902, y: 12, w: 26, h: 26 }, + paintOrder: 2, + position: "static", + pointerEvents: "auto", + cursor: "pointer", + }, + ], + }); + + expect(scene.surfaces).toEqual([ + { triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] }, + ]); + expect(renderVom(scene).text).toContain( + '@e1 button "image" [hover first: My profile | Sign out]', + ); + }); + + it("does not attach hover probes to distant geometry matches", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 10, + childIds: ["2"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "link" }, + name: { type: "computedString", value: "0" }, + backendDOMNodeId: 42, + }, + ]; + const scene = buildVomScene(axNodes, { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + surfaceProbes: [ + { + triggerBackendNodeId: 999, + triggerPoint: { x: 900, y: 20 }, + triggerAction: "hover", + subItems: ["My profile"], + }, + ], + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 42, + parentBackendNodeId: 10, + tag: "a", + attrs: {}, + rect: { x: 860, y: 120, w: 40, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + ], + }); + + expect(scene.surfaces).toBeUndefined(); + expect(renderVom(scene).text).not.toContain("[hover first:"); + }); + it("enriches names and active scope signals from AX properties", () => { const axNodes: CdpAxNode[] = [ { @@ -1953,7 +2142,7 @@ describe("handleSnapshot", () => { }); const res = await handleObserve( sm, - { session_id: "aa11" }, + { session_id: "aa11", debug_surfaces: true }, { cdp: { send: send as unknown as ( @@ -1974,6 +2163,7 @@ describe("handleSnapshot", () => { ); if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res.debug).toEqual({ surface_probes: [] }); expect(send).toHaveBeenCalledWith( 4, "Runtime.evaluate", diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 308db59..c382d96 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -342,6 +342,7 @@ export class ToolDispatcher { cdp: this.cdp, tabsApi: chromeTabsCaptureApi, conditionalSurfaceProbe: !this.hasHoverLatch(), + hoverProbeBypassOverlay: bypassOverlay, } : undefined, ), diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 11ad305..59d39ff 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -5,7 +5,9 @@ import { type ActiveScopeBlock, + applyVomInteractionRecovery, type CondSurface, + isVomReferenceNode, renderVom, type VomNode, type VomScene, @@ -908,39 +910,26 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A return blocks; } -function labelForSurfaceMatch(node: VomNode): string { - return ( - cleanAttr(node.name) ?? - cleanAttr(node.text) ?? - cleanAttr(node.attrs?.["aria-label"]) ?? - cleanAttr(node.attrs?.title) ?? - "" - ); -} - function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] { const probes = captured.surfaceProbes ?? []; if (probes.length === 0) return []; const surfaces: CondSurface[] = []; - const nodesWithLabels = nodes - .map((node) => ({ node, key: normalizeProbeKey(labelForSurfaceMatch(node)) })) - .filter((entry) => entry.key.length > 0); + const recoveredNodes = applyVomInteractionRecovery(nodes); + const signals = capturedOnlySignals(captured.nodes); const used = new Set(); for (const probe of probes) { - const probeKey = normalizeProbeKey(probe.triggerLabel); - if (!probeKey || probe.subItems.length === 0) continue; - const match = - nodesWithLabels.find((entry) => entry.key === probeKey && !used.has(entry.node.id)) ?? - nodesWithLabels.find( - (entry) => - !used.has(entry.node.id) && - (probeKey.startsWith(entry.key) || entry.key.startsWith(probeKey)), - ); + if (probe.subItems.length === 0 || used.has(probe.triggerBackendNodeId)) continue; + const match = findSurfaceTriggerNode( + probe.triggerBackendNodeId, + recoveredNodes, + signals, + probe.triggerPoint, + ); if (!match) continue; - used.add(match.node.id); + used.add(match.id); surfaces.push({ - triggerId: match.node.id, + triggerId: match.id, triggerAction: probe.triggerAction, subItems: probe.subItems, }); @@ -948,6 +937,77 @@ function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel) return surfaces; } +function findSurfaceTriggerNode( + triggerBackendNodeId: number, + nodes: VomNode[], + signals: VomNodeDomSignals, + triggerPoint?: { x: number; y: number }, +): VomNode | undefined { + const renderedById = new Map(nodes.map((node) => [node.id, node])); + const exact = renderedById.get(triggerBackendNodeId); + if (exact && isSurfaceAttachableNode(exact)) return exact; + + const domDescendant = nodes.find( + (node) => + isSurfaceAttachableNode(node) && + (node.domParentId === triggerBackendNodeId || + node.domAncestorIds?.includes(triggerBackendNodeId) === true), + ); + if (domDescendant) return domDescendant; + + const queue = [...(signals.childrenByParentId.get(triggerBackendNodeId) ?? [])]; + while (queue.length > 0) { + const child = queue.shift() as CapturedNode; + const rendered = renderedById.get(child.backendNodeId); + if (rendered && isSurfaceAttachableNode(rendered)) return rendered; + queue.push(...(signals.childrenByParentId.get(child.backendNodeId) ?? [])); + } + + let current = signals.capturedByBackendId.get(triggerBackendNodeId); + while (current?.parentBackendNodeId !== null && current?.parentBackendNodeId !== undefined) { + const parent = renderedById.get(current.parentBackendNodeId); + if (parent && isSurfaceAttachableNode(parent)) return parent; + current = signals.capturedByBackendId.get(current.parentBackendNodeId); + } + + if (triggerPoint) { + return findSurfaceNodeByPoint(triggerPoint, nodes, signals); + } + + return undefined; +} + +function isSurfaceAttachableNode(node: VomNode): boolean { + return isVomReferenceNode(node); +} + +function findSurfaceNodeByPoint( + point: { x: number; y: number }, + nodes: VomNode[], + signals: VomNodeDomSignals, +): VomNode | undefined { + let best: { node: VomNode; score: number } | undefined; + for (const node of nodes) { + if (!isSurfaceAttachableNode(node)) continue; + const rect = node.rect ?? signals.capturedByBackendId.get(node.id)?.rect; + if (!rect) continue; + const contains = + point.x >= rect.x && + point.x <= rect.x + rect.w && + point.y >= rect.y && + point.y <= rect.y + rect.h; + const centerX = rect.x + rect.w / 2; + const centerY = rect.y + rect.h / 2; + const distance = Math.hypot(point.x - centerX, point.y - centerY); + if (!contains && distance > 40) continue; + const score = contains ? distance : distance + 1_000; + if (!best || score < best.score) { + best = { node, score }; + } + } + return best?.node; +} + function axNodeInOverlaySubtree( axNode: CdpAxNode, axById: Map, @@ -1073,6 +1133,7 @@ export interface SnapshotDeps { query(q: chrome.tabs.QueryInfo): Promise; }; conditionalSurfaceProbe?: boolean; + hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise; } let defaultDeps: SnapshotDeps | null = null; @@ -1218,9 +1279,13 @@ async function captureForVom( cdp: CdpRunner, tabId: number, conditionalSurfaceProbe: boolean, + hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise, ): Promise { try { - return await captureViewModel(cdp, tabId, { conditionalSurfaceProbe }); + return await captureViewModel(cdp, tabId, { + conditionalSurfaceProbe, + hoverProbeBypassOverlay, + }); } catch { return fallbackCapturedViewModel(cdp, tabId); } @@ -1261,7 +1326,12 @@ async function handleVomObservation( const axNodes = result.nodes ?? []; const effectiveConditionalSurfaceProbe = deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe; - const captured = await captureForVom(deps.cdp, target.tabId, effectiveConditionalSurfaceProbe); + const captured = await captureForVom( + deps.cdp, + target.tabId, + effectiveConditionalSurfaceProbe, + deps.hoverProbeBypassOverlay, + ); const scene = buildVomScene(axNodes, captured, { pageUrl: target.url }); const rendered = renderVom(scene, { maxDepth: params.max_depth, @@ -1278,6 +1348,19 @@ async function handleVomObservation( ref_count: rendered.refs.length, tab_id: target.tabId, truncated: rendered.truncated, + ...(toolName === "observe" && (params as ObserveParams).debug_surfaces + ? { + debug: { + surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({ + trigger_backend_node_id: probe.triggerBackendNodeId, + ...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}), + trigger_action: probe.triggerAction, + sub_items: probe.subItems, + ...(probe.confidence ? { confidence: probe.confidence } : {}), + })), + }, + } + : {}), }); } catch (err) { return { diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index 2fe1f2e..086e216 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -51,6 +51,156 @@ function fakeSnapshotReply() { }; } +function hoverTriggerSnapshotReply() { + const S = [ + "html", + "body", + "button", + "class", + "user-avatar", + "position", + "static", + "pointer-events", + "auto", + "cursor", + "pointer", + ]; + const i = (s: string) => S.indexOf(s); + return { + strings: S, + documents: [ + { + nodes: { + parentIndex: [-1, 0, 1], + nodeType: [1, 1, 1], + nodeName: [i("html"), i("body"), i("button")], + backendNodeId: [10, 11, 12], + attributes: [[], [], [i("class"), i("user-avatar")]], + }, + layout: { + nodeIndex: [1, 2], + styles: [ + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + ], + bounds: [ + [0, 0, 1000, 800], + [940, 16, 32, 32], + ], + paintOrders: [0, 1], + }, + }, + ], + }; +} + +function twoHoverTriggerSnapshotReply() { + const S = [ + "html", + "body", + "button", + "class", + "user-avatar", + "secondary-trigger", + "position", + "static", + "pointer-events", + "auto", + "cursor", + "pointer", + ]; + const i = (s: string) => S.indexOf(s); + return { + strings: S, + documents: [ + { + nodes: { + parentIndex: [-1, 0, 1, 1], + nodeType: [1, 1, 1, 1], + nodeName: [i("html"), i("body"), i("button"), i("button")], + backendNodeId: [10, 11, 12, 13], + attributes: [ + [], + [], + [i("class"), i("user-avatar")], + [i("class"), i("secondary-trigger")], + ], + }, + layout: { + nodeIndex: [1, 2, 3], + styles: [ + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + ], + bounds: [ + [0, 0, 1000, 800], + [940, 16, 32, 32], + [100, 120, 32, 32], + ], + paintOrders: [0, 1, 2], + }, + }, + ], + }; +} + +function nestedHoverTriggerSnapshotReply() { + const S = [ + "html", + "body", + "div", + "img", + "class", + "tg-avatar", + "tg-avatar__inner", + "tg-avatar__image", + "position", + "static", + "pointer-events", + "auto", + "cursor", + "pointer", + ]; + const i = (s: string) => S.indexOf(s); + return { + strings: S, + documents: [ + { + nodes: { + parentIndex: [-1, 0, 1, 2, 3], + nodeType: [1, 1, 1, 1, 1], + nodeName: [i("html"), i("body"), i("div"), i("div"), i("img")], + backendNodeId: [10, 11, 12, 13, 14], + attributes: [ + [], + [], + [i("class"), i("tg-avatar")], + [i("class"), i("tg-avatar__inner")], + [i("class"), i("tg-avatar__image")], + ], + }, + layout: { + nodeIndex: [1, 2, 3, 4], + styles: [ + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + ], + bounds: [ + [0, 0, 1000, 800], + [940, 16, 32, 32], + [942, 18, 28, 28], + [944, 20, 24, 24], + ], + paintOrders: [0, 1, 2, 3], + }, + }, + ], + }; +} + function makeCdp(snapshot: unknown) { return { send: vi.fn(async (_tab: number, method: string) => { @@ -126,11 +276,11 @@ describe("captureViewModel", () => { }); it("runs hover surface probes when explicitly enabled", async () => { - let runtimeCalls = 0; + let hoverStateCalls = 0; const cdp = { send: vi.fn(async (_tabId: number, method: string, params?: object) => { if (method === "DOMSnapshot.enable") return {}; - if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply(); + if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply(); if (method === "Page.getLayoutMetrics") { return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, @@ -141,22 +291,23 @@ describe("captureViewModel", () => { if (expression.includes("input,textarea,select")) { return { result: { value: { controls: [], childFrames: [] } } }; } - runtimeCalls += 1; - return runtimeCalls === 1 - ? { - result: { - value: [ - { - triggerSel: ".menu", - affectedSub: " .items", - label: "Products", - x: 50, - y: 20, - }, - ], - }, - } - : { result: { value: ["Shoes", "Bags"] } }; + if (expression.includes("document.styleSheets")) { + return { result: { value: [] } }; + } + if (expression.includes("querySelectorAll(selectors)")) { + hoverStateCalls += 1; + return hoverStateCalls === 1 + ? { result: { value: [] } } + : { + result: { + value: [ + { text: "My profile", role: "", tag: "a", x: 900, y: 60 }, + { text: "Sign out", role: "", tag: "a", x: 900, y: 90 }, + ], + }, + }; + } + throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`); } if (method === "Input.dispatchMouseEvent") return {}; throw new Error(`unexpected ${method}`); @@ -166,12 +317,18 @@ describe("captureViewModel", () => { const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true }); expect(result.surfaceProbes).toEqual([ - { triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] }, + { + triggerBackendNodeId: 12, + triggerPoint: { x: 956, y: 32 }, + triggerAction: "hover", + subItems: ["My profile", "Sign out"], + confidence: "high", + }, ]); expect(cdp.send).toHaveBeenCalledWith( 4, "Input.dispatchMouseEvent", - expect.objectContaining({ type: "mouseMoved", x: 50, y: 20 }), + expect.objectContaining({ type: "mouseMoved", x: 956, y: 32 }), ); expect(cdp.send).toHaveBeenCalledWith( 4, @@ -180,6 +337,91 @@ describe("captureViewModel", () => { ); }); + it("uses a fresh baseline for each hover candidate", async () => { + let hoverStateCalls = 0; + const cdp = { + send: vi.fn(async (_tabId: number, method: string, params?: object) => { + if (method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply(); + if (method === "Page.getLayoutMetrics") { + return { + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + }; + } + if (method === "Runtime.evaluate") { + const expression = (params as { expression?: string } | undefined)?.expression ?? ""; + if (expression.includes("input,textarea,select")) { + return { result: { value: { controls: [], childFrames: [] } } }; + } + if (expression.includes("document.styleSheets")) { + return { result: { value: [] } }; + } + if (expression.includes("querySelectorAll(selectors)")) { + hoverStateCalls += 1; + const visibleMenu = [ + { text: "My profile", role: "", tag: "a", x: 900, y: 60 }, + { text: "Sign out", role: "", tag: "a", x: 900, y: 90 }, + ]; + return hoverStateCalls === 1 + ? { result: { value: [] } } + : { result: { value: visibleMenu } }; + } + throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`); + } + if (method === "Input.dispatchMouseEvent") return {}; + throw new Error(`unexpected ${method}`); + }) as unknown as (tabId: number, method: string, params?: object) => Promise, + }; + + const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true }); + + expect(result.surfaceProbes).toEqual([ + expect.objectContaining({ + triggerBackendNodeId: 12, + subItems: ["My profile", "Sign out"], + }), + ]); + expect(hoverStateCalls).toBeGreaterThanOrEqual(4); + }); + + it("deduplicates nested hover candidates for one visual trigger", async () => { + let hoverMoves = 0; + const cdp = { + send: vi.fn(async (_tabId: number, method: string, params?: object) => { + if (method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply(); + if (method === "Page.getLayoutMetrics") { + return { + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + }; + } + if (method === "Runtime.evaluate") { + const expression = (params as { expression?: string } | undefined)?.expression ?? ""; + if (expression.includes("input,textarea,select")) { + return { result: { value: { controls: [], childFrames: [] } } }; + } + if (expression.includes("document.styleSheets")) { + return { result: { value: [] } }; + } + if (expression.includes("querySelectorAll(selectors)")) { + return { result: { value: [] } }; + } + throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`); + } + if (method === "Input.dispatchMouseEvent") { + const point = params as { x?: number; y?: number }; + if (point.x !== -10 && point.x !== 0) hoverMoves += 1; + return {}; + } + throw new Error(`unexpected ${method}`); + }) as unknown as (tabId: number, method: string, params?: object) => Promise, + }; + + await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true }); + + expect(hoverMoves).toBe(1); + }); + it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => { // DOMSnapshot inlines an open shadow root's content as descendants of the // host. The agent's WXT overlay host carries a fixed full-viewport diff --git a/apps/extension/src/tools/vom/capture.ts b/apps/extension/src/tools/vom/capture.ts index d8358f5..44aba56 100644 --- a/apps/extension/src/tools/vom/capture.ts +++ b/apps/extension/src/tools/vom/capture.ts @@ -45,9 +45,11 @@ export interface CapturedNode { export type CapturedIframeNodes = Map; export interface CapturedSurfaceProbe { - triggerLabel: string; + triggerBackendNodeId: number; + triggerPoint?: { x: number; y: number }; triggerAction: "hover" | "focus" | string; subItems: string[]; + confidence?: "high" | "medium" | "low"; } export interface CapturedViewModel { @@ -61,6 +63,7 @@ export interface CapturedViewModel { export interface CaptureViewModelOptions { conditionalSurfaceProbe?: boolean; + hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise; } /** Sparse array format Chrome uses for infrequently-set per-node fields. */ @@ -225,11 +228,12 @@ interface RuntimeEvaluateReply { } interface HoverCandidate { - triggerSel: string; - affectedSub: string; - label: string; + backendNodeId: number; + label?: string; x: number; y: number; + score: number; + reasons: string[]; } interface CdpDomNode { @@ -322,17 +326,19 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set(reply: RuntimeEvaluateReply): T | undefined { return reply.result?.value as T | undefined; } -function hoverCssScanExpression(): string { +function hoverCssTriggerScanExpression(): string { return `(() => { const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"]; - const pairs = []; + const centres = []; const seenRules = new Set(); for (const sheet of Array.from(document.styleSheets)) { let rules; @@ -348,90 +354,73 @@ function hoverCssScanExpression(): string { const hoverIndex = part.indexOf(":hover"); if (hoverIndex < 0) continue; const triggerSel = part.slice(0, hoverIndex).trim(); - const affectedSub = part.slice(hoverIndex + 6).trim(); if (!triggerSel) continue; - const key = triggerSel + "||" + affectedSub; - if (seenRules.has(key)) continue; - seenRules.add(key); - pairs.push({ triggerSel, affectedSub }); + if (seenRules.has(triggerSel)) continue; + seenRules.add(triggerSel); + let elements; + try { elements = Array.from(document.querySelectorAll(triggerSel)); } catch { continue; } + for (const el of elements) { + if (!(el instanceof HTMLElement)) continue; + const rect = el.getBoundingClientRect(); + const style = getComputedStyle(el); + if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue; + centres.push({ x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }); + if (centres.length >= 24) return centres; + } } } } - - const candidates = []; - const seenLabels = new Set(); - const isUnsafeTrigger = (el) => { - if (!(el instanceof HTMLElement)) return true; - const tag = el.tagName.toLowerCase(); - if (["input", "textarea", "select", "option"].includes(tag)) return true; - if (el.isContentEditable) return true; - if (el.hasAttribute("disabled") || el.hasAttribute("inert")) return true; - if ((el.getAttribute("aria-disabled") || "").toLowerCase() === "true") return true; - return false; - }; - for (const pair of pairs) { - let elements; - try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; } - for (const el of elements) { - if (!(el instanceof Element)) continue; - if (isUnsafeTrigger(el)) continue; - const rect = el.getBoundingClientRect(); - const style = getComputedStyle(el); - if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue; - const label = (el.textContent || "").replace(/\\s+/g, " ").trim(); - if (!label || seenLabels.has(label)) continue; - seenLabels.add(label); - candidates.push({ - triggerSel: pair.triggerSel, - affectedSub: pair.affectedSub, - label, - x: rect.left + rect.width / 2, - y: rect.top + rect.height / 2, - }); - if (candidates.length >= ${MAX_HOVER_TRIGGERS}) return candidates; - } - } - return candidates; + return centres; })()`; } -function hoverCollectExpression(candidate: HoverCandidate): string { +interface HoverRuntimeItem { + text: string; + role: string; + tag: string; + x: number; + y: number; +} + +function hoverStateExpression(): string { return `(() => { - const triggerSel = ${JSON.stringify(candidate.triggerSel)}; - const rawAffectedSub = ${JSON.stringify(candidate.affectedSub)}; - const x = ${JSON.stringify(candidate.x)}; - const y = ${JSON.stringify(candidate.y)}; - const hit = document.elementFromPoint(x, y); - const trigger = hit instanceof Element ? hit.closest(triggerSel) : null; - if (!trigger) return []; - - const affectedSub = rawAffectedSub.replace(/^[>+~]\\s*/, "").trim(); - let targets = []; - if (!affectedSub) { - targets = [trigger]; - } else { - try { targets = Array.from(trigger.querySelectorAll(affectedSub)); } catch { targets = []; } - } const items = []; const seen = new Set(); - const push = (value) => { - const text = String(value || "").replace(/\\s+/g, " ").trim(); + const selectors = [ + "a", + "button", + "[role='menuitem']", + "[role='menuitemcheckbox']", + "[role='menuitemradio']", + "[role='option']", + "[role='tab']", + "[role='link']", + "[role='button']" + ].join(","); + const push = (el) => { + if (!(el instanceof HTMLElement)) return; + const style = getComputedStyle(el); + const rect = el.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) return; + if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") return; + const text = String( + el.getAttribute("aria-label") || + el.getAttribute("title") || + el.textContent || + "" + ).replace(/\\s+/g, " ").trim(); if (!text || seen.has(text)) return; seen.add(text); - items.push(text); + items.push({ + text, + role: (el.getAttribute("role") || "").toLowerCase(), + tag: el.tagName.toLowerCase(), + x: rect.left, + y: rect.top, + }); }; - for (const target of targets) { - if (!(target instanceof Element)) continue; - const style = getComputedStyle(target); - if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue; - const clickable = target.querySelectorAll('a, button, [role="menuitem"], [role="option"]'); - if (clickable.length > 0) { - for (const child of Array.from(clickable)) push(child.textContent); - } else { - push(target.textContent); - } - } - return items.slice(0, 12); + for (const el of Array.from(document.querySelectorAll(selectors))) push(el); + return items.slice(0, 400); })()`; } @@ -453,45 +442,313 @@ async function clearHover(cdp: CdpRunner, tabId: number): Promise { ); } -async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise { +function isVisibleProbeNode(node: CapturedNode): boolean { + const rect = node.rect; + return ( + !!rect && + rect.w > 0 && + rect.h > 0 && + node.pointerEvents !== "none" && + node.attrs.hidden === undefined && + node.attrs.inert === undefined && + (node.attrs["aria-hidden"] ?? "").toLowerCase() !== "true" + ); +} + +function isUnsafeHoverTrigger(node: CapturedNode): boolean { + const tag = node.tag.toLowerCase(); + if (["input", "textarea", "select", "option"].includes(tag)) return true; + if ((node.attrs.contenteditable ?? "").toLowerCase() === "true") return true; + if (node.attrs.disabled !== undefined || node.attrs.inert !== undefined) return true; + return (node.attrs["aria-disabled"] ?? "").toLowerCase() === "true"; +} + +function capturedText(node: CapturedNode): string | undefined { + const value = + node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? ""; + const clean = value.replace(/\s+/g, " ").trim(); + return clean || undefined; +} + +function hasGraphicDescendant( + node: CapturedNode, + childrenByParentId: Map, + depth = 0, +): boolean { + if (depth > 3) return false; + for (const child of childrenByParentId.get(node.backendNodeId) ?? []) { + const tag = child.tag.toLowerCase(); + if (["img", "svg", "use", "path", "i"].includes(tag)) return true; + if (hasGraphicDescendant(child, childrenByParentId, depth + 1)) return true; + } + return false; +} + +function hasHoverPopupSignal(node: CapturedNode): boolean { + const attrs = node.attrs; + const haystack = [ + attrs.id, + attrs.class, + attrs["data-testid"], + attrs["data-test"], + attrs["data-cy"], + attrs["aria-haspopup"], + attrs["aria-controls"], + ] + .filter(Boolean) + .join(" ") + .toLowerCase(); + return /(menu|dropdown|popover|popup|avatar|profile|account|user)/.test(haystack); +} + +function roleOf(node: CapturedNode): string { + return (node.attrs.role ?? "").toLowerCase(); +} + +function hasDirectInteractiveSignal(node: CapturedNode): boolean { + const tag = node.tag.toLowerCase(); + const role = roleOf(node); + return ( + ["button", "a", "summary"].includes(tag) || + ["button", "link", "menuitem", "tab", "combobox"].includes(role) || + node.attrs.tabindex !== undefined || + node.cursor === "pointer" || + node.attrs.onclick !== undefined || + node.attrs.onmouseenter !== undefined || + node.attrs.onmouseover !== undefined || + node.attrs["aria-haspopup"] !== undefined || + node.attrs["aria-expanded"] === "false" + ); +} + +function scoreHoverCandidate( + node: CapturedNode, + childrenByParentId: Map, + cssHoverPoints: Array<{ x: number; y: number }>, +): HoverCandidate | null { + if (!isVisibleProbeNode(node) || isUnsafeHoverTrigger(node)) { + return null; + } + const rect = node.rect; + if (!rect) return null; + const area = rect.w * rect.h; + if (area <= 0 || area > 160_000) return null; + + const reasons: string[] = []; + let score = 0; + const tag = node.tag.toLowerCase(); + const role = roleOf(node); + const label = capturedText(node); + const graphic = hasGraphicDescendant(node, childrenByParentId); + const directInteractive = hasDirectInteractiveSignal(node); + const popupSignal = hasHoverPopupSignal(node); + const compactTopbarIcon = !label && graphic && rect.y < 120 && rect.w <= 96 && rect.h <= 96; + if (!directInteractive && !popupSignal && !compactTopbarIcon) return null; + + if (node.attrs["aria-haspopup"] !== undefined) { + score += 80; + reasons.push("aria-haspopup"); + } + if ((node.attrs["aria-expanded"] ?? "").toLowerCase() === "false") { + score += 55; + reasons.push("collapsed"); + } + if (popupSignal) { + score += 45; + reasons.push("popup-signal"); + } + if (tag === "button" || role === "button") { + score += 30; + reasons.push("button"); + } + if (node.cursor === "pointer") { + score += 25; + reasons.push("pointer"); + } + if (!label && graphic) { + score += 40; + reasons.push("icon-only"); + } + if (rect.y < 120) { + score += 25; + reasons.push("topbar"); + } + if (rect.w <= 80 && rect.h <= 80) { + score += 20; + reasons.push("compact"); + } + if ( + cssHoverPoints.some( + (point) => + point.x >= rect.x && + point.x <= rect.x + rect.w && + point.y >= rect.y && + point.y <= rect.y + rect.h, + ) + ) { + score += 40; + reasons.push("css-hover"); + } + + if (score < 45) return null; + return { + backendNodeId: node.backendNodeId, + label, + x: rect.x + rect.w / 2, + y: rect.y + rect.h / 2, + score, + reasons, + }; +} + +function buildHoverCandidates( + nodes: CapturedNode[], + cssHoverPoints: Array<{ x: number; y: number }>, +): HoverCandidate[] { + const childrenByParentId = new Map(); + const parentByBackendId = new Map(); + for (const node of nodes) { + parentByBackendId.set(node.backendNodeId, node.parentBackendNodeId); + if (node.parentBackendNodeId === null) continue; + const children = childrenByParentId.get(node.parentBackendNodeId) ?? []; + children.push(node); + childrenByParentId.set(node.parentBackendNodeId, children); + } + + const candidates = nodes + .map((node) => scoreHoverCandidate(node, childrenByParentId, cssHoverPoints)) + .filter((candidate): candidate is HoverCandidate => candidate !== null) + .sort((a, b) => b.score - a.score); + + const deduped: HoverCandidate[] = []; + const seen = new Set(); + for (const candidate of candidates) { + if (seen.has(candidate.backendNodeId)) continue; + if (deduped.some((existing) => sameHoverCluster(existing, candidate, parentByBackendId))) { + continue; + } + seen.add(candidate.backendNodeId); + deduped.push(candidate); + if (deduped.length >= MAX_HOVER_TRIGGERS) break; + } + return deduped; +} + +function sameHoverCluster( + a: HoverCandidate, + b: HoverCandidate, + parentByBackendId: Map, +): boolean { + if (Math.hypot(a.x - b.x, a.y - b.y) <= 8) return true; + return ( + isBackendAncestor(a.backendNodeId, b.backendNodeId, parentByBackendId) || + isBackendAncestor(b.backendNodeId, a.backendNodeId, parentByBackendId) + ); +} + +function isBackendAncestor( + ancestorId: number, + nodeId: number, + parentByBackendId: Map, +): boolean { + let parentId = parentByBackendId.get(nodeId); + let guard = 0; + while (parentId !== null && parentId !== undefined && guard < parentByBackendId.size) { + if (parentId === ancestorId) return true; + parentId = parentByBackendId.get(parentId); + guard += 1; + } + return false; +} + +function diffHoverItems(before: HoverRuntimeItem[], after: HoverRuntimeItem[]): string[] { + const beforeKeys = new Set(before.map((item) => item.text.toLowerCase())); + const out: string[] = []; + const seen = new Set(); + for (const item of after) { + const text = item.text.replace(/\s+/g, " ").trim(); + const key = text.toLowerCase(); + if (!text || beforeKeys.has(key) || seen.has(key)) continue; + seen.add(key); + out.push(text); + if (out.length >= MAX_HOVER_SUB_ITEMS) break; + } + return out; +} + +function confidenceForHover( + candidate: HoverCandidate, + subItems: string[], +): "high" | "medium" | "low" { + if (subItems.length >= 2 && candidate.score >= 80) return "high"; + if (subItems.length >= 2 || candidate.score >= 80) return "medium"; + return "low"; +} + +async function probeHoverSurfaces( + cdp: CdpRunner, + tabId: number, + nodes: CapturedNode[], + options: CaptureViewModelOptions, +): Promise { const started = Date.now(); try { - const scan = await cdp.send(tabId, "Runtime.evaluate", { - expression: hoverCssScanExpression(), + const cssScan = await cdp.send(tabId, "Runtime.evaluate", { + expression: hoverCssTriggerScanExpression(), returnByValue: true, }); - const candidates = runtimeValue(scan) ?? []; + const cssHoverPoints = runtimeValue>(cssScan) ?? []; + const candidates = buildHoverCandidates(nodes, cssHoverPoints); + if (candidates.length === 0) return []; + const results: CapturedSurfaceProbe[] = []; - const seen = new Set(); - for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) { - if (Date.now() - started > MAX_HOVER_PROBE_MS) break; - if (!candidate.label || seen.has(candidate.label)) continue; - try { - await cdp.send(tabId, "Input.dispatchMouseEvent", { - type: "mouseMoved", - x: candidate.x, - y: candidate.y, - }); - await wait(HOVER_SETTLE_MS); - const collected = await cdp.send(tabId, "Runtime.evaluate", { - expression: hoverCollectExpression(candidate), - returnByValue: true, - }); - const subItems = (runtimeValue(collected) ?? []) - .map((item) => item.replace(/\s+/g, " ").trim()) - .filter(Boolean); - if (subItems.length === 0) continue; - seen.add(candidate.label); - results.push({ - triggerLabel: candidate.label, - triggerAction: "hover", - subItems, - }); - } catch { - continue; - } finally { - await clearHover(cdp, tabId); + const seen = new Set(); + await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined); + try { + for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) { + if (Date.now() - started > MAX_HOVER_PROBE_MS) break; + if (results.length >= MAX_HOVER_SURFACES) break; + if (seen.has(candidate.backendNodeId)) continue; + try { + await clearHover(cdp, tabId); + await wait(HOVER_SETTLE_MS); + const baselineReply = await cdp.send(tabId, "Runtime.evaluate", { + expression: hoverStateExpression(), + returnByValue: true, + }); + const baselineItems = runtimeValue(baselineReply) ?? []; + + await cdp.send(tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: candidate.x, + y: candidate.y, + }); + await wait(HOVER_SETTLE_MS); + const collected = await cdp.send(tabId, "Runtime.evaluate", { + expression: hoverStateExpression(), + returnByValue: true, + }); + const subItems = diffHoverItems( + baselineItems, + runtimeValue(collected) ?? [], + ); + if (subItems.length === 0) continue; + seen.add(candidate.backendNodeId); + results.push({ + triggerBackendNodeId: candidate.backendNodeId, + triggerPoint: { x: candidate.x, y: candidate.y }, + triggerAction: "hover", + subItems, + confidence: confidenceForHover(candidate, subItems), + }); + } catch { + continue; + } finally { + await clearHover(cdp, tabId); + } } + } finally { + await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined); } return results; } catch (err) { @@ -797,7 +1054,9 @@ export async function captureViewModel( await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]); - const surfaceProbes = options.conditionalSurfaceProbe ? await probeHoverSurfaces(cdp, tabId) : []; + const surfaceProbes = options.conditionalSurfaceProbe + ? await probeHoverSurfaces(cdp, tabId, nodes, options) + : []; return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds }; } diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index c3ebdff..4533d7d 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -314,8 +314,21 @@ export interface SnapshotResult { dialogs?: JavaScriptDialogInfo[]; } -export type ObserveParams = SnapshotParams; -export type ObserveResult = SnapshotResult; +export interface ObserveParams extends SnapshotParams { + debug_surfaces?: boolean; +} + +export interface ObserveResult extends SnapshotResult { + debug?: { + surface_probes?: Array<{ + trigger_backend_node_id: number; + trigger_point?: { x: number; y: number }; + trigger_action: string; + sub_items: string[]; + confidence?: string; + }>; + }; +} export interface GetHtmlParams { session_id: string; diff --git a/crates/bsk-cli/src/cli/observe.rs b/crates/bsk-cli/src/cli/observe.rs index 72efa28..bae34fe 100644 --- a/crates/bsk-cli/src/cli/observe.rs +++ b/crates/bsk-cli/src/cli/observe.rs @@ -29,6 +29,10 @@ pub struct ObserveArgs { /// Soft cap on rendered tokens (~4 chars/token). #[arg(long = "max-tokens")] pub max_tokens: Option, + + /// Include conditional surface probe diagnostics in JSON output. + #[arg(long = "debug-surfaces")] + pub debug_surfaces: bool, } pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> { @@ -42,6 +46,7 @@ fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError> tab_id: args.tab_id, max_depth: args.max_depth, max_tokens: args.max_tokens, + debug_surfaces: args.debug_surfaces, }; let reply: ObserveResult = call(sock, params)?; match format { diff --git a/crates/bsk-cli/tests/tools_ipc.rs b/crates/bsk-cli/tests/tools_ipc.rs index 63714a0..d40bb9c 100644 --- a/crates/bsk-cli/tests/tools_ipc.rs +++ b/crates/bsk-cli/tests/tools_ipc.rs @@ -478,6 +478,7 @@ async fn observe_returns_semantic_text_and_ref_count() { tab_id: 13, truncated: false, dialogs: vec![], + debug: None, }) .unwrap(), ) @@ -492,6 +493,7 @@ async fn observe_returns_semantic_text_and_ref_count() { tab_id: None, max_depth: None, max_tokens: None, + debug_surfaces: false, }, ) .await diff --git a/crates/bsk-protocol/schema/tool_observe_params.json b/crates/bsk-protocol/schema/tool_observe_params.json index 2da54a9..10e57a0 100644 --- a/crates/bsk-protocol/schema/tool_observe_params.json +++ b/crates/bsk-protocol/schema/tool_observe_params.json @@ -7,6 +7,11 @@ "session_id" ], "properties": { + "debug_surfaces": { + "description": "Include implementation diagnostics for conditional surface probes.", + "default": false, + "type": "boolean" + }, "max_depth": { "description": "Cap the depth of the rendered VOM tree.", "type": [ diff --git a/crates/bsk-protocol/schema/tool_observe_result.json b/crates/bsk-protocol/schema/tool_observe_result.json index 2b2d9fd..22ee410 100644 --- a/crates/bsk-protocol/schema/tool_observe_result.json +++ b/crates/bsk-protocol/schema/tool_observe_result.json @@ -8,6 +8,16 @@ "text" ], "properties": { + "debug": { + "anyOf": [ + { + "$ref": "#/definitions/ObserveDebug" + }, + { + "type": "null" + } + ] + }, "dialogs": { "type": "array", "items": { @@ -103,6 +113,73 @@ "prompt", "beforeunload" ] + }, + "ObserveDebug": { + "type": "object", + "properties": { + "surface_probes": { + "type": "array", + "items": { + "$ref": "#/definitions/SurfaceProbeDebug" + } + } + } + }, + "SurfaceProbeDebug": { + "type": "object", + "required": [ + "sub_items", + "trigger_action", + "trigger_backend_node_id" + ], + "properties": { + "confidence": { + "type": [ + "string", + "null" + ] + }, + "sub_items": { + "type": "array", + "items": { + "type": "string" + } + }, + "trigger_action": { + "type": "string" + }, + "trigger_backend_node_id": { + "type": "integer", + "format": "int64" + }, + "trigger_point": { + "anyOf": [ + { + "$ref": "#/definitions/SurfaceProbePoint" + }, + { + "type": "null" + } + ] + } + } + }, + "SurfaceProbePoint": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number", + "format": "double" + }, + "y": { + "type": "number", + "format": "double" + } + } } } } diff --git a/crates/bsk-protocol/src/tools/observation.rs b/crates/bsk-protocol/src/tools/observation.rs index 715ddc9..0103ea5 100644 --- a/crates/bsk-protocol/src/tools/observation.rs +++ b/crates/bsk-protocol/src/tools/observation.rs @@ -74,6 +74,32 @@ pub struct ObserveParams { /// best-effort heuristic based on character count). #[serde(default, skip_serializing_if = "Option::is_none")] pub max_tokens: Option, + /// Include implementation diagnostics for conditional surface probes. + #[serde(default)] + pub debug_surfaces: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SurfaceProbePoint { + pub x: f64, + pub y: f64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SurfaceProbeDebug { + pub trigger_backend_node_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trigger_point: Option, + pub trigger_action: String, + pub sub_items: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub confidence: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ObserveDebug { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub surface_probes: Vec, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] @@ -92,6 +118,8 @@ pub struct ObserveResult { pub truncated: bool, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub dialogs: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub debug: Option, } // --------------------------------------------------------------------------- @@ -280,6 +308,7 @@ mod tests { tab_id: 42, truncated: false, dialogs: Vec::new(), + debug: None, }; let v = serde_json::to_value(&r).unwrap(); assert_eq!(v.get("ref_count").and_then(|v| v.as_u64()), Some(1)); diff --git a/packages/vom/src/index.ts b/packages/vom/src/index.ts index 45fe4ac..745eb00 100644 --- a/packages/vom/src/index.ts +++ b/packages/vom/src/index.ts @@ -1,4 +1,4 @@ -export { renderVom } from "./render"; +export { applyVomInteractionRecovery, isVomReferenceNode, renderVom } from "./render"; export type { ActiveScopeBlock, BlockingLayer, diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 742b58a..0926fee 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -197,7 +197,7 @@ function shouldRender(node: VomNode): boolean { return cleaned(node.name) !== undefined; } -function shouldReference(node: VomNode): boolean { +export function isVomReferenceNode(node: VomNode): boolean { return INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node); } @@ -342,7 +342,7 @@ function isAncestorOf( return false; } -function applyRecovery(nodes: VomNode[]): VomNode[] { +export function applyVomInteractionRecovery(nodes: VomNode[]): VomNode[] { const parentMap = buildParentMap(nodes); const children = buildChildren(nodes); const recoveredIds = new Set(); @@ -350,7 +350,7 @@ function applyRecovery(nodes: VomNode[]): VomNode[] { const recoveredNameById = new Map(); for (const node of nodes) { - if (shouldReference(node)) continue; + if (isVomReferenceNode(node)) continue; const role = recoveredRole(node, children); if (!role || !passesRecoveryGuards(node, children)) continue; recoveredIds.add(node.id); @@ -397,7 +397,7 @@ function buildChildren(nodes: VomNode[]): Map { function duplicateReferenceNames(nodes: VomNode[]): Set { const counts = new Map(); for (const node of nodes) { - if (!shouldReference(node)) continue; + if (!isVomReferenceNode(node)) continue; const name = cleaned(node.name); if (!name) continue; const key = normalizeContextKey(name); @@ -427,7 +427,7 @@ function contextLabel(node: VomNode, targetName: string): string | undefined { } function weakLabelContext(node: VomNode, targetName: string): string | undefined { - if (shouldReference(node)) return undefined; + if (isVomReferenceNode(node)) return undefined; return weakContextText( cleaned(node.name) ?? cleaned(node.text) ?? cleaned(node.nearbyText), targetName, @@ -509,7 +509,7 @@ function collectWeakLabelsFromSubtree( state: RenderState, labels: string[], ): void { - if (shouldReference(node)) return; + if (isVomReferenceNode(node)) return; const label = weakLabelContext(node, nodeName); if (label) { labels.push(label); @@ -666,7 +666,7 @@ function descendantTextCoveredByName(node: VomNode, state: RenderState): boolean const texts: string[] = []; while (stack.length > 0) { const child = stack.pop() as VomNode; - if (shouldReference(child)) return false; + if (isVomReferenceNode(child)) return false; const text = cleaned(child.name) ?? cleaned(child.text) ?? cleaned(child.value); if (text) texts.push(text); stack.push(...(state.children.get(child.id) ?? [])); @@ -750,7 +750,7 @@ function renderTree( continue; } - const ref = shouldReference(node) ? `e${state.nextRef}` : undefined; + const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined; const context = ref ? handleContext(node, state) : []; const surface = state.surfaceMap.get(node.id); const line = renderNodeLine(node, depth, ref, context, surface); @@ -915,7 +915,7 @@ function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] { const blockedRoots = new Set(); for (const target of nodes) { - if (!shouldReference(target)) continue; + if (!isVomReferenceNode(target)) continue; const blocked = candidates.some((candidate) => isBlockedByRegion(target, candidate.node, parentMap, candidate.viewportCoverage), ); @@ -970,7 +970,7 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp } export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult { - const nodes = applyRecovery(scene.nodes); + const nodes = applyVomInteractionRecovery(scene.nodes); const renderScene = { ...scene, nodes }; const layer = detectBlockingLayer(nodes, scene.viewport); if (layer) return renderDoubleLayer(renderScene, layer, options); From d24bb6aa93a7f6f23e02acc696e11721faa926f8 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Thu, 6 Aug 2026 15:48:11 +0800 Subject: [PATCH 03/12] feat(extension): dispatcher fix --- .../src/tools/__tests__/dispatcher.test.ts | 112 +++++++++++- apps/extension/src/tools/dispatcher.ts | 170 ++++++++++++------ 2 files changed, 223 insertions(+), 59 deletions(-) diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 1b99653..ec2c84c 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -346,20 +346,23 @@ describe("ToolDispatcher", () => { }; const dispatcher = new ToolDispatcher({ transport, sessions, cdp }); - (dispatcher as unknown as { rememberHover: (result: unknown) => unknown }).rememberHover({ + ( + dispatcher as unknown as { rememberHover: (sessionId: string, result: unknown) => unknown } + ).rememberHover("aa11", { tab_id: 7, x: 10, y: 20, }); const helpers = dispatcher as unknown as { withHoverReassert: ( + params: { session_id: string; tab_id?: number }, work: () => Promise, options?: { releaseAfter?: boolean }, ) => Promise; - setHoverBypass: (tabId: number, enabled: boolean) => Promise; + setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise; }; - await helpers.withHoverReassert(async () => { + await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => { order.push("observe"); return {}; }); @@ -369,8 +372,9 @@ describe("ToolDispatcher", () => { expect.objectContaining({ enabled: false }), ); - await helpers.setHoverBypass(7, true); + await helpers.setHoverBypass("aa11", 7, true); await helpers.withHoverReassert( + { session_id: "aa11", tab_id: 7 }, async () => { order.push("click"); return {}; @@ -389,6 +393,106 @@ describe("ToolDispatcher", () => { ); }); + it("does not reassert or disable hover latches from another session", async () => { + vi.stubGlobal("chrome", { + tabs: { + sendMessage: vi.fn(async () => undefined), + }, + }); + const { transport } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 1), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + const cdp = { + send: vi.fn(async () => ({})), + detachSession: vi.fn(async () => {}), + ensureNetworkCapture: vi.fn(async () => {}), + networkEntriesSince: vi.fn(() => ({ + tab_id: 7, + entries: [], + next_since: 0, + truncated: false, + })), + setDeviceMetricsOverride: vi.fn(async () => {}), + clearDeviceMetricsOverride: vi.fn(async () => {}), + setUserAgentOverride: vi.fn(async () => {}), + setTouchEmulationEnabled: vi.fn(async () => {}), + }; + const dispatcher = new ToolDispatcher({ transport, sessions, cdp }); + const helpers = dispatcher as unknown as { + rememberHover: (sessionId: string, result: unknown) => unknown; + setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise; + withHoverReassert: ( + params: { session_id: string; tab_id?: number }, + work: () => Promise, + options?: { releaseAfter?: boolean }, + ) => Promise; + releaseHoverLatch: (sessionId?: string, tabId?: number) => Promise; + }; + + helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 }); + await helpers.setHoverBypass("aa11", 7, true); + + await helpers.withHoverReassert({ session_id: "bb22", tab_id: 7 }, async () => ({}), { + releaseAfter: true, + }); + + expect(cdp.send).not.toHaveBeenCalled(); + expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + + await helpers.releaseHoverLatch("aa11", 7); + expect(chrome.tabs.sendMessage).toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + }); + + it("releases the current session hover latch before navigation-style work", async () => { + vi.stubGlobal("chrome", { + tabs: { + sendMessage: vi.fn(async () => undefined), + }, + }); + const { transport } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 1), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + const dispatcher = new ToolDispatcher({ transport, sessions }); + const helpers = dispatcher as unknown as { + rememberHover: (sessionId: string, result: unknown) => unknown; + setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise; + withHoverReleaseForRequest: ( + params: { session_id: string; tab_id?: number }, + work: () => Promise, + ) => Promise; + withHoverReassert: ( + params: { session_id: string; tab_id?: number }, + work: () => Promise, + ) => Promise; + }; + helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 }); + await helpers.setHoverBypass("aa11", 7, true); + + await helpers.withHoverReleaseForRequest({ session_id: "aa11", tab_id: 7 }, async () => ({})); + await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({})); + + expect(chrome.tabs.sendMessage).toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + }); + it("disconnects the transport when send() fails so keepalive can rebuild", async () => { const { transport, deliver } = fakeTransport(); const sessions = new SessionManager({ diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index c382d96..4ca8297 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -83,6 +83,13 @@ type DispatcherCdpRunner = CdpRunner & detachSession(sessionId: string): Promise; }; +interface HoverLatch { + sessionId: string; + tabId: number; + x: number; + y: number; +} + export interface DispatcherDeps { transport: Transport; sessions: SessionManager; @@ -128,8 +135,8 @@ export class ToolDispatcher { private readonly approveBorrow?: BorrowConfirmationApprover; private readonly helpNotificationCopy?: () => { title: string; body: string }; private subscription: { dispose(): void } | null = null; - private readonly hoverBypassTabs = new Set(); - private readonly hoverLatches = new Map(); + private readonly hoverBypassTabs = new Map(); + private readonly hoverLatches = new Map(); /** * Per-rpc-id `AbortController` registry. Populated inside * [`dispatch`] before we await the tool handler and torn down in @@ -276,7 +283,7 @@ export class ToolDispatcher { case "tool.session_start": return handleSessionStart(this.sessions, req.params as SessionStartParams); case "tool.session_stop": { - await this.releaseHoverLatch(); + await this.releaseHoverLatch((req.params as SessionStopParams).session_id); return handleSessionStop(this.sessions, req.params as SessionStopParams, { cdp: this.cdp, }); @@ -286,7 +293,9 @@ export class ToolDispatcher { case "tool.tab_create": return handleTabCreate(this.sessions, req.params as TabCreateParams); case "tool.tab_close": - return handleTabClose(this.sessions, req.params as TabCloseParams); + return this.withHoverReleaseForRequest(req.params as TabCloseParams, () => + handleTabClose(this.sessions, req.params as TabCloseParams), + ); case "tool.tab_select": return handleTabSelect(this.sessions, req.params as TabSelectParams); case "tool.tab_borrow": @@ -325,7 +334,7 @@ export class ToolDispatcher { this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined, ); case "tool.snapshot": - return this.withHoverReassert(() => + return this.withHoverReassert(req.params as SnapshotParams, () => handleSnapshot( this.sessions, req.params as SnapshotParams, @@ -333,7 +342,7 @@ export class ToolDispatcher { ), ); case "tool.observe": - return this.withHoverReassert(() => + return this.withHoverReassert(req.params as ObserveParams, () => handleObserve( this.sessions, req.params as ObserveParams, @@ -341,7 +350,9 @@ export class ToolDispatcher { ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi, - conditionalSurfaceProbe: !this.hasHoverLatch(), + conditionalSurfaceProbe: !this.hasHoverLatchForRequest( + req.params as ObserveParams, + ), hoverProbeBypassOverlay: bypassOverlay, } : undefined, @@ -354,31 +365,40 @@ export class ToolDispatcher { this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined, ); case "tool.navigate": - return handleNavigate( - this.sessions, - req.params as NavigateParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + return this.withHoverReleaseForRequest(req.params as NavigateParams, () => + handleNavigate( + this.sessions, + req.params as NavigateParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), ); case "tool.navigate_back": - return handleNavigateBack( - this.sessions, - req.params as NavigateBackParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + return this.withHoverReleaseForRequest(req.params as NavigateBackParams, () => + handleNavigateBack( + this.sessions, + req.params as NavigateBackParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), ); case "tool.navigate_forward": - return handleNavigateForward( - this.sessions, - req.params as NavigateForwardParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + return this.withHoverReleaseForRequest(req.params as NavigateForwardParams, () => + handleNavigateForward( + this.sessions, + req.params as NavigateForwardParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), ); case "tool.reload": - return handleReload( - this.sessions, - req.params as ReloadParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + return this.withHoverReleaseForRequest(req.params as ReloadParams, () => + handleReload( + this.sessions, + req.params as ReloadParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), ); case "tool.click": return this.withHoverReassert( + req.params as ClickParams, () => handleClick( this.sessions, @@ -403,30 +423,37 @@ export class ToolDispatcher { cdp: this.cdp, tabsApi: chromeTabsApi, signal, - bypassOverlay: (tabId, enabled) => this.setHoverBypass(tabId, enabled), + bypassOverlay: (tabId, enabled) => + this.setHoverBypass((req.params as HoverParams).session_id, tabId, enabled), keepOverlayBypassAfterHover: true, } : undefined, ); - return this.rememberHover(result); + return this.rememberHover((req.params as HoverParams).session_id, result); } case "tool.fill": - return handleFill( - this.sessions, - req.params as FillParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + return this.withHoverReleaseForRequest(req.params as FillParams, () => + handleFill( + this.sessions, + req.params as FillParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), ); case "tool.press": - return handlePress( - this.sessions, - req.params as PressParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + return this.withHoverReleaseForRequest(req.params as PressParams, () => + handlePress( + this.sessions, + req.params as PressParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), ); case "tool.select": - return handleSelect( - this.sessions, - req.params as SelectParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + return this.withHoverReleaseForRequest(req.params as SelectParams, () => + handleSelect( + this.sessions, + req.params as SelectParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), ); case "tool.evaluate": return handleEvaluate( @@ -500,11 +527,11 @@ export class ToolDispatcher { } } - private async setHoverBypass(tabId: number, enabled: boolean): Promise { + private async setHoverBypass(sessionId: string, tabId: number, enabled: boolean): Promise { if (enabled) { if (this.hoverBypassTabs.has(tabId)) return; await bypassOverlay(tabId, true); - this.hoverBypassTabs.add(tabId); + this.hoverBypassTabs.set(tabId, sessionId); } else { if (!this.hoverBypassTabs.has(tabId)) return; await bypassOverlay(tabId, false); @@ -512,51 +539,84 @@ export class ToolDispatcher { } } - private rememberHover(result: HoverResult | RpcError): HoverResult | RpcError { + private rememberHover(sessionId: string, result: HoverResult | RpcError): HoverResult | RpcError { if (!isRpcError(result)) { - this.hoverLatches.set(result.tab_id, { x: result.x, y: result.y }); + this.hoverLatches.set(result.tab_id, { + sessionId, + tabId: result.tab_id, + x: result.x, + y: result.y, + }); } return result; } - private hasHoverLatch(): boolean { - return this.hoverLatches.size > 0; + private hasHoverLatchForRequest(params: { session_id: string; tab_id?: number }): boolean { + return this.hoverLatchesForRequest(params).length > 0; } private async withHoverReassert( + params: { session_id: string; tab_id?: number }, work: () => Promise, options: { releaseAfter?: boolean } = {}, ): Promise { - await this.reassertHover(); + await this.reassertHover(params); try { return await work(); } finally { if (options.releaseAfter) { - await this.releaseHoverLatch(); + await this.releaseHoverLatch(params.session_id, params.tab_id); } } } - private async reassertHover(): Promise { + private async withHoverReleaseForRequest( + params: { session_id: string; tab_id?: number }, + work: () => Promise, + ): Promise { + await this.releaseHoverLatch(params.session_id, params.tab_id); + return work(); + } + + private hoverLatchesForRequest(params: { session_id: string; tab_id?: number }): HoverLatch[] { + return [...this.hoverLatches.values()].filter((latch) => { + if (latch.sessionId !== params.session_id) return false; + return params.tab_id === undefined || latch.tabId === params.tab_id; + }); + } + + private async reassertHover(params: { session_id: string; tab_id?: number }): Promise { if (!this.cdp) return; await Promise.all( - [...this.hoverLatches.entries()].map(([tabId, point]) => - this.cdp!.send(tabId, "Input.dispatchMouseEvent", { + this.hoverLatchesForRequest(params).map((latch) => + this.cdp!.send(latch.tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", - x: point.x, - y: point.y, + x: latch.x, + y: latch.y, }).catch((err) => { console.debug("[bsk dispatcher] hover reassert failed", err); - this.hoverLatches.delete(tabId); + this.hoverLatches.delete(latch.tabId); }), ), ); } - private async releaseHoverLatch(): Promise { - const tabs = new Set([...this.hoverBypassTabs, ...this.hoverLatches.keys()]); - this.hoverBypassTabs.clear(); - this.hoverLatches.clear(); + private async releaseHoverLatch(sessionId?: string, tabId?: number): Promise { + const matchesScope = (entrySessionId: string, entryTabId: number): boolean => { + if (sessionId !== undefined && entrySessionId !== sessionId) return false; + return tabId === undefined || entryTabId === tabId; + }; + const tabs = new Set(); + for (const [bypassTabId, bypassSessionId] of this.hoverBypassTabs) { + if (!matchesScope(bypassSessionId, bypassTabId)) continue; + tabs.add(bypassTabId); + this.hoverBypassTabs.delete(bypassTabId); + } + for (const latch of this.hoverLatches.values()) { + if (!matchesScope(latch.sessionId, latch.tabId)) continue; + tabs.add(latch.tabId); + this.hoverLatches.delete(latch.tabId); + } await Promise.all([...tabs].map((tabId) => bypassOverlay(tabId, false))); } } From cde6f22ecf05d2068992b77f4e8245bfadc08f87 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Thu, 6 Aug 2026 16:14:18 +0800 Subject: [PATCH 04/12] fix(extension): ci and bug fix --- .../src/lib/recording-step-buffer.ts | 8 +++ .../src/tools/__tests__/dispatcher.test.ts | 16 ++--- .../src/tools/__tests__/observation.test.ts | 63 +++++++++++++++++++ apps/extension/src/tools/observation.ts | 1 + 4 files changed, 81 insertions(+), 7 deletions(-) diff --git a/apps/extension/src/lib/recording-step-buffer.ts b/apps/extension/src/lib/recording-step-buffer.ts index 0c7c26e..c1c5086 100644 --- a/apps/extension/src/lib/recording-step-buffer.ts +++ b/apps/extension/src/lib/recording-step-buffer.ts @@ -21,6 +21,14 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null { ...(pageUrl ? { page_url: pageUrl } : {}), } : null; + case "hover": + return payload.target + ? { + op: "hover", + target: payload.target, + ...(pageUrl ? { page_url: pageUrl } : {}), + } + : null; case "fill": return payload.target ? { diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index ec2c84c..30cc60b 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -9,6 +9,8 @@ import type { } from "@/transport/types"; import { ToolDispatcher } from "../dispatcher"; +type TestDispatcherCdp = NonNullable[0]["cdp"]>; + function fakeTransport() { const handlers = new Set(); const stateHandlers = new Set(); @@ -129,7 +131,7 @@ describe("ToolDispatcher", () => { setUserAgentOverride: vi.fn(async () => {}), setTouchEmulationEnabled: vi.fn(async () => {}), }; - const dispatcher = new ToolDispatcher({ transport, sessions, cdp }); + const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); dispatcher.start(); deliver(makeRequest("tool.console", { session_id: "aa11" })); @@ -172,7 +174,7 @@ describe("ToolDispatcher", () => { setUserAgentOverride: vi.fn(async () => {}), setTouchEmulationEnabled: vi.fn(async () => {}), }; - const dispatcher = new ToolDispatcher({ transport, sessions, cdp }); + const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); dispatcher.start(); deliver(makeRequest("tool.session_stop", { session_id: "aa11" })); @@ -327,9 +329,9 @@ describe("ToolDispatcher", () => { }); const order: string[] = []; const cdp = { - send: vi.fn(async () => { + send: vi.fn(async () => { order.push("hover"); - return {}; + return {} as T; }), detachSession: vi.fn(async () => {}), ensureNetworkCapture: vi.fn(async () => {}), @@ -344,7 +346,7 @@ describe("ToolDispatcher", () => { setUserAgentOverride: vi.fn(async () => {}), setTouchEmulationEnabled: vi.fn(async () => {}), }; - const dispatcher = new ToolDispatcher({ transport, sessions, cdp }); + const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); ( dispatcher as unknown as { rememberHover: (sessionId: string, result: unknown) => unknown } @@ -408,7 +410,7 @@ describe("ToolDispatcher", () => { }, }); const cdp = { - send: vi.fn(async () => ({})), + send: vi.fn(async () => ({} as T)), detachSession: vi.fn(async () => {}), ensureNetworkCapture: vi.fn(async () => {}), networkEntriesSince: vi.fn(() => ({ @@ -422,7 +424,7 @@ describe("ToolDispatcher", () => { setUserAgentOverride: vi.fn(async () => {}), setTouchEmulationEnabled: vi.fn(async () => {}), }; - const dispatcher = new ToolDispatcher({ transport, sessions, cdp }); + const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); const helpers = dispatcher as unknown as { rememberHover: (sessionId: string, result: unknown) => unknown; setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise; diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 9d9e003..89a4ec3 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -1543,6 +1543,69 @@ describe("buildVomScene", () => { expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]'); }); + it("deduplicates hover surface probes by original trigger backend id", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 10, + childIds: ["2"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "image" }, + backendDOMNodeId: 21, + }, + ]; + const scene = buildVomScene(axNodes, { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + surfaceProbes: [ + { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] }, + { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] }, + ], + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "div", + attrs: { class: "tg-avatar" }, + rect: { x: 900, y: 10, w: 30, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 21, + parentBackendNodeId: 20, + tag: "div", + attrs: { class: "tg-avatar__inner" }, + rect: { x: 902, y: 12, w: 26, h: 26 }, + paintOrder: 2, + position: "static", + pointerEvents: "auto", + }, + ], + }); + + expect(scene.surfaces).toEqual([ + { triggerId: 21, triggerAction: "hover", subItems: ["My profile"] }, + ]); + }); + it("maps hover surface probes to DOM-recovered custom controls", () => { const axNodes: CdpAxNode[] = [ { diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 59d39ff..a1da513 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -927,6 +927,7 @@ function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel) probe.triggerPoint, ); if (!match) continue; + used.add(probe.triggerBackendNodeId); used.add(match.id); surfaces.push({ triggerId: match.id, From 9d4ac9059213154b673f9455f2361b7dd3b9bc0b Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Thu, 6 Aug 2026 16:16:41 +0800 Subject: [PATCH 05/12] fix(extension): ci and bug fix --- apps/extension/src/tools/__tests__/dispatcher.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 30cc60b..c4f7a86 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -329,7 +329,7 @@ describe("ToolDispatcher", () => { }); const order: string[] = []; const cdp = { - send: vi.fn(async () => { + send: vi.fn(async () => { order.push("hover"); return {} as T; }), @@ -410,7 +410,7 @@ describe("ToolDispatcher", () => { }, }); const cdp = { - send: vi.fn(async () => ({} as T)), + send: vi.fn(async () => ({}) as T), detachSession: vi.fn(async () => {}), ensureNetworkCapture: vi.fn(async () => {}), networkEntriesSince: vi.fn(() => ({ From 4ba6fb45e1a68468f9fc7fd6626bca15ee82cc13 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Thu, 6 Aug 2026 17:33:46 +0800 Subject: [PATCH 06/12] feat(extension): recording hover --- .../content/__tests__/record-capture.test.ts | 349 ++++++++++++++++++ apps/extension/src/content/record-capture.ts | 230 +++++++++++- .../src/content/record-hover-surface.ts | 102 +++++ .../__tests__/hover-trigger-policy.test.ts | 112 ++++++ .../extension/src/lib/hover-trigger-policy.ts | 166 +++++++++ .../src/tools/vom/__tests__/capture.test.ts | 4 +- apps/extension/src/tools/vom/capture.ts | 139 ++----- 7 files changed, 978 insertions(+), 124 deletions(-) create mode 100644 apps/extension/src/content/record-hover-surface.ts create mode 100644 apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts create mode 100644 apps/extension/src/lib/hover-trigger-policy.ts diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts index 38ac5d7..816f81d 100644 --- a/apps/extension/src/content/__tests__/record-capture.test.ts +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -156,4 +156,353 @@ describe("record-capture semantic", () => { target: { name: "下一步", role: "button" }, }); }); + + it("records a hover trigger before clicking its revealed menu item", () => { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-menu", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const item = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { role: "link", name: "My profile" }, + }); + }); + + it("records hover for compact topbar image buttons before menu item clicks", () => { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-topbar", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const item = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "image" }, + }); + }); + + it("records hover when the revealed menu item is inside the hover root", () => { + document.body.innerHTML = ` +
+ image + +
+ `; + const capture = startRecordCapture("rec-hover-contained-menu", (step) => steps.push(step)); + const trigger = document.querySelector('[role="button"]')!; + const item = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "image" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { role: "link", name: "My profile" }, + }); + }); + + it("does not let a revealed topbar menu link replace the pending hover trigger", () => { + document.body.innerHTML = ` + + image + + + `; + const capture = startRecordCapture("rec-hover-topbar-link", (step) => steps.push(step)); + const trigger = document.querySelector('a[aria-label="image"]')!; + const item = document.querySelector("ul a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(item, "getBoundingClientRect").mockReturnValue({ + x: 820, + y: 72, + top: 72, + left: 820, + right: 916, + bottom: 104, + width: 96, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: el === item ? "pointer" : "", + pointerEvents: "auto", + } as CSSStyleDeclaration; + return style; + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "link", name: "image" }, + }); + }); + + it("keeps the first hover trigger over a later lower-score accepted hover", () => { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-latch", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const laterHover = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(laterHover, "getBoundingClientRect").mockReturnValue({ + x: 820, + y: 72, + top: 72, + left: 820, + right: 916, + bottom: 104, + width: 96, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: el === laterHover ? "pointer" : "", + pointerEvents: "auto", + } as CSSStyleDeclaration; + return style; + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + laterHover.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + laterHover.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { role: "button", name: "My profile" }, + }); + }); + + it("records avatar div hover with a semantic image target", () => { + document.body.innerHTML = ` +
+ +
+ + `; + const capture = startRecordCapture("rec-hover-avatar-div", (step) => steps.push(step)); + const trigger = document.querySelector(".tg-avatar")!; + const image = document.querySelector("img")!; + const item = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(image, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(window, "getComputedStyle").mockReturnValue({ + cursor: "pointer", + pointerEvents: "auto", + } as CSSStyleDeclaration); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + image.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { tag: "img", role: "img", name: "image" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { role: "link", name: "My profile" }, + }); + }); + + it("does not record hover before an unrelated page click", () => { + document.body.innerHTML = ` + +
+ Projects +
+ `; + const capture = startRecordCapture("rec-hover-unrelated-click", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const link = document.querySelector("main a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + link.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["click"]); + expect(steps[0]).toMatchObject({ + op: "click", + target: { role: "link", name: "Projects" }, + }); + }); + + it("does not record hover before clicking the same trigger", () => { + document.body.innerHTML = ` + + `; + const capture = startRecordCapture("rec-hover-same-trigger", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + trigger.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["click"]); + }); + + it("does not record hover for ordinary controls without popup signals", () => { + const capture = startRecordCapture("rec-hover-noise", (step) => steps.push(step)); + const button = document.querySelector("button")!; + + button.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + button.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["click"]); + }); }); diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index 8c9d5d0..a632154 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -1,9 +1,15 @@ import { describeEventTarget, describeTarget, + resolveClickableElement, resolveHoverElement, type TargetDescriptor, } from "@/lib/describe-target"; +import { + evaluateHoverTrigger, + type HoverTriggerDecision, + type HoverTriggerRect, +} from "@/lib/hover-trigger-policy"; import { isRecordCancelMessage, isRecordStartMessage, @@ -20,6 +26,7 @@ import { type RecordStopMessage, } from "@/lib/record-bridge"; import { shouldRecordPress } from "@/lib/trace-reducer"; +import { decideHoverSurfaceRelation } from "./record-hover-surface"; const pendingStepSends = new Map>>(); const failedStepDeliveries = new Set(); @@ -58,11 +65,15 @@ interface PendingHover { element: Element; target: TargetDescriptor; recordedAt: number; + score: number; + reasons: string[]; } type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement; const HOVER_BEFORE_CLICK_MAX_MS = 10_000; +const HOVER_REPLACE_SCORE_MARGIN = 50; +const RECORD_DEBUG_STORAGE_KEY = "bsk_record_debug"; function eventTarget(event: Event): EventTarget | null { return event.composedPath()[0] ?? event.target; @@ -135,12 +146,160 @@ function fillableValue(el: FillableElement): string { return el.textContent ?? ""; } +function recordDebug(message: string, data?: unknown): void { + try { + if (localStorage.getItem(RECORD_DEBUG_STORAGE_KEY) !== "1") return; + console.info("[bsk record]", message, data === undefined ? "" : JSON.stringify(data)); + } catch { + // Recording must not depend on page storage or console availability. + } +} + +function elementDebugSummary(el: Element): Record { + return { + tag: el.tagName.toLowerCase(), + id: el.id || undefined, + class: typeof el.className === "string" ? el.className || undefined : undefined, + role: el.getAttribute("role") ?? undefined, + ariaLabel: el.getAttribute("aria-label") ?? undefined, + text: el.textContent?.replace(/\s+/g, " ").trim().slice(0, 80) || undefined, + }; +} + +function hoverTriggerAttrs(el: Element): Record { + return { + id: el.id || undefined, + class: typeof el.className === "string" ? el.className || undefined : undefined, + "data-testid": el.getAttribute("data-testid") ?? undefined, + "data-test": el.getAttribute("data-test") ?? undefined, + "data-cy": el.getAttribute("data-cy") ?? undefined, + "aria-label": el.getAttribute("aria-label") ?? undefined, + "aria-haspopup": el.getAttribute("aria-haspopup") ?? undefined, + "aria-controls": el.getAttribute("aria-controls") ?? undefined, + "aria-expanded": el.getAttribute("aria-expanded") ?? undefined, + "aria-hidden": el.getAttribute("aria-hidden") ?? undefined, + "aria-disabled": el.getAttribute("aria-disabled") ?? undefined, + contenteditable: el.getAttribute("contenteditable") ?? undefined, + disabled: el.hasAttribute("disabled") ? "" : undefined, + hidden: el.hasAttribute("hidden") ? "" : undefined, + inert: el.hasAttribute("inert") ? "" : undefined, + onclick: el.getAttribute("onclick") ?? undefined, + onmouseenter: el.getAttribute("onmouseenter") ?? undefined, + onmouseover: el.getAttribute("onmouseover") ?? undefined, + role: el.getAttribute("role") ?? undefined, + tabindex: el.getAttribute("tabindex") ?? undefined, + title: el.getAttribute("title") ?? undefined, + }; +} + +function hoverTriggerRect(el: Element): HoverTriggerRect { + const rect = el.getBoundingClientRect(); + return { x: rect.left, y: rect.top, w: rect.width, h: rect.height }; +} + +function hoverTriggerStyle(el: Element): { cursor?: string; pointerEvents?: string } { + if (!(el instanceof HTMLElement)) return {}; + const style = getComputedStyle(el); + return { cursor: style.cursor, pointerEvents: style.pointerEvents }; +} + +function hasHoverGraphicDescendant(el: Element): boolean { + return el.querySelector("img,svg,use,path,i") !== null; +} + +function isWeakHoverTarget(target: TargetDescriptor): boolean { + return !target.role && !target.name && target.tag === "div"; +} + +function looksLikeAvatarElement(el: Element): boolean { + const className = typeof el.className === "string" ? el.className : ""; + return /\b(avatar|user-avatar)\b/i.test(className); +} + +function normalizeHoverTarget( + el: Element, + desc: TargetDescriptor, + decision: HoverTriggerDecision, +): TargetDescriptor { + if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) { + return { ...desc, name: "image" }; + } + if ( + isWeakHoverTarget(desc) && + (looksLikeAvatarElement(el) || + (hasHoverGraphicDescendant(el) && decision.reasons.includes("icon-only"))) + ) { + return { tag: "img", role: "img", name: "image" }; + } + return desc; +} + +function evaluateHoverTriggerElement( + el: Element, + desc: TargetDescriptor, +): HoverTriggerDecision | null { + if (!(el instanceof HTMLElement)) return null; + const style = hoverTriggerStyle(el); + return evaluateHoverTrigger({ + tag: el.tagName.toLowerCase(), + role: desc.role, + label: desc.name, + attrs: hoverTriggerAttrs(el), + rect: hoverTriggerRect(el), + cursor: style.cursor, + pointerEvents: style.pointerEvents, + hasGraphicDescendant: hasHoverGraphicDescendant(el), + }); +} + function hoverTargetFromEvent(target: EventTarget | null): PendingHover | null { - if (!(target instanceof Element)) return null; + if (!(target instanceof Element)) { + recordDebug("hover ignored: event target is not an Element", { target }); + return null; + } const element = resolveHoverElement(target); - if (!element) return null; + if (!element) { + recordDebug("hover ignored: no resolved hover element", { target }); + return null; + } const desc = describeTarget(element); - return { element, target: desc, recordedAt: Date.now() }; + const decision = evaluateHoverTriggerElement(element, desc); + if (!decision?.eligible) { + recordDebug("hover ignored: rejected by trigger policy", { + desc, + score: decision?.score ?? 0, + reasons: decision?.reasons ?? [], + element: elementDebugSummary(element), + }); + return null; + } + const normalizedTarget = normalizeHoverTarget(element, desc, decision); + recordDebug("hover accepted", { + desc: normalizedTarget, + raw_desc: desc, + score: decision.score, + reasons: decision.reasons, + element: elementDebugSummary(element), + }); + return { + element, + target: normalizedTarget, + recordedAt: Date.now(), + score: decision.score, + reasons: decision.reasons, + }; +} + +function shouldReplacePendingHover( + current: PendingHover, + next: PendingHover, + now: number, +): boolean { + if (next.element === current.element) return false; + if (now - current.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) return true; + if (current.element.contains(next.element)) return false; + if (next.element.contains(current.element)) return true; + return next.score >= current.score + HOVER_REPLACE_SCORE_MARGIN; } /** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */ @@ -186,6 +345,7 @@ export function startRecordCapture( _requestId: string, sendStep: (step: RecordStepPayload) => void, ): RecordCaptureController { + recordDebug("record capture started", { page_url: location.href }); const emitStep = (step: RecordStepPayload) => { sendStep({ page_url: location.href, ...step }); }; @@ -277,7 +437,36 @@ export function startRecordCapture( pendingHover = null; return; } - if (pendingHover.element.contains(clickTarget)) return; + const clickElement = resolveClickableElement(clickTarget); + if (clickElement === pendingHover.element) { + recordDebug("pending hover skipped: click resolved to same trigger", { + target: pendingHover.target, + }); + return; + } + const relationTarget = clickElement ?? clickTarget; + if (!clickElement && pendingHover.element.contains(relationTarget)) { + recordDebug("pending hover skipped: anonymous click inside trigger", { + target: pendingHover.target, + }); + return; + } + const relation = decideHoverSurfaceRelation( + { + triggerElement: pendingHover.element, + triggerTarget: pendingHover.target, + }, + relationTarget, + ); + if (!relation.related) { + recordDebug("pending hover skipped: click outside hover surface", { + target: pendingHover.target, + reason: relation.reason, + }); + pendingHover = null; + return; + } + recordDebug("pending hover emitted before click", { target: pendingHover.target }); emitStep({ op: "hover", target: pendingHover.target, @@ -289,7 +478,37 @@ export function startRecordCapture( const target = eventTarget(event); if (isOverlayTarget(target)) return; const hover = hoverTargetFromEvent(target); - if (!hover || hover.element === pendingHover?.element) return; + if (!hover) return; + const now = Date.now(); + if (pendingHover && !shouldReplacePendingHover(pendingHover, hover, now)) { + recordDebug("pending hover kept over later hover", { + current: { + target: pendingHover.target, + score: pendingHover.score, + reasons: pendingHover.reasons, + }, + next: { + target: hover.target, + score: hover.score, + reasons: hover.reasons, + }, + }); + return; + } + if (pendingHover) { + recordDebug("pending hover replaced", { + current: { + target: pendingHover.target, + score: pendingHover.score, + reasons: pendingHover.reasons, + }, + next: { + target: hover.target, + score: hover.score, + reasons: hover.reasons, + }, + }); + } pendingHover = hover; }; @@ -482,6 +701,7 @@ export function startRecordCapture( return { dispose() { + recordDebug("record capture disposed", { page_url: location.href }); commitFillSession(); document.removeEventListener("click", onClick, true); document.removeEventListener("mouseover", onMouseOver, true); diff --git a/apps/extension/src/content/record-hover-surface.ts b/apps/extension/src/content/record-hover-surface.ts new file mode 100644 index 0000000..0f1a7d9 --- /dev/null +++ b/apps/extension/src/content/record-hover-surface.ts @@ -0,0 +1,102 @@ +import type { TargetDescriptor } from "@/lib/describe-target"; + +export interface HoverSurfaceDecision { + related: boolean; + reason?: string; + surface?: Element; +} + +export interface HoverSurfaceContext { + triggerElement: Element; + triggerTarget: TargetDescriptor; +} + +const HOVER_SURFACE_SELECTOR = [ + '[role="menu"]', + '[role="menubar"]', + '[role="listbox"]', + '[role="dialog"]', + '[role="tooltip"]', + '[aria-modal="true"]', + "[data-popper-placement]", + "[data-floating-ui-placement]", + "[data-headlessui-state]", + "[data-radix-popper-content-wrapper]", +].join(","); + +const HOVER_SURFACE_WORD_RE = + /\b(menu|dropdown|drop-down|popover|popper|popup|tooltip|flyout|submenu|context-menu|account-menu|user-menu|avatar-menu)\b/i; + +function elementTokenText(el: Element): string { + return [ + el.id, + typeof el.className === "string" ? el.className : "", + el.getAttribute("data-testid"), + el.getAttribute("data-test"), + el.getAttribute("data-cy"), + el.getAttribute("aria-label"), + el.getAttribute("role"), + ] + .filter((value): value is string => typeof value === "string" && value.length > 0) + .join(" "); +} + +function isHoverSurfaceElement(el: Element): boolean { + if (el.matches(HOVER_SURFACE_SELECTOR)) return true; + if (HOVER_SURFACE_WORD_RE.test(elementTokenText(el))) return true; + const tag = el.tagName.toLowerCase(); + if ((tag === "ul" || tag === "ol") && el.querySelector("a,button,[role='menuitem']")) { + return HOVER_SURFACE_WORD_RE.test(elementTokenText(el.parentElement ?? el)); + } + return false; +} + +function closestHoverSurface(clickElement: Element): Element | null { + let node: Element | null = clickElement; + let depth = 0; + while (node && node !== document.body && node !== document.documentElement && depth < 10) { + if (isHoverSurfaceElement(node)) return node; + node = node.parentElement; + depth += 1; + } + return null; +} + +function rectFor(el: Element): DOMRect | null { + const rect = el.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) return null; + return rect; +} + +function isNearTrigger(surface: Element, trigger: Element): boolean { + const surfaceRect = rectFor(surface); + const triggerRect = rectFor(trigger); + if (!surfaceRect || !triggerRect) return true; + const horizontalGap = Math.max( + 0, + triggerRect.left - surfaceRect.right, + surfaceRect.left - triggerRect.right, + ); + const verticalGap = Math.max( + 0, + triggerRect.top - surfaceRect.bottom, + surfaceRect.top - triggerRect.bottom, + ); + return horizontalGap <= 320 && verticalGap <= 320; +} + +export function decideHoverSurfaceRelation( + context: HoverSurfaceContext, + clickElement: Element, +): HoverSurfaceDecision { + const clickSurface = closestHoverSurface(clickElement); + if (clickSurface && isNearTrigger(clickSurface, context.triggerElement)) { + return { related: true, reason: "click-inside-hover-surface", surface: clickSurface }; + } + + if (context.triggerElement.contains(clickElement) && clickElement !== context.triggerElement) { + return { related: true, reason: "click-inside-trigger-contained-surface" }; + } + + return { related: false, reason: "click-outside-hover-surface" }; +} diff --git a/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts b/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts new file mode 100644 index 0000000..d3b3dce --- /dev/null +++ b/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from "vitest"; +import { evaluateHoverTrigger } from "../hover-trigger-policy"; + +describe("evaluateHoverTrigger", () => { + it("accepts explicit popup triggers", () => { + const decision = evaluateHoverTrigger({ + tag: "button", + role: "button", + label: "Open user navigation menu", + attrs: { "aria-haspopup": "menu", "aria-expanded": "false" }, + rect: { x: 900, y: 8, w: 32, h: 32 }, + cursor: "pointer", + pointerEvents: "auto", + }); + + expect(decision.eligible).toBe(true); + expect(decision.reasons).toEqual( + expect.arrayContaining(["aria-haspopup", "collapsed", "popup-signal"]), + ); + }); + + it("accepts compact topbar image buttons without popup attributes", () => { + const decision = evaluateHoverTrigger({ + tag: "button", + role: "button", + label: "image", + attrs: {}, + rect: { x: 900, y: 8, w: 32, h: 32 }, + pointerEvents: "auto", + hasGraphicDescendant: true, + }); + + expect(decision.eligible).toBe(true); + expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"])); + }); + + it("accepts compact topbar image links without popup attributes", () => { + const decision = evaluateHoverTrigger({ + tag: "a", + role: "link", + label: "image", + attrs: {}, + rect: { x: 900, y: 8, w: 32, h: 32 }, + pointerEvents: "auto", + hasGraphicDescendant: true, + }); + + expect(decision.eligible).toBe(true); + expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"])); + }); + + it("rejects topbar menu links without surface trigger evidence", () => { + const decision = evaluateHoverTrigger({ + tag: "a", + role: "link", + label: "My profile", + attrs: {}, + rect: { x: 820, y: 72, w: 96, h: 32 }, + cursor: "pointer", + pointerEvents: "auto", + }); + + expect(decision.eligible).toBe(false); + }); + + it("rejects text navigation items even when their attributes contain popup words", () => { + const decision = evaluateHoverTrigger({ + tag: "a", + role: "link", + label: "My profile", + attrs: { class: "dropdown-item profile-link" }, + rect: { x: 820, y: 72, w: 96, h: 32 }, + cursor: "pointer", + pointerEvents: "auto", + }); + + expect(decision.eligible).toBe(false); + }); + + it("rejects ordinary controls without hover popup signals", () => { + const decision = evaluateHoverTrigger({ + tag: "button", + role: "button", + label: "Search", + attrs: {}, + rect: { x: 100, y: 240, w: 120, h: 36 }, + pointerEvents: "auto", + }); + + expect(decision.eligible).toBe(false); + }); + + it("rejects unsafe or non-visible triggers", () => { + expect( + evaluateHoverTrigger({ + tag: "input", + attrs: {}, + rect: { x: 10, y: 10, w: 80, h: 24 }, + pointerEvents: "auto", + }).eligible, + ).toBe(false); + expect( + evaluateHoverTrigger({ + tag: "button", + role: "button", + attrs: { hidden: "" }, + rect: { x: 10, y: 10, w: 80, h: 24 }, + pointerEvents: "auto", + }).eligible, + ).toBe(false); + }); +}); diff --git a/apps/extension/src/lib/hover-trigger-policy.ts b/apps/extension/src/lib/hover-trigger-policy.ts new file mode 100644 index 0000000..8701c89 --- /dev/null +++ b/apps/extension/src/lib/hover-trigger-policy.ts @@ -0,0 +1,166 @@ +export interface HoverTriggerRect { + x: number; + y: number; + w: number; + h: number; +} + +export interface HoverTriggerSignals { + tag: string; + role?: string; + label?: string; + attrs?: Record; + rect?: HoverTriggerRect | null; + cursor?: string; + pointerEvents?: string; + hasGraphicDescendant?: boolean; + cssHoverMatch?: boolean; +} + +export interface HoverTriggerDecision { + eligible: boolean; + score: number; + reasons: string[]; +} + +const HOVER_TRIGGER_MIN_SCORE = 45; +const HOVER_POPUP_SIGNAL_RE = + /(menu|dropdown|popover|popup|avatar|profile|account|user|more|ellipsis|caret)/; + +function attr(signals: HoverTriggerSignals, name: string): string | undefined { + return signals.attrs?.[name.toLowerCase()]; +} + +export function isUnsafeHoverTrigger(signals: HoverTriggerSignals): boolean { + const tag = signals.tag.toLowerCase(); + if (["input", "textarea", "select", "option"].includes(tag)) return true; + if ((attr(signals, "contenteditable") ?? "").toLowerCase() === "true") return true; + if (attr(signals, "disabled") !== undefined || attr(signals, "inert") !== undefined) return true; + return (attr(signals, "aria-disabled") ?? "").toLowerCase() === "true"; +} + +export function hasHoverPopupSignal(signals: HoverTriggerSignals): boolean { + const attrs = signals.attrs ?? {}; + const haystack = [ + attrs.id, + attrs.class, + attrs["data-testid"], + attrs["data-test"], + attrs["data-cy"], + attrs["aria-label"], + attrs["aria-haspopup"], + attrs["aria-controls"], + attrs.title, + signals.role === "button" ? signals.label : undefined, + ] + .filter(Boolean) + .join(" ") + .toLowerCase(); + return HOVER_POPUP_SIGNAL_RE.test(haystack); +} + +export function hasDirectHoverInteractiveSignal(signals: HoverTriggerSignals): boolean { + const tag = signals.tag.toLowerCase(); + const role = (signals.role ?? "").toLowerCase(); + return ( + ["button", "a", "summary"].includes(tag) || + ["button", "link", "menuitem", "tab", "combobox"].includes(role) || + attr(signals, "tabindex") !== undefined || + signals.cursor === "pointer" || + attr(signals, "onclick") !== undefined || + attr(signals, "onmouseenter") !== undefined || + attr(signals, "onmouseover") !== undefined || + attr(signals, "aria-haspopup") !== undefined || + attr(signals, "aria-expanded") === "false" + ); +} + +export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTriggerDecision { + const rect = signals.rect; + if ( + !rect || + rect.w <= 0 || + rect.h <= 0 || + signals.pointerEvents === "none" || + attr(signals, "hidden") !== undefined || + attr(signals, "inert") !== undefined || + (attr(signals, "aria-hidden") ?? "").toLowerCase() === "true" || + isUnsafeHoverTrigger(signals) + ) { + return { eligible: false, score: 0, reasons: [] }; + } + + const area = rect.w * rect.h; + if (area <= 0 || area > 160_000) { + return { eligible: false, score: 0, reasons: [] }; + } + + const reasons: string[] = []; + let score = 0; + const tag = signals.tag.toLowerCase(); + const role = (signals.role ?? "").toLowerCase(); + const label = signals.label?.trim(); + const graphic = signals.hasGraphicDescendant === true; + const directInteractive = hasDirectHoverInteractiveSignal(signals); + const popupSignal = hasHoverPopupSignal(signals); + const isTextNavigationItem = + (tag === "a" || role === "link" || role.startsWith("menuitem")) && + !graphic && + !!label && + label.toLowerCase() !== "image"; + const compactTopbarIcon = + (role === "button" || role === "link" || tag === "button" || tag === "a") && + graphic && + rect.y < 120 && + rect.w <= 96 && + rect.h <= 96 && + (!label || label.toLowerCase() === "image"); + const surfaceTriggerEvidence = + (popupSignal && !isTextNavigationItem) || + compactTopbarIcon || + signals.cssHoverMatch === true || + attr(signals, "aria-haspopup") !== undefined || + (attr(signals, "aria-expanded") ?? "").toLowerCase() === "false"; + if (!directInteractive || !surfaceTriggerEvidence) { + return { eligible: false, score: 0, reasons: [] }; + } + + if (attr(signals, "aria-haspopup") !== undefined) { + score += 80; + reasons.push("aria-haspopup"); + } + if ((attr(signals, "aria-expanded") ?? "").toLowerCase() === "false") { + score += 55; + reasons.push("collapsed"); + } + if (popupSignal) { + score += 45; + reasons.push("popup-signal"); + } + if (tag === "button" || role === "button") { + score += 30; + reasons.push("button"); + } + if (signals.cursor === "pointer") { + score += 25; + reasons.push("pointer"); + } + if (graphic && (!label || label.toLowerCase() === "image")) { + score += 40; + reasons.push("icon-only"); + } + if (rect.y < 120) { + score += 25; + reasons.push("topbar"); + } + if (rect.w <= 80 && rect.h <= 80) { + score += 20; + reasons.push("compact"); + } + if (signals.cssHoverMatch) { + score += 40; + reasons.push("css-hover"); + } + + return { eligible: score >= HOVER_TRIGGER_MIN_SCORE, score, reasons }; +} diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index 086e216..bc4cd2a 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -101,7 +101,7 @@ function twoHoverTriggerSnapshotReply() { "button", "class", "user-avatar", - "secondary-trigger", + "secondary-dropdown-trigger", "position", "static", "pointer-events", @@ -123,7 +123,7 @@ function twoHoverTriggerSnapshotReply() { [], [], [i("class"), i("user-avatar")], - [i("class"), i("secondary-trigger")], + [i("class"), i("secondary-dropdown-trigger")], ], }, layout: { diff --git a/apps/extension/src/tools/vom/capture.ts b/apps/extension/src/tools/vom/capture.ts index 44aba56..0f56577 100644 --- a/apps/extension/src/tools/vom/capture.ts +++ b/apps/extension/src/tools/vom/capture.ts @@ -5,6 +5,7 @@ // `styles` columns are [position, pointer-events, cursor] in that order. import type { Rect, Viewport } from "@browser-skill/vom"; +import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; import type { CdpRunner } from "../shared"; @@ -442,27 +443,6 @@ async function clearHover(cdp: CdpRunner, tabId: number): Promise { ); } -function isVisibleProbeNode(node: CapturedNode): boolean { - const rect = node.rect; - return ( - !!rect && - rect.w > 0 && - rect.h > 0 && - node.pointerEvents !== "none" && - node.attrs.hidden === undefined && - node.attrs.inert === undefined && - (node.attrs["aria-hidden"] ?? "").toLowerCase() !== "true" - ); -} - -function isUnsafeHoverTrigger(node: CapturedNode): boolean { - const tag = node.tag.toLowerCase(); - if (["input", "textarea", "select", "option"].includes(tag)) return true; - if ((node.attrs.contenteditable ?? "").toLowerCase() === "true") return true; - if (node.attrs.disabled !== undefined || node.attrs.inert !== undefined) return true; - return (node.attrs["aria-disabled"] ?? "").toLowerCase() === "true"; -} - function capturedText(node: CapturedNode): string | undefined { const value = node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? ""; @@ -484,120 +464,45 @@ function hasGraphicDescendant( return false; } -function hasHoverPopupSignal(node: CapturedNode): boolean { - const attrs = node.attrs; - const haystack = [ - attrs.id, - attrs.class, - attrs["data-testid"], - attrs["data-test"], - attrs["data-cy"], - attrs["aria-haspopup"], - attrs["aria-controls"], - ] - .filter(Boolean) - .join(" ") - .toLowerCase(); - return /(menu|dropdown|popover|popup|avatar|profile|account|user)/.test(haystack); -} - function roleOf(node: CapturedNode): string { return (node.attrs.role ?? "").toLowerCase(); } -function hasDirectInteractiveSignal(node: CapturedNode): boolean { - const tag = node.tag.toLowerCase(); - const role = roleOf(node); - return ( - ["button", "a", "summary"].includes(tag) || - ["button", "link", "menuitem", "tab", "combobox"].includes(role) || - node.attrs.tabindex !== undefined || - node.cursor === "pointer" || - node.attrs.onclick !== undefined || - node.attrs.onmouseenter !== undefined || - node.attrs.onmouseover !== undefined || - node.attrs["aria-haspopup"] !== undefined || - node.attrs["aria-expanded"] === "false" - ); -} - function scoreHoverCandidate( node: CapturedNode, childrenByParentId: Map, cssHoverPoints: Array<{ x: number; y: number }>, ): HoverCandidate | null { - if (!isVisibleProbeNode(node) || isUnsafeHoverTrigger(node)) { - return null; - } const rect = node.rect; if (!rect) return null; - const area = rect.w * rect.h; - if (area <= 0 || area > 160_000) return null; - - const reasons: string[] = []; - let score = 0; - const tag = node.tag.toLowerCase(); - const role = roleOf(node); const label = capturedText(node); - const graphic = hasGraphicDescendant(node, childrenByParentId); - const directInteractive = hasDirectInteractiveSignal(node); - const popupSignal = hasHoverPopupSignal(node); - const compactTopbarIcon = !label && graphic && rect.y < 120 && rect.w <= 96 && rect.h <= 96; - if (!directInteractive && !popupSignal && !compactTopbarIcon) return null; - - if (node.attrs["aria-haspopup"] !== undefined) { - score += 80; - reasons.push("aria-haspopup"); - } - if ((node.attrs["aria-expanded"] ?? "").toLowerCase() === "false") { - score += 55; - reasons.push("collapsed"); - } - if (popupSignal) { - score += 45; - reasons.push("popup-signal"); - } - if (tag === "button" || role === "button") { - score += 30; - reasons.push("button"); - } - if (node.cursor === "pointer") { - score += 25; - reasons.push("pointer"); - } - if (!label && graphic) { - score += 40; - reasons.push("icon-only"); - } - if (rect.y < 120) { - score += 25; - reasons.push("topbar"); - } - if (rect.w <= 80 && rect.h <= 80) { - score += 20; - reasons.push("compact"); - } - if ( - cssHoverPoints.some( - (point) => - point.x >= rect.x && - point.x <= rect.x + rect.w && - point.y >= rect.y && - point.y <= rect.y + rect.h, - ) - ) { - score += 40; - reasons.push("css-hover"); - } + const cssHoverMatch = cssHoverPoints.some( + (point) => + point.x >= rect.x && + point.x <= rect.x + rect.w && + point.y >= rect.y && + point.y <= rect.y + rect.h, + ); + const decision = evaluateHoverTrigger({ + tag: node.tag, + role: roleOf(node), + label, + attrs: node.attrs, + rect, + cursor: node.cursor, + pointerEvents: node.pointerEvents, + hasGraphicDescendant: hasGraphicDescendant(node, childrenByParentId), + cssHoverMatch, + }); - if (score < 45) return null; + if (!decision.eligible) return null; return { backendNodeId: node.backendNodeId, label, x: rect.x + rect.w / 2, y: rect.y + rect.h / 2, - score, - reasons, + score: decision.score, + reasons: decision.reasons, }; } From 73724617f94a13a562efa6aa4590e5a256429847 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Thu, 6 Aug 2026 17:41:33 +0800 Subject: [PATCH 07/12] feat(extension): recording hover with custom div --- .../content/__tests__/record-capture.test.ts | 144 ++++++++++++++++++ apps/extension/src/content/record-capture.ts | 88 +---------- .../src/content/record-hover-surface.ts | 43 +++++- 3 files changed, 186 insertions(+), 89 deletions(-) diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts index 816f81d..e59cda0 100644 --- a/apps/extension/src/content/__tests__/record-capture.test.ts +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -470,6 +470,150 @@ describe("record-capture semantic", () => { }); }); + it("records hover before clicking an unlabelled positioned floating surface", () => { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-plain-floating-surface", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const surface = document.querySelector("div")!; + const link = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(surface, "getBoundingClientRect").mockReturnValue({ + x: 820, + y: 48, + top: 48, + left: 820, + right: 980, + bottom: 128, + width: 160, + height: 80, + toJSON: () => ({}), + }); + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: "", + display: "block", + pointerEvents: "auto", + position: el === surface ? "absolute" : "static", + visibility: "visible", + } as CSSStyleDeclaration; + return style; + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + link.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + }); + + it("does not treat ordinary document-flow divs as hover surfaces", () => { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-plain-flow-surface", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const link = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(window, "getComputedStyle").mockReturnValue({ + cursor: "", + display: "block", + pointerEvents: "auto", + position: "static", + visibility: "visible", + } as CSSStyleDeclaration); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + link.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["click"]); + }); + + it("does not treat unlabelled sticky containers as hover surfaces", () => { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-sticky-surface", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const surface = document.querySelector("div")!; + const link = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(surface, "getBoundingClientRect").mockReturnValue({ + x: 820, + y: 48, + top: 48, + left: 820, + right: 980, + bottom: 128, + width: 160, + height: 80, + toJSON: () => ({}), + }); + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: "", + display: "block", + pointerEvents: "auto", + position: el === surface ? "sticky" : "static", + visibility: "visible", + } as CSSStyleDeclaration; + return style; + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + link.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["click"]); + }); + it("does not record hover before clicking the same trigger", () => { document.body.innerHTML = ` diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index a632154..c879916 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -73,7 +73,6 @@ type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement; const HOVER_BEFORE_CLICK_MAX_MS = 10_000; const HOVER_REPLACE_SCORE_MARGIN = 50; -const RECORD_DEBUG_STORAGE_KEY = "bsk_record_debug"; function eventTarget(event: Event): EventTarget | null { return event.composedPath()[0] ?? event.target; @@ -146,26 +145,6 @@ function fillableValue(el: FillableElement): string { return el.textContent ?? ""; } -function recordDebug(message: string, data?: unknown): void { - try { - if (localStorage.getItem(RECORD_DEBUG_STORAGE_KEY) !== "1") return; - console.info("[bsk record]", message, data === undefined ? "" : JSON.stringify(data)); - } catch { - // Recording must not depend on page storage or console availability. - } -} - -function elementDebugSummary(el: Element): Record { - return { - tag: el.tagName.toLowerCase(), - id: el.id || undefined, - class: typeof el.className === "string" ? el.className || undefined : undefined, - role: el.getAttribute("role") ?? undefined, - ariaLabel: el.getAttribute("aria-label") ?? undefined, - text: el.textContent?.replace(/\s+/g, " ").trim().slice(0, 80) || undefined, - }; -} - function hoverTriggerAttrs(el: Element): Record { return { id: el.id || undefined, @@ -253,34 +232,13 @@ function evaluateHoverTriggerElement( } function hoverTargetFromEvent(target: EventTarget | null): PendingHover | null { - if (!(target instanceof Element)) { - recordDebug("hover ignored: event target is not an Element", { target }); - return null; - } + if (!(target instanceof Element)) return null; const element = resolveHoverElement(target); - if (!element) { - recordDebug("hover ignored: no resolved hover element", { target }); - return null; - } + if (!element) return null; const desc = describeTarget(element); const decision = evaluateHoverTriggerElement(element, desc); - if (!decision?.eligible) { - recordDebug("hover ignored: rejected by trigger policy", { - desc, - score: decision?.score ?? 0, - reasons: decision?.reasons ?? [], - element: elementDebugSummary(element), - }); - return null; - } + if (!decision?.eligible) return null; const normalizedTarget = normalizeHoverTarget(element, desc, decision); - recordDebug("hover accepted", { - desc: normalizedTarget, - raw_desc: desc, - score: decision.score, - reasons: decision.reasons, - element: elementDebugSummary(element), - }); return { element, target: normalizedTarget, @@ -345,7 +303,6 @@ export function startRecordCapture( _requestId: string, sendStep: (step: RecordStepPayload) => void, ): RecordCaptureController { - recordDebug("record capture started", { page_url: location.href }); const emitStep = (step: RecordStepPayload) => { sendStep({ page_url: location.href, ...step }); }; @@ -439,34 +396,22 @@ export function startRecordCapture( } const clickElement = resolveClickableElement(clickTarget); if (clickElement === pendingHover.element) { - recordDebug("pending hover skipped: click resolved to same trigger", { - target: pendingHover.target, - }); return; } const relationTarget = clickElement ?? clickTarget; if (!clickElement && pendingHover.element.contains(relationTarget)) { - recordDebug("pending hover skipped: anonymous click inside trigger", { - target: pendingHover.target, - }); return; } const relation = decideHoverSurfaceRelation( { triggerElement: pendingHover.element, - triggerTarget: pendingHover.target, }, relationTarget, ); if (!relation.related) { - recordDebug("pending hover skipped: click outside hover surface", { - target: pendingHover.target, - reason: relation.reason, - }); pendingHover = null; return; } - recordDebug("pending hover emitted before click", { target: pendingHover.target }); emitStep({ op: "hover", target: pendingHover.target, @@ -481,34 +426,8 @@ export function startRecordCapture( if (!hover) return; const now = Date.now(); if (pendingHover && !shouldReplacePendingHover(pendingHover, hover, now)) { - recordDebug("pending hover kept over later hover", { - current: { - target: pendingHover.target, - score: pendingHover.score, - reasons: pendingHover.reasons, - }, - next: { - target: hover.target, - score: hover.score, - reasons: hover.reasons, - }, - }); return; } - if (pendingHover) { - recordDebug("pending hover replaced", { - current: { - target: pendingHover.target, - score: pendingHover.score, - reasons: pendingHover.reasons, - }, - next: { - target: hover.target, - score: hover.score, - reasons: hover.reasons, - }, - }); - } pendingHover = hover; }; @@ -701,7 +620,6 @@ export function startRecordCapture( return { dispose() { - recordDebug("record capture disposed", { page_url: location.href }); commitFillSession(); document.removeEventListener("click", onClick, true); document.removeEventListener("mouseover", onMouseOver, true); diff --git a/apps/extension/src/content/record-hover-surface.ts b/apps/extension/src/content/record-hover-surface.ts index 0f1a7d9..c00d6fc 100644 --- a/apps/extension/src/content/record-hover-surface.ts +++ b/apps/extension/src/content/record-hover-surface.ts @@ -1,5 +1,3 @@ -import type { TargetDescriptor } from "@/lib/describe-target"; - export interface HoverSurfaceDecision { related: boolean; reason?: string; @@ -8,7 +6,6 @@ export interface HoverSurfaceDecision { export interface HoverSurfaceContext { triggerElement: Element; - triggerTarget: TargetDescriptor; } const HOVER_SURFACE_SELECTOR = [ @@ -26,6 +23,8 @@ const HOVER_SURFACE_SELECTOR = [ const HOVER_SURFACE_WORD_RE = /\b(menu|dropdown|drop-down|popover|popper|popup|tooltip|flyout|submenu|context-menu|account-menu|user-menu|avatar-menu)\b/i; +const HOVER_SURFACE_MAX_AXIS_GAP = 96; +const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5; function elementTokenText(el: Element): string { return [ @@ -62,6 +61,33 @@ function closestHoverSurface(clickElement: Element): Element | null { return null; } +function isPositionedFloatingElement(el: Element): boolean { + if (!(el instanceof HTMLElement)) return false; + const style = getComputedStyle(el); + if (!["absolute", "fixed"].includes(style.position)) return false; + if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") { + return false; + } + const rect = rectFor(el); + if (!rect) return false; + const viewportArea = window.innerWidth * window.innerHeight; + const area = rect.width * rect.height; + return ( + area > 0 && (!viewportArea || area <= viewportArea * HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO) + ); +} + +function closestFloatingSurface(clickElement: Element): Element | null { + let node: Element | null = clickElement; + let depth = 0; + while (node && node !== document.body && node !== document.documentElement && depth < 10) { + if (isPositionedFloatingElement(node)) return node; + node = node.parentElement; + depth += 1; + } + return null; +} + function rectFor(el: Element): DOMRect | null { const rect = el.getBoundingClientRect(); if (rect.width <= 0 || rect.height <= 0) return null; @@ -82,7 +108,7 @@ function isNearTrigger(surface: Element, trigger: Element): boolean { triggerRect.top - surfaceRect.bottom, surfaceRect.top - triggerRect.bottom, ); - return horizontalGap <= 320 && verticalGap <= 320; + return horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP; } export function decideHoverSurfaceRelation( @@ -94,6 +120,15 @@ export function decideHoverSurfaceRelation( return { related: true, reason: "click-inside-hover-surface", surface: clickSurface }; } + const floatingSurface = closestFloatingSurface(clickElement); + if (floatingSurface && isNearTrigger(floatingSurface, context.triggerElement)) { + return { + related: true, + reason: "click-inside-near-floating-surface", + surface: floatingSurface, + }; + } + if (context.triggerElement.contains(clickElement) && clickElement !== context.triggerElement) { return { related: true, reason: "click-inside-trigger-contained-surface" }; } From d685c75d25a769f1883688a2f8108df241da08d5 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Thu, 6 Aug 2026 17:47:19 +0800 Subject: [PATCH 08/12] feat(extension): emit pending hover before actions --- .../content/__tests__/record-capture.test.ts | 86 +++++++++++++++++++ apps/extension/src/content/record-capture.ts | 18 ++-- .../src/tools/__tests__/dispatcher.test.ts | 54 ++++++++++++ apps/extension/src/tools/dispatcher.ts | 44 +++++++--- 4 files changed, 183 insertions(+), 19 deletions(-) diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts index e59cda0..26280f5 100644 --- a/apps/extension/src/content/__tests__/record-capture.test.ts +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -470,6 +470,92 @@ describe("record-capture semantic", () => { }); }); + it("records hover before filling a field inside the hover surface", () => { + document.body.innerHTML = ` + +
+ + +
+ `; + const capture = startRecordCapture("rec-hover-fill-surface", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const input = document.querySelector("input")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + input.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + input.dispatchEvent(new FocusEvent("focusin", { bubbles: true })); + input.value = "Ada"; + input.dispatchEvent(new Event("input", { bubbles: true })); + input.dispatchEvent(new FocusEvent("focusout", { bubbles: true })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "fill"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + expect(steps[1]).toMatchObject({ + op: "fill", + target: { tag: "input" }, + value: "Ada", + }); + }); + + it("records hover before selecting inside the hover surface", () => { + document.body.innerHTML = ` + +
+ +
+ `; + const capture = startRecordCapture("rec-hover-select-surface", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const select = document.querySelector("select")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + select.value = "away"; + select.dispatchEvent(new Event("change", { bubbles: true })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "select"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + expect(steps[1]).toMatchObject({ + op: "select", + target: { role: "combobox", name: "Status" }, + values: ["away"], + labels: ["Away"], + }); + }); + it("records hover before clicking an unlabelled positioned floating surface", () => { document.body.innerHTML = ` diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index c879916..b21edcb 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -388,18 +388,18 @@ export function startRecordCapture( }); }; - const emitPendingHoverBeforeClick = (clickTarget: EventTarget | null) => { - if (!pendingHover || !(clickTarget instanceof Element)) return; + const emitPendingHoverBeforeAction = (actionTarget: EventTarget | null) => { + if (!pendingHover || !(actionTarget instanceof Element)) return; if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) { pendingHover = null; return; } - const clickElement = resolveClickableElement(clickTarget); - if (clickElement === pendingHover.element) { + const actionElement = resolveClickableElement(actionTarget); + if (actionElement === pendingHover.element) { return; } - const relationTarget = clickElement ?? clickTarget; - if (!clickElement && pendingHover.element.contains(relationTarget)) { + const relationTarget = actionElement ?? actionTarget; + if (!actionElement && pendingHover.element.contains(relationTarget)) { return; } const relation = decideHoverSurfaceRelation( @@ -460,6 +460,7 @@ export function startRecordCapture( const fillable = fillableFromTarget(target); if (fillable) { + emitPendingHoverBeforeAction(fillable); ensureFillSession(fillable); return; } @@ -467,6 +468,7 @@ export function startRecordCapture( if (target instanceof Element) { const nearbyFillable = nearbyFillableFromSearchChrome(target); if (nearbyFillable) { + emitPendingHoverBeforeAction(nearbyFillable); ensureFillSession(nearbyFillable); return; } @@ -486,7 +488,7 @@ export function startRecordCapture( commitFillSession(); if (target instanceof Element && target.closest("select")) return; - emitPendingHoverBeforeClick(target); + emitPendingHoverBeforeAction(target); emitClick(event); }; @@ -528,6 +530,7 @@ export function startRecordCapture( commitFillSession(); const target = eventTarget(event); if (target instanceof HTMLSelectElement) { + emitPendingHoverBeforeAction(target); const values = Array.from(target.selectedOptions).map((opt) => opt.value); const labels = Array.from(target.selectedOptions).map((opt) => (opt.label || opt.textContent || opt.value).trim(), @@ -581,6 +584,7 @@ export function startRecordCapture( } const desc = describeEventTarget(target); if (!desc && !event.key) return; + emitPendingHoverBeforeAction(target); emitStep({ op: "press", key: event.key, diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index c4f7a86..36badac 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -456,6 +456,60 @@ describe("ToolDispatcher", () => { ); }); + it("limits default-target hover reassertion to the active tab", async () => { + vi.stubGlobal("chrome", { + tabs: { + query: vi.fn(async () => [{ id: 9, windowId: 4242, active: true }]), + get: vi.fn(async (tabId: number) => ({ id: tabId, windowId: 4242, active: tabId === 9 })), + sendMessage: vi.fn(async () => undefined), + }, + }); + const { transport } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 4242), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + await sessions.start("aa11"); + const cdp = { + send: vi.fn(async () => ({}) as T), + detachSession: vi.fn(async () => {}), + ensureNetworkCapture: vi.fn(async () => {}), + networkEntriesSince: vi.fn(() => ({ + tab_id: 9, + entries: [], + next_since: 0, + truncated: false, + })), + setDeviceMetricsOverride: vi.fn(async () => {}), + clearDeviceMetricsOverride: vi.fn(async () => {}), + setUserAgentOverride: vi.fn(async () => {}), + setTouchEmulationEnabled: vi.fn(async () => {}), + }; + const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); + const helpers = dispatcher as unknown as { + rememberHover: (sessionId: string, result: unknown) => unknown; + withHoverReassert: ( + params: { session_id: string; tab_id?: number }, + work: () => Promise, + ) => Promise; + }; + + helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 }); + helpers.rememberHover("aa11", { tab_id: 9, x: 30, y: 40 }); + + await helpers.withHoverReassert({ session_id: "aa11" }, async () => ({})); + + expect(cdp.send).toHaveBeenCalledTimes(1); + expect(cdp.send).toHaveBeenCalledWith(9, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: 30, + y: 40, + }); + }); + it("releases the current session hover latch before navigation-style work", async () => { vi.stubGlobal("chrome", { tabs: { diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 4ca8297..389c9d3 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -58,7 +58,7 @@ import { type SessionStartParams, type SessionStopParams, } from "./session"; -import { chromeTabsApi } from "./shared"; +import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared"; import { type BorrowConfirmationApprover, handleTabBorrow, @@ -90,6 +90,11 @@ interface HoverLatch { y: number; } +interface HoverLatchScope { + session_id: string; + tab_id?: number; +} + export interface DispatcherDeps { transport: Transport; sessions: SessionManager; @@ -341,23 +346,24 @@ export class ToolDispatcher { this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined, ), ); - case "tool.observe": - return this.withHoverReassert(req.params as ObserveParams, () => + case "tool.observe": { + const params = req.params as ObserveParams; + const hoverScope = await this.resolveHoverLatchScope(params); + return this.withHoverReassert(params, () => handleObserve( this.sessions, - req.params as ObserveParams, + params, this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi, - conditionalSurfaceProbe: !this.hasHoverLatchForRequest( - req.params as ObserveParams, - ), + conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope), hoverProbeBypassOverlay: bypassOverlay, } : undefined, ), ); + } case "tool.get_html": return handleGetHtml( this.sessions, @@ -551,8 +557,8 @@ export class ToolDispatcher { return result; } - private hasHoverLatchForRequest(params: { session_id: string; tab_id?: number }): boolean { - return this.hoverLatchesForRequest(params).length > 0; + private hasHoverLatchForScope(scope: HoverLatchScope): boolean { + return this.hoverLatchesForRequest(scope).length > 0; } private async withHoverReassert( @@ -560,12 +566,13 @@ export class ToolDispatcher { work: () => Promise, options: { releaseAfter?: boolean } = {}, ): Promise { - await this.reassertHover(params); + const scope = await this.resolveHoverLatchScope(params); + await this.reassertHover(scope); try { return await work(); } finally { if (options.releaseAfter) { - await this.releaseHoverLatch(params.session_id, params.tab_id); + await this.releaseHoverLatch(scope.session_id, scope.tab_id); } } } @@ -574,10 +581,23 @@ export class ToolDispatcher { params: { session_id: string; tab_id?: number }, work: () => Promise, ): Promise { - await this.releaseHoverLatch(params.session_id, params.tab_id); + const scope = await this.resolveHoverLatchScope(params); + await this.releaseHoverLatch(scope.session_id, scope.tab_id); return work(); } + private async resolveHoverLatchScope(params: { + session_id: string; + tab_id?: number; + }): Promise { + if (params.tab_id !== undefined) return params; + const ctx = lookupSession(this.sessions, params, "hover latch"); + if (isRpcError(ctx)) return params; + const target = await resolveTargetTab(this.sessions, ctx, undefined, chromeTabsApi); + if (isRpcError(target)) return params; + return { session_id: params.session_id, tab_id: target.tabId }; + } + private hoverLatchesForRequest(params: { session_id: string; tab_id?: number }): HoverLatch[] { return [...this.hoverLatches.values()].filter((latch) => { if (latch.sessionId !== params.session_id) return false; From 6c610faed95943d02d6926bc90242bcd233514b9 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Thu, 6 Aug 2026 18:46:06 +0800 Subject: [PATCH 09/12] fix(extension): bugfix --- apps/extension/src/content/record-capture.ts | 4 +++ .../src/tools/__tests__/dispatcher.test.ts | 35 +++++++++++++++++++ .../src/tools/__tests__/observation.test.ts | 1 + apps/extension/src/tools/dispatcher.ts | 7 ++-- apps/extension/src/tools/observation.ts | 1 + 5 files changed, 45 insertions(+), 3 deletions(-) diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index b21edcb..77123d6 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -422,6 +422,10 @@ export function startRecordCapture( const onMouseOver = (event: MouseEvent) => { const target = eventTarget(event); if (isOverlayTarget(target)) return; + if (pendingHover && target instanceof Element && pendingHover.element.contains(target)) { + if (Date.now() - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return; + pendingHover = null; + } const hover = hoverTargetFromEvent(target); if (!hover) return; const now = Date.now(); diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 36badac..cb4e5ea 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -456,6 +456,41 @@ describe("ToolDispatcher", () => { ); }); + it("transfers hover overlay bypass ownership between sessions", async () => { + vi.stubGlobal("chrome", { + tabs: { + sendMessage: vi.fn(async () => undefined), + }, + }); + const { transport } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 1), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + const dispatcher = new ToolDispatcher({ transport, sessions }); + const helpers = dispatcher as unknown as { + setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise; + }; + + await helpers.setHoverBypass("aa11", 7, true); + await helpers.setHoverBypass("bb22", 7, true); + await helpers.setHoverBypass("aa11", 7, false); + expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + + await helpers.setHoverBypass("bb22", 7, false); + expect(chrome.tabs.sendMessage).toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + expect(chrome.tabs.sendMessage).toHaveBeenCalledTimes(2); + }); + it("limits default-target hover reassertion to the active tab", async () => { vi.stubGlobal("chrome", { tabs: { diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 89a4ec3..a85fd64 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -1566,6 +1566,7 @@ describe("buildVomScene", () => { surfaceProbes: [ { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] }, { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] }, + { triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] }, ], nodes: [ { diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 389c9d3..26ec56d 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -534,12 +534,13 @@ export class ToolDispatcher { } private async setHoverBypass(sessionId: string, tabId: number, enabled: boolean): Promise { + const owner = this.hoverBypassTabs.get(tabId); if (enabled) { - if (this.hoverBypassTabs.has(tabId)) return; - await bypassOverlay(tabId, true); + if (owner === sessionId) return; + if (owner === undefined) await bypassOverlay(tabId, true); this.hoverBypassTabs.set(tabId, sessionId); } else { - if (!this.hoverBypassTabs.has(tabId)) return; + if (owner !== sessionId) return; await bypassOverlay(tabId, false); this.hoverBypassTabs.delete(tabId); } diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index a1da513..bde04f9 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -927,6 +927,7 @@ function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel) probe.triggerPoint, ); if (!match) continue; + if (used.has(match.id)) continue; used.add(probe.triggerBackendNodeId); used.add(match.id); surfaces.push({ From eeab97bcddd25c5b11090b2cc9940852f769e485 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 7 Aug 2026 12:02:28 +0800 Subject: [PATCH 10/12] feat(extension): hover candidate pool --- .../content/__tests__/record-capture.test.ts | 237 ++++++++++++++++++ apps/extension/src/content/record-capture.ts | 171 +++++++++---- .../src/content/record-hover-surface.ts | 6 +- .../__tests__/hover-trigger-policy.test.ts | 13 + .../extension/src/lib/hover-trigger-policy.ts | 20 +- 5 files changed, 391 insertions(+), 56 deletions(-) diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts index 26280f5..16059d0 100644 --- a/apps/extension/src/content/__tests__/record-capture.test.ts +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -8,6 +8,44 @@ vi.stubGlobal("chrome", { }, }); +function mockRect( + el: Element, + rect: { left: number; top: number; width: number; height: number }, +): void { + vi.spyOn(el, "getBoundingClientRect").mockReturnValue({ + x: rect.left, + y: rect.top, + top: rect.top, + left: rect.left, + right: rect.left + rect.width, + bottom: rect.top + rect.height, + width: rect.width, + height: rect.height, + toJSON: () => ({}), + }); +} + +function mockHoverStyle(pointerElements: Element[]): void { + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: pointerElements.includes(el) ? "pointer" : "", + display: "block", + pointerEvents: "auto", + position: "static", + visibility: "visible", + } as CSSStyleDeclaration; + return style; + }); +} + +function mouseOver(el: Element): void { + el.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); +} + +function click(el: Element): void { + el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); +} + describe("handleRecordContentMessage stop/cancel", () => { it("ignores STOP when no recording is active", () => { const dispose = vi.fn(); @@ -381,6 +419,205 @@ describe("record-capture semantic", () => { }); }); + it("keeps the menu opener hover when moving over an accepted menu item", () => { + document.body.innerHTML = ` + +
    +
  • 文档C+D
    • +
+ `; + const capture = startRecordCapture("rec-hover-menu-opener", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const item = document.querySelector("li")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([item]); + + mouseOver(trigger); + mouseOver(item); + click(item); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "li", name: "文档C+D" }, + }); + }); + + it("does not latch a plain div action item inside a hover surface", () => { + document.body.innerHTML = ` + +
+
文档C+D
+
+ `; + const capture = startRecordCapture("rec-hover-menu-div-action", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const item = document.querySelector(".tg-menu-item")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([item]); + + mouseOver(trigger); + mouseOver(item); + click(item); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "div", name: "文档C+D" }, + }); + }); + + it("keeps the opener hover when a surface action is clicked after the short latch window", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T03:43:36.000Z")); + try { + document.body.innerHTML = ` + +
+
文档C+D
+
+ `; + const capture = startRecordCapture("rec-hover-menu-div-action-slow", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const item = document.querySelector(".tg-menu-item")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([item]); + + mouseOver(trigger); + vi.setSystemTime(new Date("2026-08-07T03:43:47.000Z")); + mouseOver(item); + click(item); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "div", name: "文档C+D" }, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("infers a non-policy opener hover from the later surface action", () => { + document.body.innerHTML = ` + +
+
文档C+D
+
+ `; + const capture = startRecordCapture("rec-hover-infer-opener", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const item = document.querySelector(".tg-menu-item")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([trigger, item]); + + mouseOver(trigger); + mouseOver(item); + click(item); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "div", name: "文档C+D" }, + }); + }); + + it("does not record a strong-looking surface item when clicking that item itself", () => { + document.body.innerHTML = ` + +
+ +
+ `; + const capture = startRecordCapture("rec-hover-false-submenu", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const item = document.querySelector("li")!; + const itemInner = document.querySelector("li div")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([item, itemInner]); + + mouseOver(trigger); + mouseOver(item); + click(itemInner); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "div", name: "文档C+D" }, + }); + }); + + it("records cascaded hover triggers before clicking inside the final surface", () => { + document.body.innerHTML = ` + +
    +
  • 更多模板
    • +
+ + `; + const capture = startRecordCapture("rec-hover-cascade", (step) => steps.push(step)); + const trigger = document.querySelector("button[aria-label]")!; + const nestedTrigger = document.querySelector("li")!; + const finalAction = document.querySelector(".template-submenu button")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(nestedTrigger, { left: 820, top: 48, width: 160, height: 32 }); + mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 }); + mockHoverStyle([nestedTrigger, finalAction]); + + mouseOver(trigger); + mouseOver(nestedTrigger); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "更多模板" }, + }); + expect(steps[2]).toMatchObject({ + op: "click", + target: { role: "button", name: "Blank doc" }, + }); + }); + it("records avatar div hover with a semantic image target", () => { document.body.innerHTML = `
diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index 77123d6..530085b 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -9,6 +9,8 @@ import { evaluateHoverTrigger, type HoverTriggerDecision, type HoverTriggerRect, + hasDirectHoverInteractiveSignal, + hasStrongHoverExpansionSignal, } from "@/lib/hover-trigger-policy"; import { isRecordCancelMessage, @@ -26,7 +28,7 @@ import { type RecordStopMessage, } from "@/lib/record-bridge"; import { shouldRecordPress } from "@/lib/trace-reducer"; -import { decideHoverSurfaceRelation } from "./record-hover-surface"; +import { closestRecognizedHoverSurface, decideHoverSurfaceRelation } from "./record-hover-surface"; const pendingStepSends = new Map>>(); const failedStepDeliveries = new Set(); @@ -61,18 +63,21 @@ interface FillSession { lastValue: string; } -interface PendingHover { +interface HoverCandidate { element: Element; target: TargetDescriptor; recordedAt: number; score: number; - reasons: string[]; + hasExpansionSignal: boolean; + eligible: boolean; } type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement; const HOVER_BEFORE_CLICK_MAX_MS = 10_000; +const HOVER_SURFACE_CONTEXT_MAX_MS = 30_000; const HOVER_REPLACE_SCORE_MARGIN = 50; +const HOVER_CANDIDATE_LIMIT = 24; function eventTarget(event: Event): EventTarget | null { return event.composedPath()[0] ?? event.target; @@ -213,13 +218,10 @@ function normalizeHoverTarget( return desc; } -function evaluateHoverTriggerElement( - el: Element, - desc: TargetDescriptor, -): HoverTriggerDecision | null { +function hoverTriggerSignals(el: Element, desc: TargetDescriptor) { if (!(el instanceof HTMLElement)) return null; const style = hoverTriggerStyle(el); - return evaluateHoverTrigger({ + return { tag: el.tagName.toLowerCase(), role: desc.role, label: desc.name, @@ -228,29 +230,35 @@ function evaluateHoverTriggerElement( cursor: style.cursor, pointerEvents: style.pointerEvents, hasGraphicDescendant: hasHoverGraphicDescendant(el), - }); + }; } -function hoverTargetFromEvent(target: EventTarget | null): PendingHover | null { +function hoverCandidateFromEvent(target: EventTarget | null): HoverCandidate | null { if (!(target instanceof Element)) return null; const element = resolveHoverElement(target); if (!element) return null; const desc = describeTarget(element); - const decision = evaluateHoverTriggerElement(element, desc); - if (!decision?.eligible) return null; + const signals = hoverTriggerSignals(element, desc); + if (!signals) return null; + const decision = evaluateHoverTrigger(signals); + const hasExpansionSignal = hasStrongHoverExpansionSignal(signals); + const hasDirectSignal = hasDirectHoverInteractiveSignal(signals); + if (!decision.eligible && !hasExpansionSignal && !hasDirectSignal) return null; + if (!decision.eligible && !desc.name && !desc.role) return null; const normalizedTarget = normalizeHoverTarget(element, desc, decision); return { element, target: normalizedTarget, recordedAt: Date.now(), score: decision.score, - reasons: decision.reasons, + hasExpansionSignal, + eligible: decision.eligible, }; } -function shouldReplacePendingHover( - current: PendingHover, - next: PendingHover, +function shouldReplaceHoverCandidate( + current: HoverCandidate, + next: HoverCandidate, now: number, ): boolean { if (next.element === current.element) return false; @@ -310,7 +318,9 @@ export function startRecordCapture( let composing = false; let lastUrl = location.href; let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined; - let pendingHover: PendingHover | null = null; + let pendingHover: HoverCandidate | null = null; + const recentHoverCandidates: HoverCandidate[] = []; + const emittedHoverElements = new WeakSet(); let generatedControlClick: Element | null = null; let navigationActionPending = false; let navigationActionVersion = 0; @@ -376,6 +386,68 @@ export function startRecordCapture( }); }; + const rememberHoverCandidate = (hover: HoverCandidate) => { + const duplicate = recentHoverCandidates.findIndex( + (candidate) => candidate.element === hover.element, + ); + if (duplicate >= 0) recentHoverCandidates.splice(duplicate, 1); + recentHoverCandidates.push(hover); + if (recentHoverCandidates.length > HOVER_CANDIDATE_LIMIT) { + recentHoverCandidates.splice(0, recentHoverCandidates.length - HOVER_CANDIDATE_LIMIT); + } + }; + + const actionIsOnHoverTriggerItself = ( + hover: HoverCandidate, + actionElement: Element | null, + relationTarget: Element, + ): boolean => { + const target = actionElement ?? relationTarget; + if (target === hover.element) return true; + if (!hover.element.contains(target)) return false; + const surface = closestRecognizedHoverSurface(target); + return !surface || surface === hover.element || !hover.element.contains(surface); + }; + + const hoverRelatedToAction = ( + hover: HoverCandidate, + relationTarget: Element, + now: number, + ): boolean => { + if (now - hover.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return false; + return decideHoverSurfaceRelation({ triggerElement: hover.element }, relationTarget).related; + }; + + const emitHoverStep = (hover: HoverCandidate) => { + if (emittedHoverElements.has(hover.element)) return; + emitStep({ + op: "hover", + target: hover.target, + }); + emittedHoverElements.add(hover.element); + }; + + const findHoverForAction = ( + actionElement: Element | null, + relationTarget: Element, + now: number, + ): HoverCandidate | null => { + const candidates = pendingHover + ? [ + ...recentHoverCandidates.filter((hover) => hover.element !== pendingHover?.element), + pendingHover, + ] + : recentHoverCandidates; + for (let index = candidates.length - 1; index >= 0; index -= 1) { + const hover = candidates[index]; + if (!hover) continue; + if (emittedHoverElements.has(hover.element)) continue; + if (actionIsOnHoverTriggerItself(hover, actionElement, relationTarget)) continue; + if (hoverRelatedToAction(hover, relationTarget, now)) return hover; + } + return null; + }; + const emitClick = (event: MouseEvent) => { // Only record clicks an LLM can re-identify (named interactive controls). const target = describeEventTarget(eventTarget(event)); @@ -388,51 +460,52 @@ export function startRecordCapture( }); }; - const emitPendingHoverBeforeAction = (actionTarget: EventTarget | null) => { - if (!pendingHover || !(actionTarget instanceof Element)) return; - if (Date.now() - pendingHover.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) { - pendingHover = null; - return; - } + const emitHoverCandidateBeforeAction = (actionTarget: EventTarget | null) => { + if (!(actionTarget instanceof Element)) return; const actionElement = resolveClickableElement(actionTarget); - if (actionElement === pendingHover.element) { - return; - } const relationTarget = actionElement ?? actionTarget; - if (!actionElement && pendingHover.element.contains(relationTarget)) { - return; - } - const relation = decideHoverSurfaceRelation( - { - triggerElement: pendingHover.element, - }, - relationTarget, - ); - if (!relation.related) { + const hover = findHoverForAction(actionElement, relationTarget, Date.now()); + if (!hover) { pendingHover = null; return; } - emitStep({ - op: "hover", - target: pendingHover.target, - }); + emitHoverStep(hover); pendingHover = null; }; const onMouseOver = (event: MouseEvent) => { const target = eventTarget(event); if (isOverlayTarget(target)) return; + const now = Date.now(); if (pendingHover && target instanceof Element && pendingHover.element.contains(target)) { - if (Date.now() - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return; + if (now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return; pendingHover = null; } - const hover = hoverTargetFromEvent(target); + const candidate = hoverCandidateFromEvent(target); + if (candidate) rememberHoverCandidate(candidate); + const hover = candidate?.eligible ? candidate : null; + if (pendingHover && target instanceof Element) { + const relation = decideHoverSurfaceRelation({ triggerElement: pendingHover.element }, target); + if (relation.related && now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) { + if (!hover || !hover.hasExpansionSignal) return; + if ( + hover.element === pendingHover.element || + pendingHover.element.contains(hover.element) + ) { + return; + } + emitHoverStep(pendingHover); + pendingHover = hover; + rememberHoverCandidate(hover); + return; + } + } if (!hover) return; - const now = Date.now(); - if (pendingHover && !shouldReplacePendingHover(pendingHover, hover, now)) { + if (pendingHover && !shouldReplaceHoverCandidate(pendingHover, hover, now)) { return; } pendingHover = hover; + rememberHoverCandidate(hover); }; const onClick = (event: MouseEvent) => { @@ -464,7 +537,7 @@ export function startRecordCapture( const fillable = fillableFromTarget(target); if (fillable) { - emitPendingHoverBeforeAction(fillable); + emitHoverCandidateBeforeAction(fillable); ensureFillSession(fillable); return; } @@ -472,7 +545,7 @@ export function startRecordCapture( if (target instanceof Element) { const nearbyFillable = nearbyFillableFromSearchChrome(target); if (nearbyFillable) { - emitPendingHoverBeforeAction(nearbyFillable); + emitHoverCandidateBeforeAction(nearbyFillable); ensureFillSession(nearbyFillable); return; } @@ -492,7 +565,7 @@ export function startRecordCapture( commitFillSession(); if (target instanceof Element && target.closest("select")) return; - emitPendingHoverBeforeAction(target); + emitHoverCandidateBeforeAction(target); emitClick(event); }; @@ -534,7 +607,7 @@ export function startRecordCapture( commitFillSession(); const target = eventTarget(event); if (target instanceof HTMLSelectElement) { - emitPendingHoverBeforeAction(target); + emitHoverCandidateBeforeAction(target); const values = Array.from(target.selectedOptions).map((opt) => opt.value); const labels = Array.from(target.selectedOptions).map((opt) => (opt.label || opt.textContent || opt.value).trim(), @@ -588,7 +661,7 @@ export function startRecordCapture( } const desc = describeEventTarget(target); if (!desc && !event.key) return; - emitPendingHoverBeforeAction(target); + emitHoverCandidateBeforeAction(target); emitStep({ op: "press", key: event.key, diff --git a/apps/extension/src/content/record-hover-surface.ts b/apps/extension/src/content/record-hover-surface.ts index c00d6fc..c6fbd43 100644 --- a/apps/extension/src/content/record-hover-surface.ts +++ b/apps/extension/src/content/record-hover-surface.ts @@ -50,8 +50,8 @@ function isHoverSurfaceElement(el: Element): boolean { return false; } -function closestHoverSurface(clickElement: Element): Element | null { - let node: Element | null = clickElement; +export function closestRecognizedHoverSurface(element: Element): Element | null { + let node: Element | null = element; let depth = 0; while (node && node !== document.body && node !== document.documentElement && depth < 10) { if (isHoverSurfaceElement(node)) return node; @@ -115,7 +115,7 @@ export function decideHoverSurfaceRelation( context: HoverSurfaceContext, clickElement: Element, ): HoverSurfaceDecision { - const clickSurface = closestHoverSurface(clickElement); + const clickSurface = closestRecognizedHoverSurface(clickElement); if (clickSurface && isNearTrigger(clickSurface, context.triggerElement)) { return { related: true, reason: "click-inside-hover-surface", surface: clickSurface }; } diff --git a/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts b/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts index d3b3dce..dd73c24 100644 --- a/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts +++ b/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts @@ -77,6 +77,19 @@ describe("evaluateHoverTrigger", () => { expect(decision.eligible).toBe(false); }); + it("rejects labelled dropdown list items as hover triggers", () => { + const decision = evaluateHoverTrigger({ + tag: "li", + label: "文档C+D", + attrs: { class: "dropdown-item", tabindex: "0" }, + rect: { x: 820, y: 48, w: 160, h: 32 }, + cursor: "pointer", + pointerEvents: "auto", + }); + + expect(decision.eligible).toBe(false); + }); + it("rejects ordinary controls without hover popup signals", () => { const decision = evaluateHoverTrigger({ tag: "button", diff --git a/apps/extension/src/lib/hover-trigger-policy.ts b/apps/extension/src/lib/hover-trigger-policy.ts index 8701c89..23922b4 100644 --- a/apps/extension/src/lib/hover-trigger-policy.ts +++ b/apps/extension/src/lib/hover-trigger-policy.ts @@ -26,6 +26,7 @@ export interface HoverTriggerDecision { const HOVER_TRIGGER_MIN_SCORE = 45; const HOVER_POPUP_SIGNAL_RE = /(menu|dropdown|popover|popup|avatar|profile|account|user|more|ellipsis|caret)/; +const HOVER_TEXT_ITEM_RE = /(^|[\s_-])(dropdown-item|menu-item|context-menu-item|option)($|[\s_-])/; function attr(signals: HoverTriggerSignals, name: string): string | undefined { return signals.attrs?.[name.toLowerCase()]; @@ -59,6 +60,14 @@ export function hasHoverPopupSignal(signals: HoverTriggerSignals): boolean { return HOVER_POPUP_SIGNAL_RE.test(haystack); } +export function hasStrongHoverExpansionSignal(signals: HoverTriggerSignals): boolean { + return ( + signals.cssHoverMatch === true || + attr(signals, "aria-haspopup") !== undefined || + (attr(signals, "aria-expanded") ?? "").toLowerCase() === "false" + ); +} + export function hasDirectHoverInteractiveSignal(signals: HoverTriggerSignals): boolean { const tag = signals.tag.toLowerCase(); const role = (signals.role ?? "").toLowerCase(); @@ -104,7 +113,12 @@ export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTrigger const directInteractive = hasDirectHoverInteractiveSignal(signals); const popupSignal = hasHoverPopupSignal(signals); const isTextNavigationItem = - (tag === "a" || role === "link" || role.startsWith("menuitem")) && + (tag === "a" || + tag === "li" || + role === "link" || + role === "option" || + role.startsWith("menuitem") || + HOVER_TEXT_ITEM_RE.test(attr(signals, "class") ?? "")) && !graphic && !!label && label.toLowerCase() !== "image"; @@ -118,9 +132,7 @@ export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTrigger const surfaceTriggerEvidence = (popupSignal && !isTextNavigationItem) || compactTopbarIcon || - signals.cssHoverMatch === true || - attr(signals, "aria-haspopup") !== undefined || - (attr(signals, "aria-expanded") ?? "").toLowerCase() === "false"; + hasStrongHoverExpansionSignal(signals); if (!directInteractive || !surfaceTriggerEvidence) { return { eligible: false, score: 0, reasons: [] }; } From 2ccc0aafb932377d5721b1d98c07be8f1ed5273b Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 7 Aug 2026 15:45:38 +0800 Subject: [PATCH 11/12] feat(extension): hover chain node --- .../content/__tests__/record-capture.test.ts | 508 +++++++++++++++++- apps/extension/src/content/record-capture.ts | 277 ++++++++-- .../src/content/record-hover-surface.ts | 75 ++- 3 files changed, 784 insertions(+), 76 deletions(-) diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts index 16059d0..051b743 100644 --- a/apps/extension/src/content/__tests__/record-capture.test.ts +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -25,13 +25,13 @@ function mockRect( }); } -function mockHoverStyle(pointerElements: Element[]): void { +function mockHoverStyle(pointerElements: Element[], positionedElements: Element[] = []): void { vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { const style = { cursor: pointerElements.includes(el) ? "pointer" : "", display: "block", pointerEvents: "auto", - position: "static", + position: positionedElements.includes(el) ? "absolute" : "static", visibility: "visible", } as CSSStyleDeclaration; return style; @@ -204,6 +204,7 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-menu", (step) => steps.push(step)); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".user-menu")!; const item = document.querySelector("a")!; vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ x: 900, @@ -216,6 +217,7 @@ describe("record-capture semantic", () => { height: 32, toJSON: () => ({}), }); + mockRect(menu, { left: 820, top: 48, width: 160, height: 80 }); trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); @@ -244,6 +246,7 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-topbar", (step) => steps.push(step)); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".user-menu")!; const item = document.querySelector("a")!; vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ x: 900, @@ -256,6 +259,7 @@ describe("record-capture semantic", () => { height: 32, toJSON: () => ({}), }); + mockRect(menu, { left: 820, top: 48, width: 160, height: 80 }); trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); @@ -280,18 +284,10 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-contained-menu", (step) => steps.push(step)); const trigger = document.querySelector('[role="button"]')!; + const menu = document.querySelector("ul")!; const item = document.querySelector("a")!; - vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ - x: 900, - y: 8, - top: 8, - left: 900, - right: 932, - bottom: 40, - width: 32, - height: 32, - toJSON: () => ({}), - }); + mockRect(trigger, { left: 900, top: 8, width: 32, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 160, height: 80 }); trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); @@ -320,6 +316,7 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-topbar-link", (step) => steps.push(step)); const trigger = document.querySelector('a[aria-label="image"]')!; + const menu = document.querySelector(".user-menu")!; const item = document.querySelector("ul a")!; vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ x: 900, @@ -343,6 +340,7 @@ describe("record-capture semantic", () => { height: 32, toJSON: () => ({}), }); + mockRect(menu, { left: 820, top: 48, width: 160, height: 80 }); vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { const style = { cursor: el === item ? "pointer" : "", @@ -428,8 +426,10 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-menu-opener", (step) => steps.push(step)); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; const item = document.querySelector("li")!; mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); mockHoverStyle([item]); @@ -458,8 +458,10 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-menu-div-action", (step) => steps.push(step)); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; const item = document.querySelector(".tg-menu-item")!; mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); mockHoverStyle([item]); @@ -493,8 +495,10 @@ describe("record-capture semantic", () => { steps.push(step), ); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; const item = document.querySelector(".tg-menu-item")!; mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); mockHoverStyle([item]); @@ -527,8 +531,10 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-infer-opener", (step) => steps.push(step)); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; const item = document.querySelector(".tg-menu-item")!; mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); mockHoverStyle([trigger, item]); @@ -557,9 +563,11 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-false-submenu", (step) => steps.push(step)); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; const item = document.querySelector("li")!; const itemInner = document.querySelector("li div")!; mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); mockHoverStyle([item, itemInner]); @@ -591,14 +599,18 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-cascade", (step) => steps.push(step)); const trigger = document.querySelector("button[aria-label]")!; + const menu = document.querySelector(".create-menu")!; + const submenu = document.querySelector(".template-submenu")!; const nestedTrigger = document.querySelector("li")!; const finalAction = document.querySelector(".template-submenu button")!; mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); mockRect(nestedTrigger, { left: 820, top: 48, width: 160, height: 32 }); - mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 }); mockHoverStyle([nestedTrigger, finalAction]); mouseOver(trigger); + mockRect(submenu, { left: 984, top: 48, width: 160, height: 80 }); + mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 }); mouseOver(nestedTrigger); click(finalAction); capture.dispose(); @@ -618,6 +630,468 @@ describe("record-capture semantic", () => { }); }); + it("infers the full hover opener chain for nested menu actions", () => { + document.body.innerHTML = ` + +
+ +
+ + `; + const capture = startRecordCapture("rec-hover-nested-infer", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const submenu = document.querySelector(".format-submenu")!; + const nestedTrigger = document.querySelector("li")!; + const finalAction = document.querySelector("span")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); + mockRect(nestedTrigger, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([trigger, nestedTrigger, finalAction]); + + mouseOver(trigger); + mockRect(submenu, { left: 984, top: 48, width: 160, height: 80 }); + mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 }); + mouseOver(nestedTrigger); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "文档C+D" }, + }); + expect(steps[2]).toMatchObject({ + op: "click", + target: { tag: "span", name: "Markdown" }, + }); + }); + + it("uses the nested hover trigger label without descendant submenu text", () => { + document.body.innerHTML = ` + +
+
  • + 多维表格 +
    + 表格视图 +
    看板视图
    + 甘特视图 + 日历视图 + 相册视图 + 架构视图 + 神奇表单 +
    +
    • +
    + `; + const capture = startRecordCapture("rec-hover-nested-compact-label", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const nestedTrigger = document.querySelector("li")!; + const submenu = document.querySelector("li > div")!; + const finalAction = document.querySelector("li > div div")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); + mockRect(nestedTrigger, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([trigger, nestedTrigger, finalAction], [submenu]); + + mouseOver(trigger); + mockRect(submenu, { left: 984, top: 48, width: 180, height: 220 }); + mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 }); + mouseOver(nestedTrigger); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "多维表格" }, + }); + expect(JSON.stringify(steps[1]?.target)).not.toContain("看板视图"); + expect(steps[2]).toMatchObject({ + op: "click", + target: { tag: "div", name: "看板视图" }, + }); + }); + + it("does not let a stale pass-through shortcut hover claim a later submenu", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T06:39:40.000Z")); + try { + document.body.innerHTML = ` + +
    +
  • + 文档 + C+D +
    • +
  • +
    + 多维表格 +
    +
    + 看板视图 +
    +
    • +
    + `; + const capture = startRecordCapture("rec-hover-stale-shortcut", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const shortcut = document.querySelector(".text-font-tips")!; + const nestedTrigger = document.querySelector(".create-menu-item-vika")!; + const submenu = document.querySelector(".t-dropdown__submenu-wrapper")!; + const finalAction = document.querySelector(".t-dropdown__submenu-wrapper span")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 160 }); + mockRect(shortcut, { left: 950, top: 56, width: 29, height: 22 }); + mockRect(nestedTrigger, { left: 820, top: 92, width: 160, height: 32 }); + mockHoverStyle([trigger, shortcut, nestedTrigger, finalAction], [submenu]); + + mouseOver(trigger); + mouseOver(shortcut); + mockRect(submenu, { left: 984, top: 92, width: 160, height: 80 }); + mockRect(finalAction, { left: 984, top: 92, width: 136, height: 32 }); + mouseOver(nestedTrigger); + vi.runOnlyPendingTimers(); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "多维表格" }, + }); + expect(JSON.stringify(steps)).not.toContain("C+D"); + expect(steps[2]).toMatchObject({ + op: "click", + target: { tag: "span", name: "看板视图" }, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("does not let an item inside a newly opened menu claim the parent surface", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T07:12:08.000Z")); + try { + document.body.innerHTML = ` + +
    +
  • 文档C+D
    • +
  • + 多维表格 +
    • +
    +
    + 看板视图 +
    + `; + const capture = startRecordCapture("rec-hover-parent-surface-owner", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const passThrough = document.querySelector(".create-menu-item-doc")!; + const nestedTrigger = document.querySelector(".create-menu-item-vika")!; + const submenu = document.querySelector(".t-dropdown__submenu-wrapper")!; + const finalAction = document.querySelector(".t-dropdown__submenu-wrapper span")!; + mockRect(trigger, { left: 42, top: 72, width: 60, height: 32 }); + mockRect(passThrough, { left: 17, top: 113, width: 184, height: 34 }); + mockRect(nestedTrigger, { left: 17, top: 228, width: 184, height: 34 }); + mockHoverStyle([trigger, passThrough, nestedTrigger, finalAction]); + + mouseOver(trigger); + mockRect(menu, { left: 8, top: 104, width: 201, height: 439 }); + mouseOver(passThrough); + vi.runOnlyPendingTimers(); + mockRect(submenu, { left: 201, top: 213, width: 114, height: 267 }); + mockRect(finalAction, { left: 209, top: 257, width: 97, height: 34 }); + mouseOver(nestedTrigger); + vi.runOnlyPendingTimers(); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "多维表格" }, + }); + expect(JSON.stringify(steps)).not.toContain("文档C+D"); + } finally { + vi.useRealTimers(); + } + }); + + it("infers an unowned existing parent surface when a nested submenu is opened", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T07:28:10.000Z")); + try { + document.body.innerHTML = ` + +
    +
  • 文档C+D
    • +
  • +
    + 多维表格 +
    +
    • +
    +
    + 看板视图 +
    + `; + const capture = startRecordCapture("rec-hover-existing-parent-surface", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const passThrough = document.querySelector(".create-menu-item-doc")!; + const nestedTrigger = document.querySelector(".create-menu-item-vika")!; + const submenu = document.querySelector(".t-dropdown__submenu-wrapper")!; + const finalAction = document.querySelector(".t-dropdown__submenu-wrapper span")!; + mockRect(trigger, { left: 42, top: 72, width: 60, height: 32 }); + mockRect(menu, { left: 8, top: 104, width: 201, height: 439 }); + mockRect(passThrough, { left: 17, top: 113, width: 184, height: 34 }); + mockRect(nestedTrigger, { left: 17, top: 228, width: 184, height: 34 }); + mockHoverStyle([trigger, passThrough, nestedTrigger, finalAction]); + + mouseOver(passThrough); + mouseOver(trigger); + vi.runOnlyPendingTimers(); + mockRect(submenu, { left: 201, top: 213, width: 114, height: 267 }); + mockRect(finalAction, { left: 209, top: 257, width: 97, height: 34 }); + mouseOver(nestedTrigger); + vi.runOnlyPendingTimers(); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "多维表格" }, + }); + expect(JSON.stringify(steps)).not.toContain("文档C+D"); + } finally { + vi.useRealTimers(); + } + }); + + it("assigns a side submenu to the vertically aligned hover trigger", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T06:55:38.000Z")); + try { + document.body.innerHTML = ` + +
    +
  • + 文档 + C+D +
    • +
  • +
    + 多维表格 +
    +
    • +
    +
    +
  • + 看板视图 +
    • +
    + `; + const capture = startRecordCapture("rec-hover-side-submenu-alignment", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const passThrough = document.querySelector(".create-menu-item-doc")!; + const nestedTrigger = document.querySelector(".create-menu-item-vika")!; + const submenu = document.querySelector(".t-dropdown__submenu-wrapper")!; + const finalAction = document.querySelector(".t-dropdown__submenu-wrapper span")!; + mockRect(trigger, { left: 42, top: 72, width: 60, height: 32 }); + mockRect(menu, { left: 8, top: 104, width: 201, height: 439 }); + mockRect(passThrough, { left: 17, top: 113, width: 184, height: 34 }); + mockRect(nestedTrigger, { left: 17, top: 228, width: 184, height: 34 }); + mockHoverStyle([trigger, passThrough, nestedTrigger, finalAction]); + + mouseOver(trigger); + mouseOver(passThrough); + mockRect(submenu, { left: 201, top: 213, width: 114, height: 267 }); + mockRect(finalAction, { left: 209, top: 257, width: 97, height: 34 }); + mouseOver(nestedTrigger); + mouseOver(finalAction); + vi.runOnlyPendingTimers(); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "多维表格" }, + }); + expect(JSON.stringify(steps)).not.toContain("文档C+D"); + expect(steps[2]).toMatchObject({ + op: "click", + target: { tag: "span", name: "看板视图" }, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("outputs only one parent path for noisy nested menu hover movement", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T07:02:36.000Z")); + try { + document.body.innerHTML = ` + +
    +
  • + 文档 + C+D +
    • +
  • +
    + 多维表格 +
    +
    • +
    +
    +
    +
    + 表格视图 +
    +
    +
    + `; + const capture = startRecordCapture("rec-hover-noisy-nested-path", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const passThrough = document.querySelector(".create-menu-item-doc")!; + const nestedTrigger = document.querySelector(".create-menu-item-vika")!; + const nestedInner = document.querySelector(".t-dropdown__item-content")!; + const submenu = document.querySelector(".t-dropdown__submenu-wrapper")!; + const finalItem = document.querySelector(".create-submenu-item-vika-kanban")!; + const finalHoverItem = document.querySelector(".create-submenu-item-label")!; + const finalAction = document.querySelector(".create-submenu-item-vika-kanban")!; + mockRect(trigger, { left: 42, top: 72, width: 60, height: 32 }); + mockRect(menu, { left: 8, top: 104, width: 201, height: 439 }); + mockRect(passThrough, { left: 17, top: 113, width: 184, height: 34 }); + mockRect(nestedTrigger, { left: 17, top: 228, width: 184, height: 34 }); + mockRect(nestedInner, { left: 25, top: 234, width: 168, height: 22 }); + mockHoverStyle([ + trigger, + passThrough, + nestedTrigger, + nestedInner, + finalItem, + finalHoverItem, + finalAction, + ]); + + mouseOver(trigger); + mouseOver(nestedTrigger); + mouseOver(nestedInner); + mockRect(submenu, { left: 201, top: 213, width: 114, height: 267 }); + mockRect(finalItem, { left: 209, top: 257, width: 97, height: 34 }); + mockRect(finalHoverItem, { left: 209, top: 257, width: 97, height: 34 }); + mockRect(finalAction, { left: 209, top: 257, width: 97, height: 34 }); + mouseOver(passThrough); + mouseOver(nestedInner); + mouseOver(finalHoverItem); + vi.runOnlyPendingTimers(); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { name: "多维表格" }, + }); + expect(JSON.stringify(steps)).not.toContain("文档C+D"); + expect(JSON.stringify(steps)).not.toContain('"表格视图","tag":"div"'); + expect(steps.filter((step) => step.op === "hover")).toHaveLength(2); + } finally { + vi.useRealTimers(); + } + }); + + it("does not infer sibling items in the same surface as hover openers", () => { + document.body.innerHTML = ` + +
    + + Markdown +
    + `; + const capture = startRecordCapture("rec-hover-same-surface-action", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const category = document.querySelector("li")!; + const finalAction = document.querySelector("span")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 320, height: 80 }); + mockRect(category, { left: 820, top: 48, width: 160, height: 32 }); + mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 }); + mockHoverStyle([trigger, category, finalAction]); + + mouseOver(trigger); + mouseOver(category); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "span", name: "Markdown" }, + }); + }); + it("records avatar div hover with a semantic image target", () => { document.body.innerHTML = `
    @@ -630,6 +1104,7 @@ describe("record-capture semantic", () => { const capture = startRecordCapture("rec-hover-avatar-div", (step) => steps.push(step)); const trigger = document.querySelector(".tg-avatar")!; const image = document.querySelector("img")!; + const menu = document.querySelector(".user-menu")!; const item = document.querySelector("a")!; vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ x: 900, @@ -653,6 +1128,7 @@ describe("record-capture semantic", () => { height: 32, toJSON: () => ({}), }); + mockRect(menu, { left: 820, top: 48, width: 160, height: 80 }); vi.spyOn(window, "getComputedStyle").mockReturnValue({ cursor: "pointer", pointerEvents: "auto", @@ -717,6 +1193,7 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-fill-surface", (step) => steps.push(step)); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".user-menu")!; const input = document.querySelector("input")!; vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ x: 900, @@ -729,6 +1206,7 @@ describe("record-capture semantic", () => { height: 32, toJSON: () => ({}), }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 96 }); trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); input.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); @@ -762,6 +1240,7 @@ describe("record-capture semantic", () => { `; const capture = startRecordCapture("rec-hover-select-surface", (step) => steps.push(step)); const trigger = document.querySelector("button")!; + const menu = document.querySelector(".user-menu")!; const select = document.querySelector("select")!; vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ x: 900, @@ -774,6 +1253,7 @@ describe("record-capture semantic", () => { height: 32, toJSON: () => ({}), }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 96 }); trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); select.value = "away"; diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index 530085b..d693415 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -28,7 +28,13 @@ import { type RecordStopMessage, } from "@/lib/record-bridge"; import { shouldRecordPress } from "@/lib/trace-reducer"; -import { closestRecognizedHoverSurface, decideHoverSurfaceRelation } from "./record-hover-surface"; +import { + closestHoverSurfaceCandidate, + collectHoverSurfaceStates, + decideHoverSurfaceRelation, + type HoverSurfaceState, + isHoverSurfaceCandidateElement, +} from "./record-hover-surface"; const pendingStepSends = new Map>>(); const failedStepDeliveries = new Set(); @@ -68,16 +74,23 @@ interface HoverCandidate { target: TargetDescriptor; recordedAt: number; score: number; - hasExpansionSignal: boolean; eligible: boolean; } +interface HoverSurfaceNode { + element: Element; + signature: string; + owner: HoverCandidate; + parent?: HoverSurfaceNode; +} + type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement; const HOVER_BEFORE_CLICK_MAX_MS = 10_000; const HOVER_SURFACE_CONTEXT_MAX_MS = 30_000; const HOVER_REPLACE_SCORE_MARGIN = 50; const HOVER_CANDIDATE_LIMIT = 24; +const HOVER_TRIGGER_LABEL_MAX = 48; function eventTarget(event: Event): EventTarget | null { return event.composedPath()[0] ?? event.target; @@ -191,6 +204,51 @@ function hasHoverGraphicDescendant(el: Element): boolean { return el.querySelector("img,svg,use,path,i") !== null; } +function normalizeLabelText(value: string): string { + return value.replace(/\s+/g, " ").trim(); +} + +function truncateLabel(value: string, max = HOVER_TRIGGER_LABEL_MAX): string { + const normalized = normalizeLabelText(value); + if (normalized.length <= max) return normalized; + return `${normalized.slice(0, max - 1)}…`; +} + +function collectHoverTriggerLabelText(root: Element): string { + let text = ""; + const visit = (node: Node) => { + if (node.nodeType === Node.TEXT_NODE) { + text += ` ${node.textContent ?? ""}`; + return; + } + if (!(node instanceof Element)) return; + if (node !== root && isHoverSurfaceCandidateElement(node)) { + return; + } + for (const child of node.childNodes) visit(child); + }; + for (const child of root.childNodes) visit(child); + return normalizeLabelText(text); +} + +function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor { + if (!desc.name) return desc; + const fullText = normalizeLabelText(el.textContent ?? ""); + const compactName = collectHoverTriggerLabelText(el); + if (!compactName || compactName === desc.name) return desc; + const compactComparable = compactName.replace(/\s+/g, ""); + const descComparable = desc.name.replace(/…$/, "").replace(/\s+/g, ""); + const fullTextComparable = fullText.replace(/\s+/g, ""); + if ( + descComparable.startsWith(compactComparable) && + fullTextComparable.startsWith(descComparable) && + compactName.length < desc.name.length + ) { + return { ...desc, name: truncateLabel(compactName) }; + } + return desc; +} + function isWeakHoverTarget(target: TargetDescriptor): boolean { return !target.role && !target.name && target.tag === "div"; } @@ -245,13 +303,17 @@ function hoverCandidateFromEvent(target: EventTarget | null): HoverCandidate | n const hasDirectSignal = hasDirectHoverInteractiveSignal(signals); if (!decision.eligible && !hasExpansionSignal && !hasDirectSignal) return null; if (!decision.eligible && !desc.name && !desc.role) return null; - const normalizedTarget = normalizeHoverTarget(element, desc, decision); + const normalizedTarget = normalizeHoverTarget( + element, + compactHoverTargetName(element, desc), + decision, + ); + const now = Date.now(); return { element, target: normalizedTarget, - recordedAt: Date.now(), + recordedAt: now, score: decision.score, - hasExpansionSignal, eligible: decision.eligible, }; } @@ -321,6 +383,8 @@ export function startRecordCapture( let pendingHover: HoverCandidate | null = null; const recentHoverCandidates: HoverCandidate[] = []; const emittedHoverElements = new WeakSet(); + let hoverSurfaceStates = new Map(); + const hoverSurfaceNodes = new Map(); let generatedControlClick: Element | null = null; let navigationActionPending = false; let navigationActionVersion = 0; @@ -397,25 +461,126 @@ export function startRecordCapture( } }; - const actionIsOnHoverTriggerItself = ( - hover: HoverCandidate, - actionElement: Element | null, - relationTarget: Element, - ): boolean => { - const target = actionElement ?? relationTarget; - if (target === hover.element) return true; - if (!hover.element.contains(target)) return false; - const surface = closestRecognizedHoverSurface(target); - return !surface || surface === hover.element || !hover.element.contains(surface); + const surfaceArea = (el: Element): number => { + const rect = el.getBoundingClientRect(); + return rect.width * rect.height; }; - const hoverRelatedToAction = ( - hover: HoverCandidate, - relationTarget: Element, + const smallestContaining = ( + target: Element, + items: Iterable, + elementFor: (item: T) => Element, + ): T | undefined => { + let best: T | undefined; + for (const item of items) { + const element = elementFor(item); + if (!element.contains(target)) continue; + if (!best || surfaceArea(element) < surfaceArea(elementFor(best))) { + best = item; + } + } + return best; + }; + + const ownedSurfaceContaining = (target: Element): HoverSurfaceNode | undefined => + smallestContaining(target, hoverSurfaceNodes.values(), (node) => node.element); + + const surfaceStateContaining = ( + target: Element, + states: HoverSurfaceState[], + ): HoverSurfaceState | undefined => smallestContaining(target, states, (state) => state.element); + + const ownedSurfaceForAction = (target: Element): HoverSurfaceNode | undefined => { + const closestSurface = closestHoverSurfaceCandidate(target); + if (closestSurface) { + const owned = hoverSurfaceNodes.get(closestSurface); + if (owned) return owned; + } + return ownedSurfaceContaining(target); + }; + + const pruneGoneHoverSurfaces = (currentStates: HoverSurfaceState[]) => { + const current = new Set(currentStates.map((state) => state.element)); + for (const element of hoverSurfaceNodes.keys()) { + if (!current.has(element)) hoverSurfaceNodes.delete(element); + } + }; + + const inferOwnerForParentSurface = ( + surface: Element, + childOwner: HoverCandidate, now: number, - ): boolean => { - if (now - hover.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return false; - return decideHoverSurfaceRelation({ triggerElement: hover.element }, relationTarget).related; + ): HoverCandidate | undefined => { + for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) { + const candidate = recentHoverCandidates[index]; + if (!candidate || candidate.element === childOwner.element) continue; + if (surface.contains(candidate.element)) continue; + if (candidate.recordedAt > childOwner.recordedAt) continue; + if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; + const relation = decideHoverSurfaceRelation({ triggerElement: candidate.element }, surface); + if (relation.related) return candidate; + } + return undefined; + }; + + const parentSurfaceNodeForOwner = ( + owner: HoverCandidate, + currentStates: HoverSurfaceState[], + now: number, + ): HoverSurfaceNode | undefined => { + const ownedParent = ownedSurfaceContaining(owner.element); + if (ownedParent) return ownedParent; + const parentState = surfaceStateContaining(owner.element, currentStates); + if (!parentState) return undefined; + const parentOwner = inferOwnerForParentSurface(parentState.element, owner, now); + if (!parentOwner) return undefined; + const parentNode: HoverSurfaceNode = { + element: parentState.element, + signature: parentState.signature, + owner: parentOwner, + }; + hoverSurfaceNodes.set(parentState.element, parentNode); + return parentNode; + }; + + const bindOwnedHoverSurfaces = ( + owner: HoverCandidate, + previousStates: Map, + currentStates: HoverSurfaceState[], + now: number, + ) => { + const parent = parentSurfaceNodeForOwner(owner, currentStates, now); + for (const state of currentStates) { + if (previousStates.get(state.element) === state.signature) continue; + if (state.element.contains(owner.element)) continue; + if (parent?.element === state.element) continue; + const relation = decideHoverSurfaceRelation({ triggerElement: owner.element }, state.element); + if (!relation.related || now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; + hoverSurfaceNodes.set(state.element, { + element: state.element, + signature: state.signature, + owner, + ...(parent ? { parent } : {}), + }); + } + }; + + const refreshCausedHoverSurfaces = ( + owner: HoverCandidate, + previousStates = hoverSurfaceStates, + ) => { + const now = Date.now(); + const currentStates = collectHoverSurfaceStates(); + pruneGoneHoverSurfaces(currentStates); + bindOwnedHoverSurfaces(owner, previousStates, currentStates, now); + hoverSurfaceStates = new Map(currentStates.map((state) => [state.element, state.signature])); + }; + + const scheduleCausedHoverSurfaceRefresh = ( + hover: HoverCandidate, + previousStates: Map, + ) => { + setTimeout(() => refreshCausedHoverSurfaces(hover, previousStates), 0); }; const emitHoverStep = (hover: HoverCandidate) => { @@ -427,25 +592,31 @@ export function startRecordCapture( emittedHoverElements.add(hover.element); }; - const findHoverForAction = ( - actionElement: Element | null, - relationTarget: Element, + const surfaceOwnerChain = ( + surface: HoverSurfaceNode, + actionElement: Element, now: number, - ): HoverCandidate | null => { - const candidates = pendingHover - ? [ - ...recentHoverCandidates.filter((hover) => hover.element !== pendingHover?.element), - pendingHover, - ] - : recentHoverCandidates; - for (let index = candidates.length - 1; index >= 0; index -= 1) { - const hover = candidates[index]; - if (!hover) continue; - if (emittedHoverElements.has(hover.element)) continue; - if (actionIsOnHoverTriggerItself(hover, actionElement, relationTarget)) continue; - if (hoverRelatedToAction(hover, relationTarget, now)) return hover; - } - return null; + ): HoverCandidate[] => { + const chain: HoverCandidate[] = []; + const selected = new WeakSet(); + const surfaces: HoverSurfaceNode[] = []; + let node: HoverSurfaceNode | undefined = surface; + while (node && surfaces.length < 4) { + surfaces.unshift(node); + node = node.parent; + } + for (const owned of surfaces) { + const owner = owned.owner; + if (owner.element === actionElement || actionElement.contains(owner.element)) { + continue; + } + if (selected.has(owner.element)) continue; + if (emittedHoverElements.has(owner.element)) continue; + if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; + selected.add(owner.element); + chain.push(owner); + } + return chain; }; const emitClick = (event: MouseEvent) => { @@ -462,14 +633,14 @@ export function startRecordCapture( const emitHoverCandidateBeforeAction = (actionTarget: EventTarget | null) => { if (!(actionTarget instanceof Element)) return; - const actionElement = resolveClickableElement(actionTarget); - const relationTarget = actionElement ?? actionTarget; - const hover = findHoverForAction(actionElement, relationTarget, Date.now()); - if (!hover) { + const actionElement = resolveClickableElement(actionTarget) ?? actionTarget; + const surface = ownedSurfaceForAction(actionElement); + const hoverChain = surface ? surfaceOwnerChain(surface, actionElement, Date.now()) : []; + if (hoverChain.length === 0) { pendingHover = null; return; } - emitHoverStep(hover); + for (const hover of hoverChain) emitHoverStep(hover); pendingHover = null; }; @@ -482,21 +653,16 @@ export function startRecordCapture( pendingHover = null; } const candidate = hoverCandidateFromEvent(target); - if (candidate) rememberHoverCandidate(candidate); + if (candidate) { + const previousSurfaceStates = new Map(hoverSurfaceStates); + rememberHoverCandidate(candidate); + refreshCausedHoverSurfaces(candidate, previousSurfaceStates); + scheduleCausedHoverSurfaceRefresh(candidate, previousSurfaceStates); + } const hover = candidate?.eligible ? candidate : null; if (pendingHover && target instanceof Element) { const relation = decideHoverSurfaceRelation({ triggerElement: pendingHover.element }, target); if (relation.related && now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) { - if (!hover || !hover.hasExpansionSignal) return; - if ( - hover.element === pendingHover.element || - pendingHover.element.contains(hover.element) - ) { - return; - } - emitHoverStep(pendingHover); - pendingHover = hover; - rememberHoverCandidate(hover); return; } } @@ -505,7 +671,6 @@ export function startRecordCapture( return; } pendingHover = hover; - rememberHoverCandidate(hover); }; const onClick = (event: MouseEvent) => { diff --git a/apps/extension/src/content/record-hover-surface.ts b/apps/extension/src/content/record-hover-surface.ts index c6fbd43..e737cd3 100644 --- a/apps/extension/src/content/record-hover-surface.ts +++ b/apps/extension/src/content/record-hover-surface.ts @@ -8,6 +8,11 @@ export interface HoverSurfaceContext { triggerElement: Element; } +export interface HoverSurfaceState { + element: Element; + signature: string; +} + const HOVER_SURFACE_SELECTOR = [ '[role="menu"]', '[role="menubar"]', @@ -22,8 +27,11 @@ const HOVER_SURFACE_SELECTOR = [ ].join(","); const HOVER_SURFACE_WORD_RE = - /\b(menu|dropdown|drop-down|popover|popper|popup|tooltip|flyout|submenu|context-menu|account-menu|user-menu|avatar-menu)\b/i; + /(^|[^a-z0-9])(menu|dropdown|drop-down|popover|popper|popup|tooltip|flyout|submenu|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i; +const HOVER_SURFACE_ITEM_RE = + /(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)-?item|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i; const HOVER_SURFACE_MAX_AXIS_GAP = 96; +const HOVER_SURFACE_MAX_DIAGONAL_GAP = 16; const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5; function elementTokenText(el: Element): string { @@ -40,9 +48,12 @@ function elementTokenText(el: Element): string { .join(" "); } -function isHoverSurfaceElement(el: Element): boolean { +export function isRecognizedHoverSurfaceElement(el: Element): boolean { if (el.matches(HOVER_SURFACE_SELECTOR)) return true; - if (HOVER_SURFACE_WORD_RE.test(elementTokenText(el))) return true; + const tokenText = elementTokenText(el); + if (HOVER_SURFACE_WORD_RE.test(tokenText) && !HOVER_SURFACE_ITEM_RE.test(tokenText)) { + return true; + } const tag = el.tagName.toLowerCase(); if ((tag === "ul" || tag === "ol") && el.querySelector("a,button,[role='menuitem']")) { return HOVER_SURFACE_WORD_RE.test(elementTokenText(el.parentElement ?? el)); @@ -50,18 +61,27 @@ function isHoverSurfaceElement(el: Element): boolean { return false; } +function isVisibleSurfaceElement(el: Element): boolean { + if (!(el instanceof HTMLElement)) return false; + const style = getComputedStyle(el); + if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") { + return false; + } + return rectFor(el) !== null; +} + export function closestRecognizedHoverSurface(element: Element): Element | null { let node: Element | null = element; let depth = 0; while (node && node !== document.body && node !== document.documentElement && depth < 10) { - if (isHoverSurfaceElement(node)) return node; + if (isRecognizedHoverSurfaceElement(node)) return node; node = node.parentElement; depth += 1; } return null; } -function isPositionedFloatingElement(el: Element): boolean { +export function isPositionedFloatingElement(el: Element): boolean { if (!(el instanceof HTMLElement)) return false; const style = getComputedStyle(el); if (!["absolute", "fixed"].includes(style.position)) return false; @@ -77,6 +97,21 @@ function isPositionedFloatingElement(el: Element): boolean { ); } +export function isHoverSurfaceCandidateElement(el: Element): boolean { + return isRecognizedHoverSurfaceElement(el) || isPositionedFloatingElement(el); +} + +export function closestHoverSurfaceCandidate(element: Element): Element | null { + let node: Element | null = element; + let depth = 0; + while (node && node !== document.body && node !== document.documentElement && depth < 10) { + if (isHoverSurfaceCandidateElement(node)) return node; + node = node.parentElement; + depth += 1; + } + return null; +} + function closestFloatingSurface(clickElement: Element): Element | null { let node: Element | null = clickElement; let depth = 0; @@ -88,6 +123,26 @@ function closestFloatingSurface(clickElement: Element): Element | null { return null; } +function hoverSurfaceSignature(el: Element): string { + const rect = rectFor(el); + return [ + rect + ? `${Math.round(rect.left)},${Math.round(rect.top)},${Math.round(rect.width)},${Math.round(rect.height)}` + : "", + (el.textContent ?? "").trim().slice(0, 512), + ].join("|"); +} + +export function collectHoverSurfaceStates(): HoverSurfaceState[] { + const states: HoverSurfaceState[] = []; + for (const el of document.querySelectorAll("*")) { + if (isHoverSurfaceCandidateElement(el) && isVisibleSurfaceElement(el)) { + states.push({ element: el, signature: hoverSurfaceSignature(el) }); + } + } + return states; +} + function rectFor(el: Element): DOMRect | null { const rect = el.getBoundingClientRect(); if (rect.width <= 0 || rect.height <= 0) return null; @@ -98,6 +153,10 @@ function isNearTrigger(surface: Element, trigger: Element): boolean { const surfaceRect = rectFor(surface); const triggerRect = rectFor(trigger); if (!surfaceRect || !triggerRect) return true; + const horizontalOverlap = + Math.min(triggerRect.right, surfaceRect.right) - Math.max(triggerRect.left, surfaceRect.left); + const verticalOverlap = + Math.min(triggerRect.bottom, surfaceRect.bottom) - Math.max(triggerRect.top, surfaceRect.top); const horizontalGap = Math.max( 0, triggerRect.left - surfaceRect.right, @@ -108,7 +167,11 @@ function isNearTrigger(surface: Element, trigger: Element): boolean { triggerRect.top - surfaceRect.bottom, surfaceRect.top - triggerRect.bottom, ); - return horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP; + if (horizontalOverlap > 0 && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true; + if (verticalOverlap > 0 && horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true; + return ( + horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP && verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP + ); } export function decideHoverSurfaceRelation( From 566b6c17fd8bac24d56d757063a5fee09a8bea87 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 7 Aug 2026 17:37:28 +0800 Subject: [PATCH 12/12] feat(extension): hover chain node fix --- .../content/__tests__/record-capture.test.ts | 101 ++++++++++++ .../__tests__/record-hover-surface.test.ts | 109 +++++++++++++ apps/extension/src/content/record-capture.ts | 149 ++++++++++++------ .../src/content/record-hover-surface.ts | 55 +++++-- 4 files changed, 359 insertions(+), 55 deletions(-) create mode 100644 apps/extension/src/content/__tests__/record-hover-surface.test.ts diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts index 051b743..90536f7 100644 --- a/apps/extension/src/content/__tests__/record-capture.test.ts +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -1150,6 +1150,107 @@ describe("record-capture semantic", () => { }); }); + it("does not let unrelated topbar hovers or menu pass-through items own an avatar menu", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T08:08:45.000Z")); + try { + document.body.innerHTML = ` + +
    + +
    + + `; + const star = document.querySelector("button")!; + const avatar = document.querySelector(".tg-avatar")!; + const image = document.querySelector("img")!; + const menu = document.querySelector(".user-menu")!; + const profile = document.querySelector('a[href="/u/me"]')!; + const groups = document.querySelector('a[href="/dashboard/groups"]')!; + let menuRect = { left: 0, top: 0, width: 0, height: 0 }; + let profileRect = { left: 0, top: 0, width: 0, height: 0 }; + let groupsRect = { left: 0, top: 0, width: 0, height: 0 }; + mockRect(star, { left: 760, top: 8, width: 60, height: 32 }); + mockRect(avatar, { left: 900, top: 8, width: 32, height: 32 }); + mockRect(image, { left: 900, top: 8, width: 32, height: 32 }); + vi.spyOn(menu, "getBoundingClientRect").mockImplementation( + () => + ({ + x: menuRect.left, + y: menuRect.top, + top: menuRect.top, + left: menuRect.left, + right: menuRect.left + menuRect.width, + bottom: menuRect.top + menuRect.height, + width: menuRect.width, + height: menuRect.height, + toJSON: () => ({}), + }) as DOMRect, + ); + vi.spyOn(profile, "getBoundingClientRect").mockImplementation( + () => + ({ + x: profileRect.left, + y: profileRect.top, + top: profileRect.top, + left: profileRect.left, + right: profileRect.left + profileRect.width, + bottom: profileRect.top + profileRect.height, + width: profileRect.width, + height: profileRect.height, + toJSON: () => ({}), + }) as DOMRect, + ); + vi.spyOn(groups, "getBoundingClientRect").mockImplementation( + () => + ({ + x: groupsRect.left, + y: groupsRect.top, + top: groupsRect.top, + left: groupsRect.left, + right: groupsRect.left + groupsRect.width, + bottom: groupsRect.top + groupsRect.height, + width: groupsRect.width, + height: groupsRect.height, + toJSON: () => ({}), + }) as DOMRect, + ); + mockHoverStyle([star, avatar, image, profile, groups]); + const capture = startRecordCapture("rec-hover-avatar-menu-pass-through", (step) => + steps.push(step), + ); + + mouseOver(star); + mouseOver(avatar); + mouseOver(image); + menuRect = { left: 820, top: 48, width: 160, height: 96 }; + profileRect = { left: 830, top: 56, width: 120, height: 28 }; + groupsRect = { left: 830, top: 88, width: 120, height: 28 }; + vi.runOnlyPendingTimers(); + mouseOver(profile); + mouseOver(groups); + click(groups); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { tag: "img", role: "img", name: "image" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { role: "link", name: "My groups" }, + }); + expect(JSON.stringify(steps)).not.toContain("Star"); + expect(JSON.stringify(steps)).not.toContain("My profile"); + } finally { + vi.useRealTimers(); + } + }); + it("does not record hover before an unrelated page click", () => { document.body.innerHTML = ` diff --git a/apps/extension/src/content/__tests__/record-hover-surface.test.ts b/apps/extension/src/content/__tests__/record-hover-surface.test.ts new file mode 100644 index 0000000..08b8c96 --- /dev/null +++ b/apps/extension/src/content/__tests__/record-hover-surface.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it, vi } from "vitest"; +import { isHoverSurfaceCandidateElement, isLikelyHoverSurfaceOwner } from "../record-hover-surface"; + +function mockRect( + el: Element, + rect: { left: number; top: number; width: number; height: number }, +): void { + vi.spyOn(el, "getBoundingClientRect").mockReturnValue({ + x: rect.left, + y: rect.top, + top: rect.top, + left: rect.left, + right: rect.left + rect.width, + bottom: rect.top + rect.height, + width: rect.width, + height: rect.height, + toJSON: () => ({}), + }); +} + +describe("record-hover-surface", () => { + it("recognizes popover containers without treating topbar wrappers or menu items as surfaces", () => { + document.body.innerHTML = ` +
      +
    • +
      +
      • +
    +
    +
    +
    +
  • + My profile +
    • +
  • + My groups +
    • +
    +
    +
    + `; + + expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu")!)).toBe(false); + expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover")!)).toBe(false); + expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu__item")!)).toBe( + false, + ); + expect( + isHoverSurfaceCandidateElement(document.querySelector(".header-dropdown-menu__link")!), + ).toBe(false); + expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__popper")!)).toBe( + true, + ); + expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content")!)).toBe( + true, + ); + expect( + isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content-body")!), + ).toBe(true); + }); + + it("does not treat elements covered by an open surface as the surface owner", () => { + document.body.innerHTML = ` + + Fork + +
    + `; + const avatar = document.querySelector(".avatar")!; + const fork = document.querySelector(".fork")!; + const clone = document.querySelector(".clone")!; + const surface = document.querySelector(".tg-popover__content-body")!; + mockRect(avatar, { left: 1340, top: 11, width: 26, height: 26 }); + mockRect(fork, { left: 1275, top: 56, width: 69, height: 28 }); + mockRect(clone, { left: 1159, top: 111, width: 180, height: 30 }); + mockRect(surface, { left: 1111, top: 44, width: 240, height: 496 }); + + expect(isLikelyHoverSurfaceOwner(avatar, surface)).toBe(true); + expect(isLikelyHoverSurfaceOwner(fork, surface)).toBe(false); + expect(isLikelyHoverSurfaceOwner(clone, surface)).toBe(false); + }); + + it("does not treat fixed navigation containers as plain floating surfaces", () => { + document.body.innerHTML = ` +
    + Fork + +
    +
    + `; + const navbar = document.querySelector(".navbar-app")!; + const floating = document.querySelector(".plain-floating")!; + mockRect(navbar, { left: 0, top: 0, width: 1386, height: 51 }); + mockRect(floating, { left: 1111, top: 44, width: 240, height: 496 }); + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: "", + display: "block", + pointerEvents: "auto", + position: el === navbar ? "fixed" : "absolute", + visibility: "visible", + } as CSSStyleDeclaration; + return style; + }); + + expect(isHoverSurfaceCandidateElement(navbar)).toBe(false); + expect(isHoverSurfaceCandidateElement(floating)).toBe(true); + }); +}); diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index d693415..53488d7 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -34,6 +34,7 @@ import { decideHoverSurfaceRelation, type HoverSurfaceState, isHoverSurfaceCandidateElement, + isLikelyHoverSurfaceOwner, } from "./record-hover-surface"; const pendingStepSends = new Map>>(); @@ -376,6 +377,8 @@ export function startRecordCapture( const emitStep = (step: RecordStepPayload) => { sendStep({ page_url: location.href, ...step }); }; + const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map => + new Map(states.map((state) => [state.element, state.signature])); let fillSession: FillSession | null = null; let composing = false; let lastUrl = location.href; @@ -383,7 +386,7 @@ export function startRecordCapture( let pendingHover: HoverCandidate | null = null; const recentHoverCandidates: HoverCandidate[] = []; const emittedHoverElements = new WeakSet(); - let hoverSurfaceStates = new Map(); + let hoverSurfaceStates = hoverSurfaceStateMap(collectHoverSurfaceStates()); const hoverSurfaceNodes = new Map(); let generatedControlClick: Element | null = null; let navigationActionPending = false; @@ -490,13 +493,58 @@ export function startRecordCapture( states: HoverSurfaceState[], ): HoverSurfaceState | undefined => smallestContaining(target, states, (state) => state.element); + const inferOwnerForSurface = ( + surface: Element, + now: number, + before?: HoverCandidate, + ): HoverCandidate | undefined => { + for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) { + const candidate = recentHoverCandidates[index]; + if (!candidate) continue; + if (before && candidate.element === before.element) continue; + if (before && candidate.recordedAt > before.recordedAt) continue; + if (surface.contains(candidate.element)) continue; + if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; + if (isLikelyHoverSurfaceOwner(candidate.element, surface)) return candidate; + } + return undefined; + }; + + const nodeForUnownedSurface = ( + surfaceState: HoverSurfaceState, + currentStates: HoverSurfaceState[], + now: number, + ): HoverSurfaceNode | undefined => { + const owner = inferOwnerForSurface(surfaceState.element, now); + if (!owner) return undefined; + const parent = parentSurfaceNodeForOwner(owner, currentStates, now); + const node: HoverSurfaceNode = { + element: surfaceState.element, + signature: surfaceState.signature, + owner, + ...(parent ? { parent } : {}), + }; + hoverSurfaceNodes.set(surfaceState.element, node); + return node; + }; + const ownedSurfaceForAction = (target: Element): HoverSurfaceNode | undefined => { const closestSurface = closestHoverSurfaceCandidate(target); if (closestSurface) { const owned = hoverSurfaceNodes.get(closestSurface); if (owned) return owned; } - return ownedSurfaceContaining(target); + const ownedContaining = ownedSurfaceContaining(target); + if (ownedContaining) return ownedContaining; + const now = Date.now(); + const currentStates = collectHoverSurfaceStates(); + pruneGoneHoverSurfaces(currentStates); + const surfaceState = closestSurface + ? currentStates.find((state) => state.element === closestSurface) + : surfaceStateContaining(target, currentStates); + hoverSurfaceStates = hoverSurfaceStateMap(currentStates); + if (!surfaceState) return undefined; + return nodeForUnownedSurface(surfaceState, currentStates, now); }; const pruneGoneHoverSurfaces = (currentStates: HoverSurfaceState[]) => { @@ -506,23 +554,6 @@ export function startRecordCapture( } }; - const inferOwnerForParentSurface = ( - surface: Element, - childOwner: HoverCandidate, - now: number, - ): HoverCandidate | undefined => { - for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) { - const candidate = recentHoverCandidates[index]; - if (!candidate || candidate.element === childOwner.element) continue; - if (surface.contains(candidate.element)) continue; - if (candidate.recordedAt > childOwner.recordedAt) continue; - if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; - const relation = decideHoverSurfaceRelation({ triggerElement: candidate.element }, surface); - if (relation.related) return candidate; - } - return undefined; - }; - const parentSurfaceNodeForOwner = ( owner: HoverCandidate, currentStates: HoverSurfaceState[], @@ -532,7 +563,7 @@ export function startRecordCapture( if (ownedParent) return ownedParent; const parentState = surfaceStateContaining(owner.element, currentStates); if (!parentState) return undefined; - const parentOwner = inferOwnerForParentSurface(parentState.element, owner, now); + const parentOwner = inferOwnerForSurface(parentState.element, now, owner); if (!parentOwner) return undefined; const parentNode: HoverSurfaceNode = { element: parentState.element, @@ -543,44 +574,49 @@ export function startRecordCapture( return parentNode; }; - const bindOwnedHoverSurfaces = ( + const createSurfaceNode = ( + state: HoverSurfaceState, owner: HoverCandidate, + currentStates: HoverSurfaceState[], + now: number, + ): HoverSurfaceNode | undefined => { + const parent = parentSurfaceNodeForOwner(owner, currentStates, now); + if (parent?.element === state.element) return undefined; + if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return undefined; + if (!isLikelyHoverSurfaceOwner(owner.element, state.element)) return undefined; + const node: HoverSurfaceNode = { + element: state.element, + signature: state.signature, + owner, + ...(parent ? { parent } : {}), + }; + hoverSurfaceNodes.set(state.element, node); + return node; + }; + + const bindChangedHoverSurfaces = ( previousStates: Map, currentStates: HoverSurfaceState[], now: number, ) => { - const parent = parentSurfaceNodeForOwner(owner, currentStates, now); for (const state of currentStates) { if (previousStates.get(state.element) === state.signature) continue; - if (state.element.contains(owner.element)) continue; - if (parent?.element === state.element) continue; - const relation = decideHoverSurfaceRelation({ triggerElement: owner.element }, state.element); - if (!relation.related || now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; - hoverSurfaceNodes.set(state.element, { - element: state.element, - signature: state.signature, - owner, - ...(parent ? { parent } : {}), - }); + const owner = inferOwnerForSurface(state.element, now); + if (!owner) continue; + createSurfaceNode(state, owner, currentStates, now); } }; - const refreshCausedHoverSurfaces = ( - owner: HoverCandidate, - previousStates = hoverSurfaceStates, - ) => { + const refreshHoverSurfaces = (previousStates = hoverSurfaceStates) => { const now = Date.now(); const currentStates = collectHoverSurfaceStates(); pruneGoneHoverSurfaces(currentStates); - bindOwnedHoverSurfaces(owner, previousStates, currentStates, now); - hoverSurfaceStates = new Map(currentStates.map((state) => [state.element, state.signature])); + bindChangedHoverSurfaces(previousStates, currentStates, now); + hoverSurfaceStates = hoverSurfaceStateMap(currentStates); }; - const scheduleCausedHoverSurfaceRefresh = ( - hover: HoverCandidate, - previousStates: Map, - ) => { - setTimeout(() => refreshCausedHoverSurfaces(hover, previousStates), 0); + const scheduleHoverSurfaceRefresh = (previousStates: Map) => { + setTimeout(() => refreshHoverSurfaces(previousStates), 0); }; const emitHoverStep = (hover: HoverCandidate) => { @@ -619,6 +655,21 @@ export function startRecordCapture( return chain; }; + const containedHoverOwnerForAction = ( + actionElement: Element, + now: number, + ): HoverCandidate | undefined => { + for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) { + const candidate = recentHoverCandidates[index]; + if (!candidate) continue; + if (candidate.element === actionElement) continue; + if (!candidate.element.contains(actionElement)) continue; + if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; + return candidate; + } + return undefined; + }; + const emitClick = (event: MouseEvent) => { // Only record clicks an LLM can re-identify (named interactive controls). const target = describeEventTarget(eventTarget(event)); @@ -635,7 +686,13 @@ export function startRecordCapture( if (!(actionTarget instanceof Element)) return; const actionElement = resolveClickableElement(actionTarget) ?? actionTarget; const surface = ownedSurfaceForAction(actionElement); - const hoverChain = surface ? surfaceOwnerChain(surface, actionElement, Date.now()) : []; + const now = Date.now(); + const containedOwner = surface ? undefined : containedHoverOwnerForAction(actionElement, now); + const hoverChain = surface + ? surfaceOwnerChain(surface, actionElement, now) + : containedOwner + ? [containedOwner] + : []; if (hoverChain.length === 0) { pendingHover = null; return; @@ -656,8 +713,8 @@ export function startRecordCapture( if (candidate) { const previousSurfaceStates = new Map(hoverSurfaceStates); rememberHoverCandidate(candidate); - refreshCausedHoverSurfaces(candidate, previousSurfaceStates); - scheduleCausedHoverSurfaceRefresh(candidate, previousSurfaceStates); + refreshHoverSurfaces(previousSurfaceStates); + scheduleHoverSurfaceRefresh(previousSurfaceStates); } const hover = candidate?.eligible ? candidate : null; if (pendingHover && target instanceof Element) { diff --git a/apps/extension/src/content/record-hover-surface.ts b/apps/extension/src/content/record-hover-surface.ts index e737cd3..a47b426 100644 --- a/apps/extension/src/content/record-hover-surface.ts +++ b/apps/extension/src/content/record-hover-surface.ts @@ -27,9 +27,9 @@ const HOVER_SURFACE_SELECTOR = [ ].join(","); const HOVER_SURFACE_WORD_RE = - /(^|[^a-z0-9])(menu|dropdown|drop-down|popover|popper|popup|tooltip|flyout|submenu|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i; + /(^|[^a-z0-9])(dropdown-menu|drop-down-menu|menu-list|popover__(popper|content|content-body)|popover-content|popper|popup|flyout|submenu|submenu-wrapper|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i; const HOVER_SURFACE_ITEM_RE = - /(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)-?item|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i; + /(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)(-|__?)?(item|link)|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i; const HOVER_SURFACE_MAX_AXIS_GAP = 96; const HOVER_SURFACE_MAX_DIAGONAL_GAP = 16; const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5; @@ -54,10 +54,6 @@ export function isRecognizedHoverSurfaceElement(el: Element): boolean { if (HOVER_SURFACE_WORD_RE.test(tokenText) && !HOVER_SURFACE_ITEM_RE.test(tokenText)) { return true; } - const tag = el.tagName.toLowerCase(); - if ((tag === "ul" || tag === "ol") && el.querySelector("a,button,[role='menuitem']")) { - return HOVER_SURFACE_WORD_RE.test(elementTokenText(el.parentElement ?? el)); - } return false; } @@ -84,7 +80,7 @@ export function closestRecognizedHoverSurface(element: Element): Element | null export function isPositionedFloatingElement(el: Element): boolean { if (!(el instanceof HTMLElement)) return false; const style = getComputedStyle(el); - if (!["absolute", "fixed"].includes(style.position)) return false; + if (style.position !== "absolute") return false; if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") { return false; } @@ -149,10 +145,10 @@ function rectFor(el: Element): DOMRect | null { return rect; } -function isNearTrigger(surface: Element, trigger: Element): boolean { +function surfaceRelationMetrics(surface: Element, trigger: Element) { const surfaceRect = rectFor(surface); const triggerRect = rectFor(trigger); - if (!surfaceRect || !triggerRect) return true; + if (!surfaceRect || !triggerRect) return null; const horizontalOverlap = Math.min(triggerRect.right, surfaceRect.right) - Math.max(triggerRect.left, surfaceRect.left); const verticalOverlap = @@ -167,6 +163,20 @@ function isNearTrigger(surface: Element, trigger: Element): boolean { triggerRect.top - surfaceRect.bottom, surfaceRect.top - triggerRect.bottom, ); + return { + surfaceRect, + triggerRect, + horizontalOverlap, + verticalOverlap, + horizontalGap, + verticalGap, + }; +} + +function isNearTrigger(surface: Element, trigger: Element): boolean { + const metrics = surfaceRelationMetrics(surface, trigger); + if (!metrics) return true; + const { horizontalOverlap, verticalOverlap, horizontalGap, verticalGap } = metrics; if (horizontalOverlap > 0 && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true; if (verticalOverlap > 0 && horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true; return ( @@ -174,6 +184,33 @@ function isNearTrigger(surface: Element, trigger: Element): boolean { ); } +export function isLikelyHoverSurfaceOwner(trigger: Element, surface: Element): boolean { + const metrics = surfaceRelationMetrics(surface, trigger); + if (!metrics) return true; + const { + surfaceRect, + triggerRect, + horizontalOverlap, + verticalOverlap, + horizontalGap, + verticalGap, + } = metrics; + const opensVertically = + horizontalOverlap > 0 && + verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP && + (triggerRect.bottom <= surfaceRect.top + 8 || surfaceRect.bottom <= triggerRect.top + 8); + const opensSideways = + verticalOverlap > 0 && + horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP && + (triggerRect.right <= surfaceRect.left + 8 || surfaceRect.right <= triggerRect.left + 8); + const opensFromCorner = + horizontalGap > 0 && + verticalGap > 0 && + horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP && + verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP; + return opensVertically || opensSideways || opensFromCorner; +} + export function decideHoverSurfaceRelation( context: HoverSurfaceContext, clickElement: Element,