diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts index 38ac5d7..90536f7 100644 --- a/apps/extension/src/content/__tests__/record-capture.test.ts +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -8,6 +8,44 @@ vi.stubGlobal("chrome", { }, }); +function mockRect( + el: Element, + rect: { left: number; top: number; width: number; height: number }, +): void { + vi.spyOn(el, "getBoundingClientRect").mockReturnValue({ + x: rect.left, + y: rect.top, + top: rect.top, + left: rect.left, + right: rect.left + rect.width, + bottom: rect.top + rect.height, + width: rect.width, + height: rect.height, + toJSON: () => ({}), + }); +} + +function mockHoverStyle(pointerElements: Element[], positionedElements: Element[] = []): void { + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: pointerElements.includes(el) ? "pointer" : "", + display: "block", + pointerEvents: "auto", + position: positionedElements.includes(el) ? "absolute" : "static", + visibility: "visible", + } as CSSStyleDeclaration; + return style; + }); +} + +function mouseOver(el: Element): void { + el.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); +} + +function click(el: Element): void { + el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); +} + describe("handleRecordContentMessage stop/cancel", () => { it("ignores STOP when no recording is active", () => { const dispose = vi.fn(); @@ -156,4 +194,1363 @@ describe("record-capture semantic", () => { target: { name: "下一步", role: "button" }, }); }); + + it("records a hover trigger before clicking its revealed menu item", () => { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-menu", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".user-menu")!; + const item = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + mockRect(menu, { left: 820, top: 48, width: 160, height: 80 }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { role: "link", name: "My profile" }, + }); + }); + + it("records hover for compact topbar image buttons before menu item clicks", () => { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-topbar", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".user-menu")!; + const item = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + mockRect(menu, { left: 820, top: 48, width: 160, height: 80 }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "image" }, + }); + }); + + it("records hover when the revealed menu item is inside the hover root", () => { + document.body.innerHTML = ` +
+ image + +
+ `; + const capture = startRecordCapture("rec-hover-contained-menu", (step) => steps.push(step)); + const trigger = document.querySelector('[role="button"]')!; + const menu = document.querySelector("ul")!; + const item = document.querySelector("a")!; + mockRect(trigger, { left: 900, top: 8, width: 32, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 160, height: 80 }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "image" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { role: "link", name: "My profile" }, + }); + }); + + it("does not let a revealed topbar menu link replace the pending hover trigger", () => { + document.body.innerHTML = ` + + image + + + `; + const capture = startRecordCapture("rec-hover-topbar-link", (step) => steps.push(step)); + const trigger = document.querySelector('a[aria-label="image"]')!; + const menu = document.querySelector(".user-menu")!; + const item = document.querySelector("ul a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(item, "getBoundingClientRect").mockReturnValue({ + x: 820, + y: 72, + top: 72, + left: 820, + right: 916, + bottom: 104, + width: 96, + height: 32, + toJSON: () => ({}), + }); + mockRect(menu, { left: 820, top: 48, width: 160, height: 80 }); + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: el === item ? "pointer" : "", + pointerEvents: "auto", + } as CSSStyleDeclaration; + return style; + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "link", name: "image" }, + }); + }); + + it("keeps the first hover trigger over a later lower-score accepted hover", () => { + document.body.innerHTML = ` + + My profile + `; + const capture = startRecordCapture("rec-hover-latch", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const laterHover = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(laterHover, "getBoundingClientRect").mockReturnValue({ + x: 820, + y: 72, + top: 72, + left: 820, + right: 916, + bottom: 104, + width: 96, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: el === laterHover ? "pointer" : "", + pointerEvents: "auto", + } as CSSStyleDeclaration; + return style; + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + laterHover.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + laterHover.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { role: "button", name: "My profile" }, + }); + }); + + it("keeps the menu opener hover when moving over an accepted menu item", () => { + document.body.innerHTML = ` + + + `; + const capture = startRecordCapture("rec-hover-menu-opener", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const item = document.querySelector("li")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); + mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([item]); + + mouseOver(trigger); + mouseOver(item); + click(item); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "li", name: "文档C+D" }, + }); + }); + + it("does not latch a plain div action item inside a hover surface", () => { + document.body.innerHTML = ` + +
+
文档C+D
+
+ `; + const capture = startRecordCapture("rec-hover-menu-div-action", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const item = document.querySelector(".tg-menu-item")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); + mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([item]); + + mouseOver(trigger); + mouseOver(item); + click(item); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "div", name: "文档C+D" }, + }); + }); + + it("keeps the opener hover when a surface action is clicked after the short latch window", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T03:43:36.000Z")); + try { + document.body.innerHTML = ` + +
+
文档C+D
+
+ `; + const capture = startRecordCapture("rec-hover-menu-div-action-slow", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const item = document.querySelector(".tg-menu-item")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); + mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([item]); + + mouseOver(trigger); + vi.setSystemTime(new Date("2026-08-07T03:43:47.000Z")); + mouseOver(item); + click(item); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "div", name: "文档C+D" }, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("infers a non-policy opener hover from the later surface action", () => { + document.body.innerHTML = ` + +
+
文档C+D
+
+ `; + const capture = startRecordCapture("rec-hover-infer-opener", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const item = document.querySelector(".tg-menu-item")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); + mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([trigger, item]); + + mouseOver(trigger); + mouseOver(item); + click(item); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "div", name: "文档C+D" }, + }); + }); + + it("does not record a strong-looking surface item when clicking that item itself", () => { + document.body.innerHTML = ` + +
+ +
+ `; + const capture = startRecordCapture("rec-hover-false-submenu", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const item = document.querySelector("li")!; + const itemInner = document.querySelector("li div")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); + mockRect(item, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([item, itemInner]); + + mouseOver(trigger); + mouseOver(item); + click(itemInner); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "div", name: "文档C+D" }, + }); + }); + + it("records cascaded hover triggers before clicking inside the final surface", () => { + document.body.innerHTML = ` + + + + `; + const capture = startRecordCapture("rec-hover-cascade", (step) => steps.push(step)); + const trigger = document.querySelector("button[aria-label]")!; + const menu = document.querySelector(".create-menu")!; + const submenu = document.querySelector(".template-submenu")!; + const nestedTrigger = document.querySelector("li")!; + const finalAction = document.querySelector(".template-submenu button")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); + mockRect(nestedTrigger, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([nestedTrigger, finalAction]); + + mouseOver(trigger); + mockRect(submenu, { left: 984, top: 48, width: 160, height: 80 }); + mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 }); + mouseOver(nestedTrigger); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "更多模板" }, + }); + expect(steps[2]).toMatchObject({ + op: "click", + target: { role: "button", name: "Blank doc" }, + }); + }); + + it("infers the full hover opener chain for nested menu actions", () => { + document.body.innerHTML = ` + +
+ +
+ + `; + const capture = startRecordCapture("rec-hover-nested-infer", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const submenu = document.querySelector(".format-submenu")!; + const nestedTrigger = document.querySelector("li")!; + const finalAction = document.querySelector("span")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); + mockRect(nestedTrigger, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([trigger, nestedTrigger, finalAction]); + + mouseOver(trigger); + mockRect(submenu, { left: 984, top: 48, width: 160, height: 80 }); + mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 }); + mouseOver(nestedTrigger); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "文档C+D" }, + }); + expect(steps[2]).toMatchObject({ + op: "click", + target: { tag: "span", name: "Markdown" }, + }); + }); + + it("uses the nested hover trigger label without descendant submenu text", () => { + document.body.innerHTML = ` + +
+
  • + 多维表格 +
    + 表格视图 +
    看板视图
    + 甘特视图 + 日历视图 + 相册视图 + 架构视图 + 神奇表单 +
    +
    • +
    + `; + const capture = startRecordCapture("rec-hover-nested-compact-label", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const nestedTrigger = document.querySelector("li")!; + const submenu = document.querySelector("li > div")!; + const finalAction = document.querySelector("li > div div")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 80 }); + mockRect(nestedTrigger, { left: 820, top: 48, width: 160, height: 32 }); + mockHoverStyle([trigger, nestedTrigger, finalAction], [submenu]); + + mouseOver(trigger); + mockRect(submenu, { left: 984, top: 48, width: 180, height: 220 }); + mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 }); + mouseOver(nestedTrigger); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "多维表格" }, + }); + expect(JSON.stringify(steps[1]?.target)).not.toContain("看板视图"); + expect(steps[2]).toMatchObject({ + op: "click", + target: { tag: "div", name: "看板视图" }, + }); + }); + + it("does not let a stale pass-through shortcut hover claim a later submenu", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T06:39:40.000Z")); + try { + document.body.innerHTML = ` + +
    +
  • + 文档 + C+D +
    • +
  • +
    + 多维表格 +
    +
    + 看板视图 +
    +
    • +
    + `; + const capture = startRecordCapture("rec-hover-stale-shortcut", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const shortcut = document.querySelector(".text-font-tips")!; + const nestedTrigger = document.querySelector(".create-menu-item-vika")!; + const submenu = document.querySelector(".t-dropdown__submenu-wrapper")!; + const finalAction = document.querySelector(".t-dropdown__submenu-wrapper span")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 160 }); + mockRect(shortcut, { left: 950, top: 56, width: 29, height: 22 }); + mockRect(nestedTrigger, { left: 820, top: 92, width: 160, height: 32 }); + mockHoverStyle([trigger, shortcut, nestedTrigger, finalAction], [submenu]); + + mouseOver(trigger); + mouseOver(shortcut); + mockRect(submenu, { left: 984, top: 92, width: 160, height: 80 }); + mockRect(finalAction, { left: 984, top: 92, width: 136, height: 32 }); + mouseOver(nestedTrigger); + vi.runOnlyPendingTimers(); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "多维表格" }, + }); + expect(JSON.stringify(steps)).not.toContain("C+D"); + expect(steps[2]).toMatchObject({ + op: "click", + target: { tag: "span", name: "看板视图" }, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("does not let an item inside a newly opened menu claim the parent surface", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T07:12:08.000Z")); + try { + document.body.innerHTML = ` + +
    +
  • 文档C+D
    • +
  • + 多维表格 +
    • +
    +
    + 看板视图 +
    + `; + const capture = startRecordCapture("rec-hover-parent-surface-owner", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const passThrough = document.querySelector(".create-menu-item-doc")!; + const nestedTrigger = document.querySelector(".create-menu-item-vika")!; + const submenu = document.querySelector(".t-dropdown__submenu-wrapper")!; + const finalAction = document.querySelector(".t-dropdown__submenu-wrapper span")!; + mockRect(trigger, { left: 42, top: 72, width: 60, height: 32 }); + mockRect(passThrough, { left: 17, top: 113, width: 184, height: 34 }); + mockRect(nestedTrigger, { left: 17, top: 228, width: 184, height: 34 }); + mockHoverStyle([trigger, passThrough, nestedTrigger, finalAction]); + + mouseOver(trigger); + mockRect(menu, { left: 8, top: 104, width: 201, height: 439 }); + mouseOver(passThrough); + vi.runOnlyPendingTimers(); + mockRect(submenu, { left: 201, top: 213, width: 114, height: 267 }); + mockRect(finalAction, { left: 209, top: 257, width: 97, height: 34 }); + mouseOver(nestedTrigger); + vi.runOnlyPendingTimers(); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "多维表格" }, + }); + expect(JSON.stringify(steps)).not.toContain("文档C+D"); + } finally { + vi.useRealTimers(); + } + }); + + it("infers an unowned existing parent surface when a nested submenu is opened", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T07:28:10.000Z")); + try { + document.body.innerHTML = ` + +
    +
  • 文档C+D
    • +
  • +
    + 多维表格 +
    +
    • +
    +
    + 看板视图 +
    + `; + const capture = startRecordCapture("rec-hover-existing-parent-surface", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const passThrough = document.querySelector(".create-menu-item-doc")!; + const nestedTrigger = document.querySelector(".create-menu-item-vika")!; + const submenu = document.querySelector(".t-dropdown__submenu-wrapper")!; + const finalAction = document.querySelector(".t-dropdown__submenu-wrapper span")!; + mockRect(trigger, { left: 42, top: 72, width: 60, height: 32 }); + mockRect(menu, { left: 8, top: 104, width: 201, height: 439 }); + mockRect(passThrough, { left: 17, top: 113, width: 184, height: 34 }); + mockRect(nestedTrigger, { left: 17, top: 228, width: 184, height: 34 }); + mockHoverStyle([trigger, passThrough, nestedTrigger, finalAction]); + + mouseOver(passThrough); + mouseOver(trigger); + vi.runOnlyPendingTimers(); + mockRect(submenu, { left: 201, top: 213, width: 114, height: 267 }); + mockRect(finalAction, { left: 209, top: 257, width: 97, height: 34 }); + mouseOver(nestedTrigger); + vi.runOnlyPendingTimers(); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "多维表格" }, + }); + expect(JSON.stringify(steps)).not.toContain("文档C+D"); + } finally { + vi.useRealTimers(); + } + }); + + it("assigns a side submenu to the vertically aligned hover trigger", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T06:55:38.000Z")); + try { + document.body.innerHTML = ` + +
    +
  • + 文档 + C+D +
    • +
  • +
    + 多维表格 +
    +
    • +
    +
    +
  • + 看板视图 +
    • +
    + `; + const capture = startRecordCapture("rec-hover-side-submenu-alignment", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const passThrough = document.querySelector(".create-menu-item-doc")!; + const nestedTrigger = document.querySelector(".create-menu-item-vika")!; + const submenu = document.querySelector(".t-dropdown__submenu-wrapper")!; + const finalAction = document.querySelector(".t-dropdown__submenu-wrapper span")!; + mockRect(trigger, { left: 42, top: 72, width: 60, height: 32 }); + mockRect(menu, { left: 8, top: 104, width: 201, height: 439 }); + mockRect(passThrough, { left: 17, top: 113, width: 184, height: 34 }); + mockRect(nestedTrigger, { left: 17, top: 228, width: 184, height: 34 }); + mockHoverStyle([trigger, passThrough, nestedTrigger, finalAction]); + + mouseOver(trigger); + mouseOver(passThrough); + mockRect(submenu, { left: 201, top: 213, width: 114, height: 267 }); + mockRect(finalAction, { left: 209, top: 257, width: 97, height: 34 }); + mouseOver(nestedTrigger); + mouseOver(finalAction); + vi.runOnlyPendingTimers(); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { tag: "li", name: "多维表格" }, + }); + expect(JSON.stringify(steps)).not.toContain("文档C+D"); + expect(steps[2]).toMatchObject({ + op: "click", + target: { tag: "span", name: "看板视图" }, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("outputs only one parent path for noisy nested menu hover movement", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T07:02:36.000Z")); + try { + document.body.innerHTML = ` + +
    +
  • + 文档 + C+D +
    • +
  • +
    + 多维表格 +
    +
    • +
    +
    +
    +
    + 表格视图 +
    +
    +
    + `; + const capture = startRecordCapture("rec-hover-noisy-nested-path", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const passThrough = document.querySelector(".create-menu-item-doc")!; + const nestedTrigger = document.querySelector(".create-menu-item-vika")!; + const nestedInner = document.querySelector(".t-dropdown__item-content")!; + const submenu = document.querySelector(".t-dropdown__submenu-wrapper")!; + const finalItem = document.querySelector(".create-submenu-item-vika-kanban")!; + const finalHoverItem = document.querySelector(".create-submenu-item-label")!; + const finalAction = document.querySelector(".create-submenu-item-vika-kanban")!; + mockRect(trigger, { left: 42, top: 72, width: 60, height: 32 }); + mockRect(menu, { left: 8, top: 104, width: 201, height: 439 }); + mockRect(passThrough, { left: 17, top: 113, width: 184, height: 34 }); + mockRect(nestedTrigger, { left: 17, top: 228, width: 184, height: 34 }); + mockRect(nestedInner, { left: 25, top: 234, width: 168, height: 22 }); + mockHoverStyle([ + trigger, + passThrough, + nestedTrigger, + nestedInner, + finalItem, + finalHoverItem, + finalAction, + ]); + + mouseOver(trigger); + mouseOver(nestedTrigger); + mouseOver(nestedInner); + mockRect(submenu, { left: 201, top: 213, width: 114, height: 267 }); + mockRect(finalItem, { left: 209, top: 257, width: 97, height: 34 }); + mockRect(finalHoverItem, { left: 209, top: 257, width: 97, height: 34 }); + mockRect(finalAction, { left: 209, top: 257, width: 97, height: 34 }); + mouseOver(passThrough); + mouseOver(nestedInner); + mouseOver(finalHoverItem); + vi.runOnlyPendingTimers(); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "hover", + target: { name: "多维表格" }, + }); + expect(JSON.stringify(steps)).not.toContain("文档C+D"); + expect(JSON.stringify(steps)).not.toContain('"表格视图","tag":"div"'); + expect(steps.filter((step) => step.op === "hover")).toHaveLength(2); + } finally { + vi.useRealTimers(); + } + }); + + it("does not infer sibling items in the same surface as hover openers", () => { + document.body.innerHTML = ` + +
    + + Markdown +
    + `; + const capture = startRecordCapture("rec-hover-same-surface-action", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".create-menu")!; + const category = document.querySelector("li")!; + const finalAction = document.querySelector("span")!; + mockRect(trigger, { left: 900, top: 8, width: 60, height: 32 }); + mockRect(menu, { left: 820, top: 48, width: 320, height: 80 }); + mockRect(category, { left: 820, top: 48, width: 160, height: 32 }); + mockRect(finalAction, { left: 984, top: 48, width: 136, height: 32 }); + mockHoverStyle([trigger, category, finalAction]); + + mouseOver(trigger); + mouseOver(category); + mouseOver(finalAction); + click(finalAction); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "新建" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { tag: "span", name: "Markdown" }, + }); + }); + + it("records avatar div hover with a semantic image target", () => { + document.body.innerHTML = ` +
    + +
    + + `; + const capture = startRecordCapture("rec-hover-avatar-div", (step) => steps.push(step)); + const trigger = document.querySelector(".tg-avatar")!; + const image = document.querySelector("img")!; + const menu = document.querySelector(".user-menu")!; + const item = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(image, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + mockRect(menu, { left: 820, top: 48, width: 160, height: 80 }); + vi.spyOn(window, "getComputedStyle").mockReturnValue({ + cursor: "pointer", + pointerEvents: "auto", + } as CSSStyleDeclaration); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + image.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + item.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { tag: "img", role: "img", name: "image" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { role: "link", name: "My profile" }, + }); + }); + + it("does not let unrelated topbar hovers or menu pass-through items own an avatar menu", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-08-07T08:08:45.000Z")); + try { + document.body.innerHTML = ` + +
    + +
    + + `; + const star = document.querySelector("button")!; + const avatar = document.querySelector(".tg-avatar")!; + const image = document.querySelector("img")!; + const menu = document.querySelector(".user-menu")!; + const profile = document.querySelector('a[href="/u/me"]')!; + const groups = document.querySelector('a[href="/dashboard/groups"]')!; + let menuRect = { left: 0, top: 0, width: 0, height: 0 }; + let profileRect = { left: 0, top: 0, width: 0, height: 0 }; + let groupsRect = { left: 0, top: 0, width: 0, height: 0 }; + mockRect(star, { left: 760, top: 8, width: 60, height: 32 }); + mockRect(avatar, { left: 900, top: 8, width: 32, height: 32 }); + mockRect(image, { left: 900, top: 8, width: 32, height: 32 }); + vi.spyOn(menu, "getBoundingClientRect").mockImplementation( + () => + ({ + x: menuRect.left, + y: menuRect.top, + top: menuRect.top, + left: menuRect.left, + right: menuRect.left + menuRect.width, + bottom: menuRect.top + menuRect.height, + width: menuRect.width, + height: menuRect.height, + toJSON: () => ({}), + }) as DOMRect, + ); + vi.spyOn(profile, "getBoundingClientRect").mockImplementation( + () => + ({ + x: profileRect.left, + y: profileRect.top, + top: profileRect.top, + left: profileRect.left, + right: profileRect.left + profileRect.width, + bottom: profileRect.top + profileRect.height, + width: profileRect.width, + height: profileRect.height, + toJSON: () => ({}), + }) as DOMRect, + ); + vi.spyOn(groups, "getBoundingClientRect").mockImplementation( + () => + ({ + x: groupsRect.left, + y: groupsRect.top, + top: groupsRect.top, + left: groupsRect.left, + right: groupsRect.left + groupsRect.width, + bottom: groupsRect.top + groupsRect.height, + width: groupsRect.width, + height: groupsRect.height, + toJSON: () => ({}), + }) as DOMRect, + ); + mockHoverStyle([star, avatar, image, profile, groups]); + const capture = startRecordCapture("rec-hover-avatar-menu-pass-through", (step) => + steps.push(step), + ); + + mouseOver(star); + mouseOver(avatar); + mouseOver(image); + menuRect = { left: 820, top: 48, width: 160, height: 96 }; + profileRect = { left: 830, top: 56, width: 120, height: 28 }; + groupsRect = { left: 830, top: 88, width: 120, height: 28 }; + vi.runOnlyPendingTimers(); + mouseOver(profile); + mouseOver(groups); + click(groups); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { tag: "img", role: "img", name: "image" }, + }); + expect(steps[1]).toMatchObject({ + op: "click", + target: { role: "link", name: "My groups" }, + }); + expect(JSON.stringify(steps)).not.toContain("Star"); + expect(JSON.stringify(steps)).not.toContain("My profile"); + } finally { + vi.useRealTimers(); + } + }); + + it("does not record hover before an unrelated page click", () => { + document.body.innerHTML = ` + +
    + Projects +
    + `; + const capture = startRecordCapture("rec-hover-unrelated-click", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const link = document.querySelector("main a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + link.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["click"]); + expect(steps[0]).toMatchObject({ + op: "click", + target: { role: "link", name: "Projects" }, + }); + }); + + it("records hover before filling a field inside the hover surface", () => { + document.body.innerHTML = ` + +
    + + +
    + `; + const capture = startRecordCapture("rec-hover-fill-surface", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".user-menu")!; + const input = document.querySelector("input")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 96 }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + input.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + input.dispatchEvent(new FocusEvent("focusin", { bubbles: true })); + input.value = "Ada"; + input.dispatchEvent(new Event("input", { bubbles: true })); + input.dispatchEvent(new FocusEvent("focusout", { bubbles: true })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "fill"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + expect(steps[1]).toMatchObject({ + op: "fill", + target: { tag: "input" }, + value: "Ada", + }); + }); + + it("records hover before selecting inside the hover surface", () => { + document.body.innerHTML = ` + +
    + +
    + `; + const capture = startRecordCapture("rec-hover-select-surface", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const menu = document.querySelector(".user-menu")!; + const select = document.querySelector("select")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + mockRect(menu, { left: 820, top: 48, width: 180, height: 96 }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + select.value = "away"; + select.dispatchEvent(new Event("change", { bubbles: true })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "select"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + expect(steps[1]).toMatchObject({ + op: "select", + target: { role: "combobox", name: "Status" }, + values: ["away"], + labels: ["Away"], + }); + }); + + it("records hover before clicking an unlabelled positioned floating surface", () => { + document.body.innerHTML = ` + +
    + My profile +
    + `; + const capture = startRecordCapture("rec-hover-plain-floating-surface", (step) => + steps.push(step), + ); + const trigger = document.querySelector("button")!; + const surface = document.querySelector("div")!; + const link = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(surface, "getBoundingClientRect").mockReturnValue({ + x: 820, + y: 48, + top: 48, + left: 820, + right: 980, + bottom: 128, + width: 160, + height: 80, + toJSON: () => ({}), + }); + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: "", + display: "block", + pointerEvents: "auto", + position: el === surface ? "absolute" : "static", + visibility: "visible", + } as CSSStyleDeclaration; + return style; + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + link.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + target: { role: "button", name: "Open user navigation menu" }, + }); + }); + + it("does not treat ordinary document-flow divs as hover surfaces", () => { + document.body.innerHTML = ` + +
    + My profile +
    + `; + const capture = startRecordCapture("rec-hover-plain-flow-surface", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const link = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(window, "getComputedStyle").mockReturnValue({ + cursor: "", + display: "block", + pointerEvents: "auto", + position: "static", + visibility: "visible", + } as CSSStyleDeclaration); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + link.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["click"]); + }); + + it("does not treat unlabelled sticky containers as hover surfaces", () => { + document.body.innerHTML = ` + +
    + My profile +
    + `; + const capture = startRecordCapture("rec-hover-sticky-surface", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + const surface = document.querySelector("div")!; + const link = document.querySelector("a")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + vi.spyOn(surface, "getBoundingClientRect").mockReturnValue({ + x: 820, + y: 48, + top: 48, + left: 820, + right: 980, + bottom: 128, + width: 160, + height: 80, + toJSON: () => ({}), + }); + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: "", + display: "block", + pointerEvents: "auto", + position: el === surface ? "sticky" : "static", + visibility: "visible", + } as CSSStyleDeclaration; + return style; + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + link.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["click"]); + }); + + it("does not record hover before clicking the same trigger", () => { + document.body.innerHTML = ` + + `; + const capture = startRecordCapture("rec-hover-same-trigger", (step) => steps.push(step)); + const trigger = document.querySelector("button")!; + vi.spyOn(trigger, "getBoundingClientRect").mockReturnValue({ + x: 900, + y: 8, + top: 8, + left: 900, + right: 932, + bottom: 40, + width: 32, + height: 32, + toJSON: () => ({}), + }); + + trigger.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + trigger.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["click"]); + }); + + it("does not record hover for ordinary controls without popup signals", () => { + const capture = startRecordCapture("rec-hover-noise", (step) => steps.push(step)); + const button = document.querySelector("button")!; + + button.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + button.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual(["click"]); + }); }); diff --git a/apps/extension/src/content/__tests__/record-hover-surface.test.ts b/apps/extension/src/content/__tests__/record-hover-surface.test.ts new file mode 100644 index 0000000..08b8c96 --- /dev/null +++ b/apps/extension/src/content/__tests__/record-hover-surface.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it, vi } from "vitest"; +import { isHoverSurfaceCandidateElement, isLikelyHoverSurfaceOwner } from "../record-hover-surface"; + +function mockRect( + el: Element, + rect: { left: number; top: number; width: number; height: number }, +): void { + vi.spyOn(el, "getBoundingClientRect").mockReturnValue({ + x: rect.left, + y: rect.top, + top: rect.top, + left: rect.left, + right: rect.left + rect.width, + bottom: rect.top + rect.height, + width: rect.width, + height: rect.height, + toJSON: () => ({}), + }); +} + +describe("record-hover-surface", () => { + it("recognizes popover containers without treating topbar wrappers or menu items as surfaces", () => { + document.body.innerHTML = ` + +
    +
    +
    +
  • + My profile +
    • +
  • + My groups +
    • +
    +
    +
    + `; + + expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu")!)).toBe(false); + expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover")!)).toBe(false); + expect(isHoverSurfaceCandidateElement(document.querySelector(".header-menu__item")!)).toBe( + false, + ); + expect( + isHoverSurfaceCandidateElement(document.querySelector(".header-dropdown-menu__link")!), + ).toBe(false); + expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__popper")!)).toBe( + true, + ); + expect(isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content")!)).toBe( + true, + ); + expect( + isHoverSurfaceCandidateElement(document.querySelector(".tg-popover__content-body")!), + ).toBe(true); + }); + + it("does not treat elements covered by an open surface as the surface owner", () => { + document.body.innerHTML = ` + + Fork + +
    + `; + const avatar = document.querySelector(".avatar")!; + const fork = document.querySelector(".fork")!; + const clone = document.querySelector(".clone")!; + const surface = document.querySelector(".tg-popover__content-body")!; + mockRect(avatar, { left: 1340, top: 11, width: 26, height: 26 }); + mockRect(fork, { left: 1275, top: 56, width: 69, height: 28 }); + mockRect(clone, { left: 1159, top: 111, width: 180, height: 30 }); + mockRect(surface, { left: 1111, top: 44, width: 240, height: 496 }); + + expect(isLikelyHoverSurfaceOwner(avatar, surface)).toBe(true); + expect(isLikelyHoverSurfaceOwner(fork, surface)).toBe(false); + expect(isLikelyHoverSurfaceOwner(clone, surface)).toBe(false); + }); + + it("does not treat fixed navigation containers as plain floating surfaces", () => { + document.body.innerHTML = ` +
    + Fork + +
    +
    + `; + const navbar = document.querySelector(".navbar-app")!; + const floating = document.querySelector(".plain-floating")!; + mockRect(navbar, { left: 0, top: 0, width: 1386, height: 51 }); + mockRect(floating, { left: 1111, top: 44, width: 240, height: 496 }); + vi.spyOn(window, "getComputedStyle").mockImplementation((el) => { + const style = { + cursor: "", + display: "block", + pointerEvents: "auto", + position: el === navbar ? "fixed" : "absolute", + visibility: "visible", + } as CSSStyleDeclaration; + return style; + }); + + expect(isHoverSurfaceCandidateElement(navbar)).toBe(false); + expect(isHoverSurfaceCandidateElement(floating)).toBe(true); + }); +}); diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index 12bf6af..53488d7 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -1,4 +1,17 @@ -import { describeEventTarget, describeTarget, type TargetDescriptor } from "@/lib/describe-target"; +import { + describeEventTarget, + describeTarget, + resolveClickableElement, + resolveHoverElement, + type TargetDescriptor, +} from "@/lib/describe-target"; +import { + evaluateHoverTrigger, + type HoverTriggerDecision, + type HoverTriggerRect, + hasDirectHoverInteractiveSignal, + hasStrongHoverExpansionSignal, +} from "@/lib/hover-trigger-policy"; import { isRecordCancelMessage, isRecordStartMessage, @@ -15,6 +28,14 @@ import { type RecordStopMessage, } from "@/lib/record-bridge"; import { shouldRecordPress } from "@/lib/trace-reducer"; +import { + closestHoverSurfaceCandidate, + collectHoverSurfaceStates, + decideHoverSurfaceRelation, + type HoverSurfaceState, + isHoverSurfaceCandidateElement, + isLikelyHoverSurfaceOwner, +} from "./record-hover-surface"; const pendingStepSends = new Map>>(); const failedStepDeliveries = new Set(); @@ -49,8 +70,29 @@ interface FillSession { lastValue: string; } +interface HoverCandidate { + element: Element; + target: TargetDescriptor; + recordedAt: number; + score: number; + eligible: boolean; +} + +interface HoverSurfaceNode { + element: Element; + signature: string; + owner: HoverCandidate; + parent?: HoverSurfaceNode; +} + type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement; +const HOVER_BEFORE_CLICK_MAX_MS = 10_000; +const HOVER_SURFACE_CONTEXT_MAX_MS = 30_000; +const HOVER_REPLACE_SCORE_MARGIN = 50; +const HOVER_CANDIDATE_LIMIT = 24; +const HOVER_TRIGGER_LABEL_MAX = 48; + function eventTarget(event: Event): EventTarget | null { return event.composedPath()[0] ?? event.target; } @@ -122,6 +164,173 @@ function fillableValue(el: FillableElement): string { return el.textContent ?? ""; } +function hoverTriggerAttrs(el: Element): Record { + return { + id: el.id || undefined, + class: typeof el.className === "string" ? el.className || undefined : undefined, + "data-testid": el.getAttribute("data-testid") ?? undefined, + "data-test": el.getAttribute("data-test") ?? undefined, + "data-cy": el.getAttribute("data-cy") ?? undefined, + "aria-label": el.getAttribute("aria-label") ?? undefined, + "aria-haspopup": el.getAttribute("aria-haspopup") ?? undefined, + "aria-controls": el.getAttribute("aria-controls") ?? undefined, + "aria-expanded": el.getAttribute("aria-expanded") ?? undefined, + "aria-hidden": el.getAttribute("aria-hidden") ?? undefined, + "aria-disabled": el.getAttribute("aria-disabled") ?? undefined, + contenteditable: el.getAttribute("contenteditable") ?? undefined, + disabled: el.hasAttribute("disabled") ? "" : undefined, + hidden: el.hasAttribute("hidden") ? "" : undefined, + inert: el.hasAttribute("inert") ? "" : undefined, + onclick: el.getAttribute("onclick") ?? undefined, + onmouseenter: el.getAttribute("onmouseenter") ?? undefined, + onmouseover: el.getAttribute("onmouseover") ?? undefined, + role: el.getAttribute("role") ?? undefined, + tabindex: el.getAttribute("tabindex") ?? undefined, + title: el.getAttribute("title") ?? undefined, + }; +} + +function hoverTriggerRect(el: Element): HoverTriggerRect { + const rect = el.getBoundingClientRect(); + return { x: rect.left, y: rect.top, w: rect.width, h: rect.height }; +} + +function hoverTriggerStyle(el: Element): { cursor?: string; pointerEvents?: string } { + if (!(el instanceof HTMLElement)) return {}; + const style = getComputedStyle(el); + return { cursor: style.cursor, pointerEvents: style.pointerEvents }; +} + +function hasHoverGraphicDescendant(el: Element): boolean { + return el.querySelector("img,svg,use,path,i") !== null; +} + +function normalizeLabelText(value: string): string { + return value.replace(/\s+/g, " ").trim(); +} + +function truncateLabel(value: string, max = HOVER_TRIGGER_LABEL_MAX): string { + const normalized = normalizeLabelText(value); + if (normalized.length <= max) return normalized; + return `${normalized.slice(0, max - 1)}…`; +} + +function collectHoverTriggerLabelText(root: Element): string { + let text = ""; + const visit = (node: Node) => { + if (node.nodeType === Node.TEXT_NODE) { + text += ` ${node.textContent ?? ""}`; + return; + } + if (!(node instanceof Element)) return; + if (node !== root && isHoverSurfaceCandidateElement(node)) { + return; + } + for (const child of node.childNodes) visit(child); + }; + for (const child of root.childNodes) visit(child); + return normalizeLabelText(text); +} + +function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor { + if (!desc.name) return desc; + const fullText = normalizeLabelText(el.textContent ?? ""); + const compactName = collectHoverTriggerLabelText(el); + if (!compactName || compactName === desc.name) return desc; + const compactComparable = compactName.replace(/\s+/g, ""); + const descComparable = desc.name.replace(/…$/, "").replace(/\s+/g, ""); + const fullTextComparable = fullText.replace(/\s+/g, ""); + if ( + descComparable.startsWith(compactComparable) && + fullTextComparable.startsWith(descComparable) && + compactName.length < desc.name.length + ) { + return { ...desc, name: truncateLabel(compactName) }; + } + return desc; +} + +function isWeakHoverTarget(target: TargetDescriptor): boolean { + return !target.role && !target.name && target.tag === "div"; +} + +function looksLikeAvatarElement(el: Element): boolean { + const className = typeof el.className === "string" ? el.className : ""; + return /\b(avatar|user-avatar)\b/i.test(className); +} + +function normalizeHoverTarget( + el: Element, + desc: TargetDescriptor, + decision: HoverTriggerDecision, +): TargetDescriptor { + if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) { + return { ...desc, name: "image" }; + } + if ( + isWeakHoverTarget(desc) && + (looksLikeAvatarElement(el) || + (hasHoverGraphicDescendant(el) && decision.reasons.includes("icon-only"))) + ) { + return { tag: "img", role: "img", name: "image" }; + } + return desc; +} + +function hoverTriggerSignals(el: Element, desc: TargetDescriptor) { + if (!(el instanceof HTMLElement)) return null; + const style = hoverTriggerStyle(el); + return { + tag: el.tagName.toLowerCase(), + role: desc.role, + label: desc.name, + attrs: hoverTriggerAttrs(el), + rect: hoverTriggerRect(el), + cursor: style.cursor, + pointerEvents: style.pointerEvents, + hasGraphicDescendant: hasHoverGraphicDescendant(el), + }; +} + +function hoverCandidateFromEvent(target: EventTarget | null): HoverCandidate | null { + if (!(target instanceof Element)) return null; + const element = resolveHoverElement(target); + if (!element) return null; + const desc = describeTarget(element); + const signals = hoverTriggerSignals(element, desc); + if (!signals) return null; + const decision = evaluateHoverTrigger(signals); + const hasExpansionSignal = hasStrongHoverExpansionSignal(signals); + const hasDirectSignal = hasDirectHoverInteractiveSignal(signals); + if (!decision.eligible && !hasExpansionSignal && !hasDirectSignal) return null; + if (!decision.eligible && !desc.name && !desc.role) return null; + const normalizedTarget = normalizeHoverTarget( + element, + compactHoverTargetName(element, desc), + decision, + ); + const now = Date.now(); + return { + element, + target: normalizedTarget, + recordedAt: now, + score: decision.score, + eligible: decision.eligible, + }; +} + +function shouldReplaceHoverCandidate( + current: HoverCandidate, + next: HoverCandidate, + now: number, +): boolean { + if (next.element === current.element) return false; + if (now - current.recordedAt > HOVER_BEFORE_CLICK_MAX_MS) return true; + if (current.element.contains(next.element)) return false; + if (next.element.contains(current.element)) return true; + return next.score >= current.score + HOVER_REPLACE_SCORE_MARGIN; +} + /** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */ function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean { if (!session || !(target instanceof Element)) return false; @@ -168,10 +377,17 @@ export function startRecordCapture( const emitStep = (step: RecordStepPayload) => { sendStep({ page_url: location.href, ...step }); }; + const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map => + new Map(states.map((state) => [state.element, state.signature])); let fillSession: FillSession | null = null; let composing = false; let lastUrl = location.href; let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined; + let pendingHover: HoverCandidate | null = null; + const recentHoverCandidates: HoverCandidate[] = []; + const emittedHoverElements = new WeakSet(); + let hoverSurfaceStates = hoverSurfaceStateMap(collectHoverSurfaceStates()); + const hoverSurfaceNodes = new Map(); let generatedControlClick: Element | null = null; let navigationActionPending = false; let navigationActionVersion = 0; @@ -237,6 +453,223 @@ export function startRecordCapture( }); }; + const rememberHoverCandidate = (hover: HoverCandidate) => { + const duplicate = recentHoverCandidates.findIndex( + (candidate) => candidate.element === hover.element, + ); + if (duplicate >= 0) recentHoverCandidates.splice(duplicate, 1); + recentHoverCandidates.push(hover); + if (recentHoverCandidates.length > HOVER_CANDIDATE_LIMIT) { + recentHoverCandidates.splice(0, recentHoverCandidates.length - HOVER_CANDIDATE_LIMIT); + } + }; + + const surfaceArea = (el: Element): number => { + const rect = el.getBoundingClientRect(); + return rect.width * rect.height; + }; + + const smallestContaining = ( + target: Element, + items: Iterable, + elementFor: (item: T) => Element, + ): T | undefined => { + let best: T | undefined; + for (const item of items) { + const element = elementFor(item); + if (!element.contains(target)) continue; + if (!best || surfaceArea(element) < surfaceArea(elementFor(best))) { + best = item; + } + } + return best; + }; + + const ownedSurfaceContaining = (target: Element): HoverSurfaceNode | undefined => + smallestContaining(target, hoverSurfaceNodes.values(), (node) => node.element); + + const surfaceStateContaining = ( + target: Element, + states: HoverSurfaceState[], + ): HoverSurfaceState | undefined => smallestContaining(target, states, (state) => state.element); + + const inferOwnerForSurface = ( + surface: Element, + now: number, + before?: HoverCandidate, + ): HoverCandidate | undefined => { + for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) { + const candidate = recentHoverCandidates[index]; + if (!candidate) continue; + if (before && candidate.element === before.element) continue; + if (before && candidate.recordedAt > before.recordedAt) continue; + if (surface.contains(candidate.element)) continue; + if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; + if (isLikelyHoverSurfaceOwner(candidate.element, surface)) return candidate; + } + return undefined; + }; + + const nodeForUnownedSurface = ( + surfaceState: HoverSurfaceState, + currentStates: HoverSurfaceState[], + now: number, + ): HoverSurfaceNode | undefined => { + const owner = inferOwnerForSurface(surfaceState.element, now); + if (!owner) return undefined; + const parent = parentSurfaceNodeForOwner(owner, currentStates, now); + const node: HoverSurfaceNode = { + element: surfaceState.element, + signature: surfaceState.signature, + owner, + ...(parent ? { parent } : {}), + }; + hoverSurfaceNodes.set(surfaceState.element, node); + return node; + }; + + const ownedSurfaceForAction = (target: Element): HoverSurfaceNode | undefined => { + const closestSurface = closestHoverSurfaceCandidate(target); + if (closestSurface) { + const owned = hoverSurfaceNodes.get(closestSurface); + if (owned) return owned; + } + const ownedContaining = ownedSurfaceContaining(target); + if (ownedContaining) return ownedContaining; + const now = Date.now(); + const currentStates = collectHoverSurfaceStates(); + pruneGoneHoverSurfaces(currentStates); + const surfaceState = closestSurface + ? currentStates.find((state) => state.element === closestSurface) + : surfaceStateContaining(target, currentStates); + hoverSurfaceStates = hoverSurfaceStateMap(currentStates); + if (!surfaceState) return undefined; + return nodeForUnownedSurface(surfaceState, currentStates, now); + }; + + const pruneGoneHoverSurfaces = (currentStates: HoverSurfaceState[]) => { + const current = new Set(currentStates.map((state) => state.element)); + for (const element of hoverSurfaceNodes.keys()) { + if (!current.has(element)) hoverSurfaceNodes.delete(element); + } + }; + + const parentSurfaceNodeForOwner = ( + owner: HoverCandidate, + currentStates: HoverSurfaceState[], + now: number, + ): HoverSurfaceNode | undefined => { + const ownedParent = ownedSurfaceContaining(owner.element); + if (ownedParent) return ownedParent; + const parentState = surfaceStateContaining(owner.element, currentStates); + if (!parentState) return undefined; + const parentOwner = inferOwnerForSurface(parentState.element, now, owner); + if (!parentOwner) return undefined; + const parentNode: HoverSurfaceNode = { + element: parentState.element, + signature: parentState.signature, + owner: parentOwner, + }; + hoverSurfaceNodes.set(parentState.element, parentNode); + return parentNode; + }; + + const createSurfaceNode = ( + state: HoverSurfaceState, + owner: HoverCandidate, + currentStates: HoverSurfaceState[], + now: number, + ): HoverSurfaceNode | undefined => { + const parent = parentSurfaceNodeForOwner(owner, currentStates, now); + if (parent?.element === state.element) return undefined; + if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) return undefined; + if (!isLikelyHoverSurfaceOwner(owner.element, state.element)) return undefined; + const node: HoverSurfaceNode = { + element: state.element, + signature: state.signature, + owner, + ...(parent ? { parent } : {}), + }; + hoverSurfaceNodes.set(state.element, node); + return node; + }; + + const bindChangedHoverSurfaces = ( + previousStates: Map, + currentStates: HoverSurfaceState[], + now: number, + ) => { + for (const state of currentStates) { + if (previousStates.get(state.element) === state.signature) continue; + const owner = inferOwnerForSurface(state.element, now); + if (!owner) continue; + createSurfaceNode(state, owner, currentStates, now); + } + }; + + const refreshHoverSurfaces = (previousStates = hoverSurfaceStates) => { + const now = Date.now(); + const currentStates = collectHoverSurfaceStates(); + pruneGoneHoverSurfaces(currentStates); + bindChangedHoverSurfaces(previousStates, currentStates, now); + hoverSurfaceStates = hoverSurfaceStateMap(currentStates); + }; + + const scheduleHoverSurfaceRefresh = (previousStates: Map) => { + setTimeout(() => refreshHoverSurfaces(previousStates), 0); + }; + + const emitHoverStep = (hover: HoverCandidate) => { + if (emittedHoverElements.has(hover.element)) return; + emitStep({ + op: "hover", + target: hover.target, + }); + emittedHoverElements.add(hover.element); + }; + + const surfaceOwnerChain = ( + surface: HoverSurfaceNode, + actionElement: Element, + now: number, + ): HoverCandidate[] => { + const chain: HoverCandidate[] = []; + const selected = new WeakSet(); + const surfaces: HoverSurfaceNode[] = []; + let node: HoverSurfaceNode | undefined = surface; + while (node && surfaces.length < 4) { + surfaces.unshift(node); + node = node.parent; + } + for (const owned of surfaces) { + const owner = owned.owner; + if (owner.element === actionElement || actionElement.contains(owner.element)) { + continue; + } + if (selected.has(owner.element)) continue; + if (emittedHoverElements.has(owner.element)) continue; + if (now - owner.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; + selected.add(owner.element); + chain.push(owner); + } + return chain; + }; + + const containedHoverOwnerForAction = ( + actionElement: Element, + now: number, + ): HoverCandidate | undefined => { + for (let index = recentHoverCandidates.length - 1; index >= 0; index -= 1) { + const candidate = recentHoverCandidates[index]; + if (!candidate) continue; + if (candidate.element === actionElement) continue; + if (!candidate.element.contains(actionElement)) continue; + if (now - candidate.recordedAt > HOVER_SURFACE_CONTEXT_MAX_MS) continue; + return candidate; + } + return undefined; + }; + const emitClick = (event: MouseEvent) => { // Only record clicks an LLM can re-identify (named interactive controls). const target = describeEventTarget(eventTarget(event)); @@ -249,6 +682,54 @@ export function startRecordCapture( }); }; + const emitHoverCandidateBeforeAction = (actionTarget: EventTarget | null) => { + if (!(actionTarget instanceof Element)) return; + const actionElement = resolveClickableElement(actionTarget) ?? actionTarget; + const surface = ownedSurfaceForAction(actionElement); + const now = Date.now(); + const containedOwner = surface ? undefined : containedHoverOwnerForAction(actionElement, now); + const hoverChain = surface + ? surfaceOwnerChain(surface, actionElement, now) + : containedOwner + ? [containedOwner] + : []; + if (hoverChain.length === 0) { + pendingHover = null; + return; + } + for (const hover of hoverChain) emitHoverStep(hover); + pendingHover = null; + }; + + const onMouseOver = (event: MouseEvent) => { + const target = eventTarget(event); + if (isOverlayTarget(target)) return; + const now = Date.now(); + if (pendingHover && target instanceof Element && pendingHover.element.contains(target)) { + if (now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) return; + pendingHover = null; + } + const candidate = hoverCandidateFromEvent(target); + if (candidate) { + const previousSurfaceStates = new Map(hoverSurfaceStates); + rememberHoverCandidate(candidate); + refreshHoverSurfaces(previousSurfaceStates); + scheduleHoverSurfaceRefresh(previousSurfaceStates); + } + const hover = candidate?.eligible ? candidate : null; + if (pendingHover && target instanceof Element) { + const relation = decideHoverSurfaceRelation({ triggerElement: pendingHover.element }, target); + if (relation.related && now - pendingHover.recordedAt <= HOVER_BEFORE_CLICK_MAX_MS) { + return; + } + } + if (!hover) return; + if (pendingHover && !shouldReplaceHoverCandidate(pendingHover, hover, now)) { + return; + } + pendingHover = hover; + }; + const onClick = (event: MouseEvent) => { if (event.button !== 0) return; const target = eventTarget(event); @@ -278,6 +759,7 @@ export function startRecordCapture( const fillable = fillableFromTarget(target); if (fillable) { + emitHoverCandidateBeforeAction(fillable); ensureFillSession(fillable); return; } @@ -285,6 +767,7 @@ export function startRecordCapture( if (target instanceof Element) { const nearbyFillable = nearbyFillableFromSearchChrome(target); if (nearbyFillable) { + emitHoverCandidateBeforeAction(nearbyFillable); ensureFillSession(nearbyFillable); return; } @@ -304,6 +787,7 @@ export function startRecordCapture( commitFillSession(); if (target instanceof Element && target.closest("select")) return; + emitHoverCandidateBeforeAction(target); emitClick(event); }; @@ -345,6 +829,7 @@ export function startRecordCapture( commitFillSession(); const target = eventTarget(event); if (target instanceof HTMLSelectElement) { + emitHoverCandidateBeforeAction(target); const values = Array.from(target.selectedOptions).map((opt) => opt.value); const labels = Array.from(target.selectedOptions).map((opt) => (opt.label || opt.textContent || opt.value).trim(), @@ -398,6 +883,7 @@ export function startRecordCapture( } const desc = describeEventTarget(target); if (!desc && !event.key) return; + emitHoverCandidateBeforeAction(target); emitStep({ op: "press", key: event.key, @@ -408,6 +894,7 @@ export function startRecordCapture( }; document.addEventListener("click", onClick, true); + document.addEventListener("mouseover", onMouseOver, true); document.addEventListener("focusin", onFocusIn, true); document.addEventListener("focusout", onFocusOut, true); document.addEventListener("input", onInput, true); @@ -438,6 +925,7 @@ export function startRecordCapture( dispose() { commitFillSession(); document.removeEventListener("click", onClick, true); + document.removeEventListener("mouseover", onMouseOver, true); document.removeEventListener("focusin", onFocusIn, true); document.removeEventListener("focusout", onFocusOut, true); document.removeEventListener("input", onInput, true); diff --git a/apps/extension/src/content/record-hover-surface.ts b/apps/extension/src/content/record-hover-surface.ts new file mode 100644 index 0000000..a47b426 --- /dev/null +++ b/apps/extension/src/content/record-hover-surface.ts @@ -0,0 +1,237 @@ +export interface HoverSurfaceDecision { + related: boolean; + reason?: string; + surface?: Element; +} + +export interface HoverSurfaceContext { + triggerElement: Element; +} + +export interface HoverSurfaceState { + element: Element; + signature: string; +} + +const HOVER_SURFACE_SELECTOR = [ + '[role="menu"]', + '[role="menubar"]', + '[role="listbox"]', + '[role="dialog"]', + '[role="tooltip"]', + '[aria-modal="true"]', + "[data-popper-placement]", + "[data-floating-ui-placement]", + "[data-headlessui-state]", + "[data-radix-popper-content-wrapper]", +].join(","); + +const HOVER_SURFACE_WORD_RE = + /(^|[^a-z0-9])(dropdown-menu|drop-down-menu|menu-list|popover__(popper|content|content-body)|popover-content|popper|popup|flyout|submenu|submenu-wrapper|context-menu|account-menu|user-menu|avatar-menu)(?=[^a-z0-9]|$)/i; +const HOVER_SURFACE_ITEM_RE = + /(^|[^a-z0-9])((dropdown|menu|submenu|context-menu)(-|__?)?(item|link)|item-(text|label|content)|item__?(text|label|content))(?=[^a-z0-9]|$)/i; +const HOVER_SURFACE_MAX_AXIS_GAP = 96; +const HOVER_SURFACE_MAX_DIAGONAL_GAP = 16; +const HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO = 0.5; + +function elementTokenText(el: Element): string { + return [ + el.id, + typeof el.className === "string" ? el.className : "", + el.getAttribute("data-testid"), + el.getAttribute("data-test"), + el.getAttribute("data-cy"), + el.getAttribute("aria-label"), + el.getAttribute("role"), + ] + .filter((value): value is string => typeof value === "string" && value.length > 0) + .join(" "); +} + +export function isRecognizedHoverSurfaceElement(el: Element): boolean { + if (el.matches(HOVER_SURFACE_SELECTOR)) return true; + const tokenText = elementTokenText(el); + if (HOVER_SURFACE_WORD_RE.test(tokenText) && !HOVER_SURFACE_ITEM_RE.test(tokenText)) { + return true; + } + return false; +} + +function isVisibleSurfaceElement(el: Element): boolean { + if (!(el instanceof HTMLElement)) return false; + const style = getComputedStyle(el); + if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") { + return false; + } + return rectFor(el) !== null; +} + +export function closestRecognizedHoverSurface(element: Element): Element | null { + let node: Element | null = element; + let depth = 0; + while (node && node !== document.body && node !== document.documentElement && depth < 10) { + if (isRecognizedHoverSurfaceElement(node)) return node; + node = node.parentElement; + depth += 1; + } + return null; +} + +export function isPositionedFloatingElement(el: Element): boolean { + if (!(el instanceof HTMLElement)) return false; + const style = getComputedStyle(el); + if (style.position !== "absolute") return false; + if (style.pointerEvents === "none" || style.visibility === "hidden" || style.display === "none") { + return false; + } + const rect = rectFor(el); + if (!rect) return false; + const viewportArea = window.innerWidth * window.innerHeight; + const area = rect.width * rect.height; + return ( + area > 0 && (!viewportArea || area <= viewportArea * HOVER_SURFACE_MAX_VIEWPORT_AREA_RATIO) + ); +} + +export function isHoverSurfaceCandidateElement(el: Element): boolean { + return isRecognizedHoverSurfaceElement(el) || isPositionedFloatingElement(el); +} + +export function closestHoverSurfaceCandidate(element: Element): Element | null { + let node: Element | null = element; + let depth = 0; + while (node && node !== document.body && node !== document.documentElement && depth < 10) { + if (isHoverSurfaceCandidateElement(node)) return node; + node = node.parentElement; + depth += 1; + } + return null; +} + +function closestFloatingSurface(clickElement: Element): Element | null { + let node: Element | null = clickElement; + let depth = 0; + while (node && node !== document.body && node !== document.documentElement && depth < 10) { + if (isPositionedFloatingElement(node)) return node; + node = node.parentElement; + depth += 1; + } + return null; +} + +function hoverSurfaceSignature(el: Element): string { + const rect = rectFor(el); + return [ + rect + ? `${Math.round(rect.left)},${Math.round(rect.top)},${Math.round(rect.width)},${Math.round(rect.height)}` + : "", + (el.textContent ?? "").trim().slice(0, 512), + ].join("|"); +} + +export function collectHoverSurfaceStates(): HoverSurfaceState[] { + const states: HoverSurfaceState[] = []; + for (const el of document.querySelectorAll("*")) { + if (isHoverSurfaceCandidateElement(el) && isVisibleSurfaceElement(el)) { + states.push({ element: el, signature: hoverSurfaceSignature(el) }); + } + } + return states; +} + +function rectFor(el: Element): DOMRect | null { + const rect = el.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) return null; + return rect; +} + +function surfaceRelationMetrics(surface: Element, trigger: Element) { + const surfaceRect = rectFor(surface); + const triggerRect = rectFor(trigger); + if (!surfaceRect || !triggerRect) return null; + const horizontalOverlap = + Math.min(triggerRect.right, surfaceRect.right) - Math.max(triggerRect.left, surfaceRect.left); + const verticalOverlap = + Math.min(triggerRect.bottom, surfaceRect.bottom) - Math.max(triggerRect.top, surfaceRect.top); + const horizontalGap = Math.max( + 0, + triggerRect.left - surfaceRect.right, + surfaceRect.left - triggerRect.right, + ); + const verticalGap = Math.max( + 0, + triggerRect.top - surfaceRect.bottom, + surfaceRect.top - triggerRect.bottom, + ); + return { + surfaceRect, + triggerRect, + horizontalOverlap, + verticalOverlap, + horizontalGap, + verticalGap, + }; +} + +function isNearTrigger(surface: Element, trigger: Element): boolean { + const metrics = surfaceRelationMetrics(surface, trigger); + if (!metrics) return true; + const { horizontalOverlap, verticalOverlap, horizontalGap, verticalGap } = metrics; + if (horizontalOverlap > 0 && verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true; + if (verticalOverlap > 0 && horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP) return true; + return ( + horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP && verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP + ); +} + +export function isLikelyHoverSurfaceOwner(trigger: Element, surface: Element): boolean { + const metrics = surfaceRelationMetrics(surface, trigger); + if (!metrics) return true; + const { + surfaceRect, + triggerRect, + horizontalOverlap, + verticalOverlap, + horizontalGap, + verticalGap, + } = metrics; + const opensVertically = + horizontalOverlap > 0 && + verticalGap <= HOVER_SURFACE_MAX_AXIS_GAP && + (triggerRect.bottom <= surfaceRect.top + 8 || surfaceRect.bottom <= triggerRect.top + 8); + const opensSideways = + verticalOverlap > 0 && + horizontalGap <= HOVER_SURFACE_MAX_AXIS_GAP && + (triggerRect.right <= surfaceRect.left + 8 || surfaceRect.right <= triggerRect.left + 8); + const opensFromCorner = + horizontalGap > 0 && + verticalGap > 0 && + horizontalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP && + verticalGap <= HOVER_SURFACE_MAX_DIAGONAL_GAP; + return opensVertically || opensSideways || opensFromCorner; +} + +export function decideHoverSurfaceRelation( + context: HoverSurfaceContext, + clickElement: Element, +): HoverSurfaceDecision { + const clickSurface = closestRecognizedHoverSurface(clickElement); + if (clickSurface && isNearTrigger(clickSurface, context.triggerElement)) { + return { related: true, reason: "click-inside-hover-surface", surface: clickSurface }; + } + + const floatingSurface = closestFloatingSurface(clickElement); + if (floatingSurface && isNearTrigger(floatingSurface, context.triggerElement)) { + return { + related: true, + reason: "click-inside-near-floating-surface", + surface: floatingSurface, + }; + } + + if (context.triggerElement.contains(clickElement) && clickElement !== context.triggerElement) { + return { related: true, reason: "click-inside-trigger-contained-surface" }; + } + + return { related: false, reason: "click-outside-hover-surface" }; +} diff --git a/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts b/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts new file mode 100644 index 0000000..dd73c24 --- /dev/null +++ b/apps/extension/src/lib/__tests__/hover-trigger-policy.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "vitest"; +import { evaluateHoverTrigger } from "../hover-trigger-policy"; + +describe("evaluateHoverTrigger", () => { + it("accepts explicit popup triggers", () => { + const decision = evaluateHoverTrigger({ + tag: "button", + role: "button", + label: "Open user navigation menu", + attrs: { "aria-haspopup": "menu", "aria-expanded": "false" }, + rect: { x: 900, y: 8, w: 32, h: 32 }, + cursor: "pointer", + pointerEvents: "auto", + }); + + expect(decision.eligible).toBe(true); + expect(decision.reasons).toEqual( + expect.arrayContaining(["aria-haspopup", "collapsed", "popup-signal"]), + ); + }); + + it("accepts compact topbar image buttons without popup attributes", () => { + const decision = evaluateHoverTrigger({ + tag: "button", + role: "button", + label: "image", + attrs: {}, + rect: { x: 900, y: 8, w: 32, h: 32 }, + pointerEvents: "auto", + hasGraphicDescendant: true, + }); + + expect(decision.eligible).toBe(true); + expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"])); + }); + + it("accepts compact topbar image links without popup attributes", () => { + const decision = evaluateHoverTrigger({ + tag: "a", + role: "link", + label: "image", + attrs: {}, + rect: { x: 900, y: 8, w: 32, h: 32 }, + pointerEvents: "auto", + hasGraphicDescendant: true, + }); + + expect(decision.eligible).toBe(true); + expect(decision.reasons).toEqual(expect.arrayContaining(["icon-only", "topbar", "compact"])); + }); + + it("rejects topbar menu links without surface trigger evidence", () => { + const decision = evaluateHoverTrigger({ + tag: "a", + role: "link", + label: "My profile", + attrs: {}, + rect: { x: 820, y: 72, w: 96, h: 32 }, + cursor: "pointer", + pointerEvents: "auto", + }); + + expect(decision.eligible).toBe(false); + }); + + it("rejects text navigation items even when their attributes contain popup words", () => { + const decision = evaluateHoverTrigger({ + tag: "a", + role: "link", + label: "My profile", + attrs: { class: "dropdown-item profile-link" }, + rect: { x: 820, y: 72, w: 96, h: 32 }, + cursor: "pointer", + pointerEvents: "auto", + }); + + expect(decision.eligible).toBe(false); + }); + + it("rejects labelled dropdown list items as hover triggers", () => { + const decision = evaluateHoverTrigger({ + tag: "li", + label: "文档C+D", + attrs: { class: "dropdown-item", tabindex: "0" }, + rect: { x: 820, y: 48, w: 160, h: 32 }, + cursor: "pointer", + pointerEvents: "auto", + }); + + expect(decision.eligible).toBe(false); + }); + + it("rejects ordinary controls without hover popup signals", () => { + const decision = evaluateHoverTrigger({ + tag: "button", + role: "button", + label: "Search", + attrs: {}, + rect: { x: 100, y: 240, w: 120, h: 36 }, + pointerEvents: "auto", + }); + + expect(decision.eligible).toBe(false); + }); + + it("rejects unsafe or non-visible triggers", () => { + expect( + evaluateHoverTrigger({ + tag: "input", + attrs: {}, + rect: { x: 10, y: 10, w: 80, h: 24 }, + pointerEvents: "auto", + }).eligible, + ).toBe(false); + expect( + evaluateHoverTrigger({ + tag: "button", + role: "button", + attrs: { hidden: "" }, + rect: { x: 10, y: 10, w: 80, h: 24 }, + pointerEvents: "auto", + }).eligible, + ).toBe(false); + }); +}); diff --git a/apps/extension/src/lib/__tests__/trace-reducer.test.ts b/apps/extension/src/lib/__tests__/trace-reducer.test.ts index 777a86c..8cc773c 100644 --- a/apps/extension/src/lib/__tests__/trace-reducer.test.ts +++ b/apps/extension/src/lib/__tests__/trace-reducer.test.ts @@ -130,6 +130,30 @@ describe("reduceTraceSteps", () => { }); }); + it("keeps hover steps before menu clicks", () => { + const { steps } = reduceTraceSteps( + [ + { + op: "hover", + target: { tag: "span", role: "button", name: "Account" }, + page_url: "https://example.com/app", + }, + { + op: "click", + target: { tag: "a", role: "link", name: "Profile" }, + page_url: "https://example.com/app", + }, + ], + "https://example.com/app", + ); + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + page: "p1", + target: { name: "Account" }, + }); + }); + it("resolveTraceStartUrl prefers explicit start URL", () => { expect( resolveTraceStartUrl( diff --git a/apps/extension/src/lib/describe-target.ts b/apps/extension/src/lib/describe-target.ts index 7d3898a..0a1fcb2 100644 --- a/apps/extension/src/lib/describe-target.ts +++ b/apps/extension/src/lib/describe-target.ts @@ -228,6 +228,20 @@ export function resolveClickableElement(target: Element): Element | null { return null; } +export function resolveHoverElement(target: Element): Element | null { + const clickable = resolveClickableElement(target); + if (clickable) return clickable; + + let node: Element | null = target; + let depth = 0; + while (node && node !== document.body && node !== document.documentElement && depth < 8) { + if (looksClickable(node)) return node; + node = node.parentElement; + depth += 1; + } + return null; +} + /** * Teachable clicks: the LLM must get a label it can later find via snapshot * (visible name), or at least a form `name_attr` for checkbox/radio. diff --git a/apps/extension/src/lib/hover-trigger-policy.ts b/apps/extension/src/lib/hover-trigger-policy.ts new file mode 100644 index 0000000..23922b4 --- /dev/null +++ b/apps/extension/src/lib/hover-trigger-policy.ts @@ -0,0 +1,178 @@ +export interface HoverTriggerRect { + x: number; + y: number; + w: number; + h: number; +} + +export interface HoverTriggerSignals { + tag: string; + role?: string; + label?: string; + attrs?: Record; + rect?: HoverTriggerRect | null; + cursor?: string; + pointerEvents?: string; + hasGraphicDescendant?: boolean; + cssHoverMatch?: boolean; +} + +export interface HoverTriggerDecision { + eligible: boolean; + score: number; + reasons: string[]; +} + +const HOVER_TRIGGER_MIN_SCORE = 45; +const HOVER_POPUP_SIGNAL_RE = + /(menu|dropdown|popover|popup|avatar|profile|account|user|more|ellipsis|caret)/; +const HOVER_TEXT_ITEM_RE = /(^|[\s_-])(dropdown-item|menu-item|context-menu-item|option)($|[\s_-])/; + +function attr(signals: HoverTriggerSignals, name: string): string | undefined { + return signals.attrs?.[name.toLowerCase()]; +} + +export function isUnsafeHoverTrigger(signals: HoverTriggerSignals): boolean { + const tag = signals.tag.toLowerCase(); + if (["input", "textarea", "select", "option"].includes(tag)) return true; + if ((attr(signals, "contenteditable") ?? "").toLowerCase() === "true") return true; + if (attr(signals, "disabled") !== undefined || attr(signals, "inert") !== undefined) return true; + return (attr(signals, "aria-disabled") ?? "").toLowerCase() === "true"; +} + +export function hasHoverPopupSignal(signals: HoverTriggerSignals): boolean { + const attrs = signals.attrs ?? {}; + const haystack = [ + attrs.id, + attrs.class, + attrs["data-testid"], + attrs["data-test"], + attrs["data-cy"], + attrs["aria-label"], + attrs["aria-haspopup"], + attrs["aria-controls"], + attrs.title, + signals.role === "button" ? signals.label : undefined, + ] + .filter(Boolean) + .join(" ") + .toLowerCase(); + return HOVER_POPUP_SIGNAL_RE.test(haystack); +} + +export function hasStrongHoverExpansionSignal(signals: HoverTriggerSignals): boolean { + return ( + signals.cssHoverMatch === true || + attr(signals, "aria-haspopup") !== undefined || + (attr(signals, "aria-expanded") ?? "").toLowerCase() === "false" + ); +} + +export function hasDirectHoverInteractiveSignal(signals: HoverTriggerSignals): boolean { + const tag = signals.tag.toLowerCase(); + const role = (signals.role ?? "").toLowerCase(); + return ( + ["button", "a", "summary"].includes(tag) || + ["button", "link", "menuitem", "tab", "combobox"].includes(role) || + attr(signals, "tabindex") !== undefined || + signals.cursor === "pointer" || + attr(signals, "onclick") !== undefined || + attr(signals, "onmouseenter") !== undefined || + attr(signals, "onmouseover") !== undefined || + attr(signals, "aria-haspopup") !== undefined || + attr(signals, "aria-expanded") === "false" + ); +} + +export function evaluateHoverTrigger(signals: HoverTriggerSignals): HoverTriggerDecision { + const rect = signals.rect; + if ( + !rect || + rect.w <= 0 || + rect.h <= 0 || + signals.pointerEvents === "none" || + attr(signals, "hidden") !== undefined || + attr(signals, "inert") !== undefined || + (attr(signals, "aria-hidden") ?? "").toLowerCase() === "true" || + isUnsafeHoverTrigger(signals) + ) { + return { eligible: false, score: 0, reasons: [] }; + } + + const area = rect.w * rect.h; + if (area <= 0 || area > 160_000) { + return { eligible: false, score: 0, reasons: [] }; + } + + const reasons: string[] = []; + let score = 0; + const tag = signals.tag.toLowerCase(); + const role = (signals.role ?? "").toLowerCase(); + const label = signals.label?.trim(); + const graphic = signals.hasGraphicDescendant === true; + const directInteractive = hasDirectHoverInteractiveSignal(signals); + const popupSignal = hasHoverPopupSignal(signals); + const isTextNavigationItem = + (tag === "a" || + tag === "li" || + role === "link" || + role === "option" || + role.startsWith("menuitem") || + HOVER_TEXT_ITEM_RE.test(attr(signals, "class") ?? "")) && + !graphic && + !!label && + label.toLowerCase() !== "image"; + const compactTopbarIcon = + (role === "button" || role === "link" || tag === "button" || tag === "a") && + graphic && + rect.y < 120 && + rect.w <= 96 && + rect.h <= 96 && + (!label || label.toLowerCase() === "image"); + const surfaceTriggerEvidence = + (popupSignal && !isTextNavigationItem) || + compactTopbarIcon || + hasStrongHoverExpansionSignal(signals); + if (!directInteractive || !surfaceTriggerEvidence) { + return { eligible: false, score: 0, reasons: [] }; + } + + if (attr(signals, "aria-haspopup") !== undefined) { + score += 80; + reasons.push("aria-haspopup"); + } + if ((attr(signals, "aria-expanded") ?? "").toLowerCase() === "false") { + score += 55; + reasons.push("collapsed"); + } + if (popupSignal) { + score += 45; + reasons.push("popup-signal"); + } + if (tag === "button" || role === "button") { + score += 30; + reasons.push("button"); + } + if (signals.cursor === "pointer") { + score += 25; + reasons.push("pointer"); + } + if (graphic && (!label || label.toLowerCase() === "image")) { + score += 40; + reasons.push("icon-only"); + } + if (rect.y < 120) { + score += 25; + reasons.push("topbar"); + } + if (rect.w <= 80 && rect.h <= 80) { + score += 20; + reasons.push("compact"); + } + if (signals.cssHoverMatch) { + score += 40; + reasons.push("css-hover"); + } + + return { eligible: score >= HOVER_TRIGGER_MIN_SCORE, score, reasons }; +} diff --git a/apps/extension/src/lib/record-bridge.ts b/apps/extension/src/lib/record-bridge.ts index cb5d446..cb7eb88 100644 --- a/apps/extension/src/lib/record-bridge.ts +++ b/apps/extension/src/lib/record-bridge.ts @@ -33,7 +33,7 @@ export interface RecordStartMessage { } export interface RecordStepPayload { - op: "click" | "fill" | "press" | "select" | "navigate"; + op: "click" | "hover" | "fill" | "press" | "select" | "navigate"; target?: TargetDescriptor; value?: string; key?: string; diff --git a/apps/extension/src/lib/recording-step-buffer.ts b/apps/extension/src/lib/recording-step-buffer.ts index 0c7c26e..c1c5086 100644 --- a/apps/extension/src/lib/recording-step-buffer.ts +++ b/apps/extension/src/lib/recording-step-buffer.ts @@ -21,6 +21,14 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null { ...(pageUrl ? { page_url: pageUrl } : {}), } : null; + case "hover": + return payload.target + ? { + op: "hover", + target: payload.target, + ...(pageUrl ? { page_url: pageUrl } : {}), + } + : null; case "fill": return payload.target ? { diff --git a/apps/extension/src/lib/trace-reducer.ts b/apps/extension/src/lib/trace-reducer.ts index e3572ee..6da0c3d 100644 --- a/apps/extension/src/lib/trace-reducer.ts +++ b/apps/extension/src/lib/trace-reducer.ts @@ -139,6 +139,13 @@ function toV2Step( }, effectForNavigation(step.navigated_to, urlToId), ); + case "hover": + return { + op: "hover", + id, + page, + target: step.target, + }; case "fill": return { op: "fill", diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 060b30a..cb4e5ea 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -9,6 +9,8 @@ import type { } from "@/transport/types"; import { ToolDispatcher } from "../dispatcher"; +type TestDispatcherCdp = NonNullable[0]["cdp"]>; + function fakeTransport() { const handlers = new Set(); const stateHandlers = new Set(); @@ -129,7 +131,7 @@ describe("ToolDispatcher", () => { setUserAgentOverride: vi.fn(async () => {}), setTouchEmulationEnabled: vi.fn(async () => {}), }; - const dispatcher = new ToolDispatcher({ transport, sessions, cdp }); + const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); dispatcher.start(); deliver(makeRequest("tool.console", { session_id: "aa11" })); @@ -172,7 +174,7 @@ describe("ToolDispatcher", () => { setUserAgentOverride: vi.fn(async () => {}), setTouchEmulationEnabled: vi.fn(async () => {}), }; - const dispatcher = new ToolDispatcher({ transport, sessions, cdp }); + const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); dispatcher.start(); deliver(makeRequest("tool.session_stop", { session_id: "aa11" })); @@ -311,6 +313,277 @@ describe("ToolDispatcher", () => { expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11"); }); + it("reasserts remembered hover before follow-up work and releases only after actions", async () => { + vi.stubGlobal("chrome", { + tabs: { + sendMessage: vi.fn(async () => undefined), + }, + }); + const { transport } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 1), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + const order: string[] = []; + const cdp = { + send: vi.fn(async () => { + order.push("hover"); + return {} as T; + }), + detachSession: vi.fn(async () => {}), + ensureNetworkCapture: vi.fn(async () => {}), + networkEntriesSince: vi.fn(() => ({ + tab_id: 7, + entries: [], + next_since: 0, + truncated: false, + })), + setDeviceMetricsOverride: vi.fn(async () => {}), + clearDeviceMetricsOverride: vi.fn(async () => {}), + setUserAgentOverride: vi.fn(async () => {}), + setTouchEmulationEnabled: vi.fn(async () => {}), + }; + const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); + + ( + dispatcher as unknown as { rememberHover: (sessionId: string, result: unknown) => unknown } + ).rememberHover("aa11", { + tab_id: 7, + x: 10, + y: 20, + }); + const helpers = dispatcher as unknown as { + withHoverReassert: ( + params: { session_id: string; tab_id?: number }, + work: () => Promise, + options?: { releaseAfter?: boolean }, + ) => Promise; + setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise; + }; + + await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => { + order.push("observe"); + return {}; + }); + expect(order).toEqual(["hover", "observe"]); + expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + + await helpers.setHoverBypass("aa11", 7, true); + await helpers.withHoverReassert( + { session_id: "aa11", tab_id: 7 }, + async () => { + order.push("click"); + return {}; + }, + { releaseAfter: true }, + ); + + expect(cdp.send).toHaveBeenLastCalledWith(7, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: 10, + y: 20, + }); + expect(chrome.tabs.sendMessage).toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + }); + + it("does not reassert or disable hover latches from another session", async () => { + vi.stubGlobal("chrome", { + tabs: { + sendMessage: vi.fn(async () => undefined), + }, + }); + const { transport } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 1), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + const cdp = { + send: vi.fn(async () => ({}) as T), + detachSession: vi.fn(async () => {}), + ensureNetworkCapture: vi.fn(async () => {}), + networkEntriesSince: vi.fn(() => ({ + tab_id: 7, + entries: [], + next_since: 0, + truncated: false, + })), + setDeviceMetricsOverride: vi.fn(async () => {}), + clearDeviceMetricsOverride: vi.fn(async () => {}), + setUserAgentOverride: vi.fn(async () => {}), + setTouchEmulationEnabled: vi.fn(async () => {}), + }; + const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); + const helpers = dispatcher as unknown as { + rememberHover: (sessionId: string, result: unknown) => unknown; + setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise; + withHoverReassert: ( + params: { session_id: string; tab_id?: number }, + work: () => Promise, + options?: { releaseAfter?: boolean }, + ) => Promise; + releaseHoverLatch: (sessionId?: string, tabId?: number) => Promise; + }; + + helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 }); + await helpers.setHoverBypass("aa11", 7, true); + + await helpers.withHoverReassert({ session_id: "bb22", tab_id: 7 }, async () => ({}), { + releaseAfter: true, + }); + + expect(cdp.send).not.toHaveBeenCalled(); + expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + + await helpers.releaseHoverLatch("aa11", 7); + expect(chrome.tabs.sendMessage).toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + }); + + it("transfers hover overlay bypass ownership between sessions", async () => { + vi.stubGlobal("chrome", { + tabs: { + sendMessage: vi.fn(async () => undefined), + }, + }); + const { transport } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 1), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + const dispatcher = new ToolDispatcher({ transport, sessions }); + const helpers = dispatcher as unknown as { + setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise; + }; + + await helpers.setHoverBypass("aa11", 7, true); + await helpers.setHoverBypass("bb22", 7, true); + await helpers.setHoverBypass("aa11", 7, false); + expect(chrome.tabs.sendMessage).not.toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + + await helpers.setHoverBypass("bb22", 7, false); + expect(chrome.tabs.sendMessage).toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + expect(chrome.tabs.sendMessage).toHaveBeenCalledTimes(2); + }); + + it("limits default-target hover reassertion to the active tab", async () => { + vi.stubGlobal("chrome", { + tabs: { + query: vi.fn(async () => [{ id: 9, windowId: 4242, active: true }]), + get: vi.fn(async (tabId: number) => ({ id: tabId, windowId: 4242, active: tabId === 9 })), + sendMessage: vi.fn(async () => undefined), + }, + }); + const { transport } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 4242), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + await sessions.start("aa11"); + const cdp = { + send: vi.fn(async () => ({}) as T), + detachSession: vi.fn(async () => {}), + ensureNetworkCapture: vi.fn(async () => {}), + networkEntriesSince: vi.fn(() => ({ + tab_id: 9, + entries: [], + next_since: 0, + truncated: false, + })), + setDeviceMetricsOverride: vi.fn(async () => {}), + clearDeviceMetricsOverride: vi.fn(async () => {}), + setUserAgentOverride: vi.fn(async () => {}), + setTouchEmulationEnabled: vi.fn(async () => {}), + }; + const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); + const helpers = dispatcher as unknown as { + rememberHover: (sessionId: string, result: unknown) => unknown; + withHoverReassert: ( + params: { session_id: string; tab_id?: number }, + work: () => Promise, + ) => Promise; + }; + + helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 }); + helpers.rememberHover("aa11", { tab_id: 9, x: 30, y: 40 }); + + await helpers.withHoverReassert({ session_id: "aa11" }, async () => ({})); + + expect(cdp.send).toHaveBeenCalledTimes(1); + expect(cdp.send).toHaveBeenCalledWith(9, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: 30, + y: 40, + }); + }); + + it("releases the current session hover latch before navigation-style work", async () => { + vi.stubGlobal("chrome", { + tabs: { + sendMessage: vi.fn(async () => undefined), + }, + }); + const { transport } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 1), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + const dispatcher = new ToolDispatcher({ transport, sessions }); + const helpers = dispatcher as unknown as { + rememberHover: (sessionId: string, result: unknown) => unknown; + setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise; + withHoverReleaseForRequest: ( + params: { session_id: string; tab_id?: number }, + work: () => Promise, + ) => Promise; + withHoverReassert: ( + params: { session_id: string; tab_id?: number }, + work: () => Promise, + ) => Promise; + }; + helpers.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 }); + await helpers.setHoverBypass("aa11", 7, true); + + await helpers.withHoverReleaseForRequest({ session_id: "aa11", tab_id: 7 }, async () => ({})); + await helpers.withHoverReassert({ session_id: "aa11", tab_id: 7 }, async () => ({})); + + expect(chrome.tabs.sendMessage).toHaveBeenCalledWith( + 7, + expect.objectContaining({ enabled: false }), + ); + }); + it("disconnects the transport when send() fails so keepalive can rebuild", async () => { const { transport, deliver } = fakeTransport(); const sessions = new SessionManager({ diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index de56af4..6f8eeed 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -4,6 +4,7 @@ import type { CdpRunner } from "@/tools/shared"; import { handleClick, handleFill, + handleHover, handlePress, handleSelect, modifiersBitfield, @@ -435,6 +436,50 @@ describe("handleClick", () => { }); }); +describe("handleHover", () => { + it("keeps overlay bypass enabled after a successful hover when requested", async () => { + const bypassOverlay = vi.fn().mockResolvedValue(undefined); + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), + "Runtime.evaluate": (params: unknown) => { + const expr = String((params as { expression?: string })?.expression ?? ""); + if (expr.includes("overlayHostPresent") && !expr.includes("hitIndex")) { + return { result: { value: { overlayHostPresent: true, overlayHostConnected: true } } }; + } + if (expr.includes("hitIndex")) { + return { + result: { + value: { overlayHostPresent: true, overlayHostConnected: true, hitIndex: 0 }, + }, + }; + } + throw new Error(`unexpected Runtime.evaluate: ${expr.slice(0, 80)}`); + }, + "Input.dispatchMouseEvent": () => ({}), + }); + + const res = await handleHover( + sm, + { session_id: "aa11", ref: "@e3", settle_ms: 0 }, + { + cdp: fake.cdp, + tabsApi: fake.tabsApi, + bypassOverlay, + keepOverlayBypassAfterHover: true, + }, + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", x: 60, y: 40 }); + expect(bypassOverlay).toHaveBeenCalledTimes(1); + expect(bypassOverlay).toHaveBeenCalledWith(4, true); + }); +}); + describe("handleFill", () => { it("returns not_found for unknown ref", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index d6cd621..a85fd64 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -1430,7 +1430,7 @@ describe("buildVomScene", () => { expect(renderVom(scene).text).toContain("[§ active: Reviews (12)]"); }); - it("maps hover surface probes to matching node labels", () => { + it("maps hover surface probes to matching backend node ids", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", @@ -1451,7 +1451,7 @@ describe("buildVomScene", () => { iframeNodes: new Map(), excludedBackendNodeIds: new Set(), surfaceProbes: [ - { triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] }, + { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, ], nodes: [ { @@ -1480,7 +1480,260 @@ describe("buildVomScene", () => { expect(scene.surfaces).toEqual([ { triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, ]); - expect(renderVom(scene).text).toContain('@e1 button "Products" [hover: Shoes | Bags]'); + expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]'); + }); + + it("maps hover surface probes to rendered descendants in the same trigger subtree", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 10, + childIds: ["2"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "image" }, + backendDOMNodeId: 21, + }, + ]; + const scene = buildVomScene(axNodes, { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + surfaceProbes: [ + { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] }, + ], + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "div", + attrs: { class: "tg-avatar" }, + rect: { x: 900, y: 10, w: 30, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 21, + parentBackendNodeId: 20, + tag: "div", + attrs: { class: "tg-avatar__inner" }, + rect: { x: 902, y: 12, w: 26, h: 26 }, + paintOrder: 2, + position: "static", + pointerEvents: "auto", + }, + ], + }); + + expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]'); + }); + + it("deduplicates hover surface probes by original trigger backend id", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 10, + childIds: ["2"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "image" }, + backendDOMNodeId: 21, + }, + ]; + const scene = buildVomScene(axNodes, { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + surfaceProbes: [ + { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] }, + { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] }, + { triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] }, + ], + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "div", + attrs: { class: "tg-avatar" }, + rect: { x: 900, y: 10, w: 30, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 21, + parentBackendNodeId: 20, + tag: "div", + attrs: { class: "tg-avatar__inner" }, + rect: { x: 902, y: 12, w: 26, h: 26 }, + paintOrder: 2, + position: "static", + pointerEvents: "auto", + }, + ], + }); + + expect(scene.surfaces).toEqual([ + { triggerId: 21, triggerAction: "hover", subItems: ["My profile"] }, + ]); + }); + + it("maps hover surface probes to DOM-recovered custom controls", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 10, + childIds: ["2"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "img" }, + name: { type: "computedString", value: "image" }, + backendDOMNodeId: 21, + }, + ]; + const scene = buildVomScene(axNodes, { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + surfaceProbes: [ + { + triggerBackendNodeId: 20, + triggerPoint: { x: 915, y: 25 }, + triggerAction: "hover", + subItems: ["My profile", "Sign out"], + }, + ], + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + cursor: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "div", + attrs: { class: "tg-avatar" }, + rect: { x: 900, y: 10, w: 30, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + cursor: "pointer", + }, + { + backendNodeId: 21, + parentBackendNodeId: 20, + tag: "div", + attrs: { class: "tg-avatar__inner" }, + rect: { x: 902, y: 12, w: 26, h: 26 }, + paintOrder: 2, + position: "static", + pointerEvents: "auto", + cursor: "pointer", + }, + ], + }); + + expect(scene.surfaces).toEqual([ + { triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] }, + ]); + expect(renderVom(scene).text).toContain( + '@e1 button "image" [hover first: My profile | Sign out]', + ); + }); + + it("does not attach hover probes to distant geometry matches", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 10, + childIds: ["2"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "link" }, + name: { type: "computedString", value: "0" }, + backendDOMNodeId: 42, + }, + ]; + const scene = buildVomScene(axNodes, { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + surfaceProbes: [ + { + triggerBackendNodeId: 999, + triggerPoint: { x: 900, y: 20 }, + triggerAction: "hover", + subItems: ["My profile"], + }, + ], + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 42, + parentBackendNodeId: 10, + tag: "a", + attrs: {}, + rect: { x: 860, y: 120, w: 40, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + ], + }); + + expect(scene.surfaces).toBeUndefined(); + expect(renderVom(scene).text).not.toContain("[hover first:"); }); it("enriches names and active scope signals from AX properties", () => { @@ -1953,7 +2206,7 @@ describe("handleSnapshot", () => { }); const res = await handleObserve( sm, - { session_id: "aa11" }, + { session_id: "aa11", debug_surfaces: true }, { cdp: { send: send as unknown as ( @@ -1974,6 +2227,7 @@ describe("handleSnapshot", () => { ); if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res.debug).toEqual({ surface_probes: [] }); expect(send).toHaveBeenCalledWith( 4, "Runtime.evaluate", diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index dd419fb..26ec56d 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -8,6 +8,8 @@ import type { EvaluateParams, FillParams, GetHtmlParams, + HoverParams, + HoverResult, NavigateBackParams, NavigateForwardParams, NavigateParams, @@ -33,7 +35,7 @@ import { handleConsole } from "./console"; import { type EmulateCdpRunner, handleEmulate } from "./emulate"; import { handleEvaluate } from "./evaluate"; import { handleRequestHelp } from "./human-loop"; -import { handleClick, handleFill, handlePress, handleSelect } from "./interaction"; +import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction"; import { handleNavigate, handleNavigateBack, @@ -56,7 +58,7 @@ import { type SessionStartParams, type SessionStopParams, } from "./session"; -import { chromeTabsApi } from "./shared"; +import { chromeTabsApi, lookupSession, resolveTargetTab } from "./shared"; import { type BorrowConfirmationApprover, handleTabBorrow, @@ -81,6 +83,18 @@ type DispatcherCdpRunner = CdpRunner & detachSession(sessionId: string): Promise; }; +interface HoverLatch { + sessionId: string; + tabId: number; + x: number; + y: number; +} + +interface HoverLatchScope { + session_id: string; + tab_id?: number; +} + export interface DispatcherDeps { transport: Transport; sessions: SessionManager; @@ -126,6 +140,8 @@ export class ToolDispatcher { private readonly approveBorrow?: BorrowConfirmationApprover; private readonly helpNotificationCopy?: () => { title: string; body: string }; private subscription: { dispose(): void } | null = null; + private readonly hoverBypassTabs = new Map(); + private readonly hoverLatches = new Map(); /** * Per-rpc-id `AbortController` registry. Populated inside * [`dispatch`] before we await the tool handler and torn down in @@ -271,16 +287,20 @@ export class ToolDispatcher { switch (req.method) { case "tool.session_start": return handleSessionStart(this.sessions, req.params as SessionStartParams); - case "tool.session_stop": + case "tool.session_stop": { + await this.releaseHoverLatch((req.params as SessionStopParams).session_id); return handleSessionStop(this.sessions, req.params as SessionStopParams, { cdp: this.cdp, }); + } case "tool.tab_list": return handleTabList(this.sessions, req.params as TabListParams); case "tool.tab_create": return handleTabCreate(this.sessions, req.params as TabCreateParams); case "tool.tab_close": - return handleTabClose(this.sessions, req.params as TabCloseParams); + return this.withHoverReleaseForRequest(req.params as TabCloseParams, () => + handleTabClose(this.sessions, req.params as TabCloseParams), + ); case "tool.tab_select": return handleTabSelect(this.sessions, req.params as TabSelectParams); case "tool.tab_borrow": @@ -319,17 +339,31 @@ export class ToolDispatcher { this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined, ); case "tool.snapshot": - return handleSnapshot( - this.sessions, - req.params as SnapshotParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined, + return this.withHoverReassert(req.params as SnapshotParams, () => + handleSnapshot( + this.sessions, + req.params as SnapshotParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined, + ), ); - case "tool.observe": - return handleObserve( - this.sessions, - req.params as ObserveParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined, + case "tool.observe": { + const params = req.params as ObserveParams; + const hoverScope = await this.resolveHoverLatchScope(params); + return this.withHoverReassert(params, () => + handleObserve( + this.sessions, + params, + this.cdp + ? { + cdp: this.cdp, + tabsApi: chromeTabsCaptureApi, + conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope), + hoverProbeBypassOverlay: bypassOverlay, + } + : undefined, + ), ); + } case "tool.get_html": return handleGetHtml( this.sessions, @@ -337,68 +371,95 @@ export class ToolDispatcher { this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined, ); case "tool.navigate": - return handleNavigate( - this.sessions, - req.params as NavigateParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + return this.withHoverReleaseForRequest(req.params as NavigateParams, () => + handleNavigate( + this.sessions, + req.params as NavigateParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), ); case "tool.navigate_back": - return handleNavigateBack( - this.sessions, - req.params as NavigateBackParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + return this.withHoverReleaseForRequest(req.params as NavigateBackParams, () => + handleNavigateBack( + this.sessions, + req.params as NavigateBackParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), ); case "tool.navigate_forward": - return handleNavigateForward( - this.sessions, - req.params as NavigateForwardParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + return this.withHoverReleaseForRequest(req.params as NavigateForwardParams, () => + handleNavigateForward( + this.sessions, + req.params as NavigateForwardParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), ); case "tool.reload": - return handleReload( - this.sessions, - req.params as ReloadParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + return this.withHoverReleaseForRequest(req.params as ReloadParams, () => + handleReload( + this.sessions, + req.params as ReloadParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), ); case "tool.click": - return handleClick( - this.sessions, + return this.withHoverReassert( req.params as ClickParams, + () => + handleClick( + this.sessions, + req.params as ClickParams, + this.cdp + ? { + cdp: this.cdp, + tabsApi: chromeTabsApi, + signal, + bypassOverlay, + } + : undefined, + ), + { releaseAfter: true }, + ); + case "tool.hover": { + const result = await handleHover( + this.sessions, + req.params as HoverParams, this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal, - bypassOverlay: async (tabId, enabled) => { - try { - await chrome.tabs.sendMessage(tabId, { - type: OVERLAY_AUTOMATION_BYPASS, - enabled, - }); - } catch { - // Content script may be unavailable on restricted pages. - } - }, + bypassOverlay: (tabId, enabled) => + this.setHoverBypass((req.params as HoverParams).session_id, tabId, enabled), + keepOverlayBypassAfterHover: true, } : undefined, ); + return this.rememberHover((req.params as HoverParams).session_id, result); + } case "tool.fill": - return handleFill( - this.sessions, - req.params as FillParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + return this.withHoverReleaseForRequest(req.params as FillParams, () => + handleFill( + this.sessions, + req.params as FillParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), ); case "tool.press": - return handlePress( - this.sessions, - req.params as PressParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + return this.withHoverReleaseForRequest(req.params as PressParams, () => + handlePress( + this.sessions, + req.params as PressParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), ); case "tool.select": - return handleSelect( - this.sessions, - req.params as SelectParams, - this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + return this.withHoverReleaseForRequest(req.params as SelectParams, () => + handleSelect( + this.sessions, + req.params as SelectParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), ); case "tool.evaluate": return handleEvaluate( @@ -471,6 +532,114 @@ export class ToolDispatcher { } satisfies RpcError; } } + + private async setHoverBypass(sessionId: string, tabId: number, enabled: boolean): Promise { + const owner = this.hoverBypassTabs.get(tabId); + if (enabled) { + if (owner === sessionId) return; + if (owner === undefined) await bypassOverlay(tabId, true); + this.hoverBypassTabs.set(tabId, sessionId); + } else { + if (owner !== sessionId) return; + await bypassOverlay(tabId, false); + this.hoverBypassTabs.delete(tabId); + } + } + + private rememberHover(sessionId: string, result: HoverResult | RpcError): HoverResult | RpcError { + if (!isRpcError(result)) { + this.hoverLatches.set(result.tab_id, { + sessionId, + tabId: result.tab_id, + x: result.x, + y: result.y, + }); + } + return result; + } + + private hasHoverLatchForScope(scope: HoverLatchScope): boolean { + return this.hoverLatchesForRequest(scope).length > 0; + } + + private async withHoverReassert( + params: { session_id: string; tab_id?: number }, + work: () => Promise, + options: { releaseAfter?: boolean } = {}, + ): Promise { + const scope = await this.resolveHoverLatchScope(params); + await this.reassertHover(scope); + try { + return await work(); + } finally { + if (options.releaseAfter) { + await this.releaseHoverLatch(scope.session_id, scope.tab_id); + } + } + } + + private async withHoverReleaseForRequest( + params: { session_id: string; tab_id?: number }, + work: () => Promise, + ): Promise { + const scope = await this.resolveHoverLatchScope(params); + await this.releaseHoverLatch(scope.session_id, scope.tab_id); + return work(); + } + + private async resolveHoverLatchScope(params: { + session_id: string; + tab_id?: number; + }): Promise { + if (params.tab_id !== undefined) return params; + const ctx = lookupSession(this.sessions, params, "hover latch"); + if (isRpcError(ctx)) return params; + const target = await resolveTargetTab(this.sessions, ctx, undefined, chromeTabsApi); + if (isRpcError(target)) return params; + return { session_id: params.session_id, tab_id: target.tabId }; + } + + private hoverLatchesForRequest(params: { session_id: string; tab_id?: number }): HoverLatch[] { + return [...this.hoverLatches.values()].filter((latch) => { + if (latch.sessionId !== params.session_id) return false; + return params.tab_id === undefined || latch.tabId === params.tab_id; + }); + } + + private async reassertHover(params: { session_id: string; tab_id?: number }): Promise { + if (!this.cdp) return; + await Promise.all( + this.hoverLatchesForRequest(params).map((latch) => + this.cdp!.send(latch.tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: latch.x, + y: latch.y, + }).catch((err) => { + console.debug("[bsk dispatcher] hover reassert failed", err); + this.hoverLatches.delete(latch.tabId); + }), + ), + ); + } + + private async releaseHoverLatch(sessionId?: string, tabId?: number): Promise { + const matchesScope = (entrySessionId: string, entryTabId: number): boolean => { + if (sessionId !== undefined && entrySessionId !== sessionId) return false; + return tabId === undefined || entryTabId === tabId; + }; + const tabs = new Set(); + for (const [bypassTabId, bypassSessionId] of this.hoverBypassTabs) { + if (!matchesScope(bypassSessionId, bypassTabId)) continue; + tabs.add(bypassTabId); + this.hoverBypassTabs.delete(bypassTabId); + } + for (const latch of this.hoverLatches.values()) { + if (!matchesScope(latch.sessionId, latch.tabId)) continue; + tabs.add(latch.tabId); + this.hoverLatches.delete(latch.tabId); + } + await Promise.all([...tabs].map((tabId) => bypassOverlay(tabId, false))); + } } function isRpcError(v: unknown): v is RpcError { @@ -497,6 +666,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.navigate_forward": case "tool.reload": case "tool.click": + case "tool.hover": case "tool.fill": case "tool.press": case "tool.select": @@ -512,6 +682,17 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { } } +async function bypassOverlay(tabId: number, enabled: boolean): Promise { + try { + await chrome.tabs.sendMessage(tabId, { + type: OVERLAY_AUTOMATION_BYPASS, + enabled, + }); + } catch { + // Content script may be unavailable on restricted pages. + } +} + /** * Resolves never; rejects with `AbortLikeError` as soon as the signal * fires (or immediately if it is already aborted). Used by the diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 87decea..ee66dac 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -17,6 +17,8 @@ import type { ClickResult, FillParams, FillResult, + HoverParams, + HoverResult, KeyModifier, MouseButton, PressParams, @@ -53,9 +55,12 @@ export interface InteractionDeps { defaultTimeoutMs?: number; /** Temporarily disable overlay click blocker during CDP automation. */ bypassOverlay?: (tabId: number, enabled: boolean) => Promise; + /** Keep hover hit-testing active for the caller's next observation/action. */ + keepOverlayBypassAfterHover?: boolean; } const DEFAULT_TIMEOUT_MS = 30_000; +const DEFAULT_HOVER_SETTLE_MS = 200; let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null; function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } { @@ -100,6 +105,27 @@ function throwIfAborted(signal: AbortSignal | undefined): RpcError | null { return null; } +function isAbortLikeError(err: unknown): boolean { + return err instanceof DOMException && err.name === "AbortError"; +} + +async function wait(ms: number, signal?: AbortSignal): Promise { + if (signal?.aborted) throw new DOMException("aborted", "AbortError"); + await new Promise((resolve, reject) => { + const cleanup = () => signal?.removeEventListener("abort", onAbort); + const timer = setTimeout(() => { + cleanup(); + resolve(); + }, ms); + const onAbort = () => { + clearTimeout(timer); + cleanup(); + reject(new DOMException("aborted", "AbortError")); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + /** * Resolve `{ref?, selector?}` into a `backendNodeId`. Returns an * `RpcError` if the caller supplied neither (or both), or if neither @@ -286,7 +312,104 @@ export async function handleClick( message: err instanceof Error ? err.message : String(err), }; } finally { - if (automationBypassEnabled && deps.bypassOverlay) { + if (automationBypassEnabled && deps.bypassOverlay && !deps.keepOverlayBypassAfterHover) { + try { + await deps.bypassOverlay(target.tabId, false); + } catch (err) { + console.debug("[bsk interaction] overlay bypass disable failed", err); + } + } + } + + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: node.usedRef, + used_selector: node.usedSelector, + x: centre.x, + y: centre.y, + }); +} + +// --------------------------------------------------------------------------- +// tool.hover +// --------------------------------------------------------------------------- + +export async function handleHover( + manager: SessionManager, + params: HoverParams, + deps: InteractionDeps = getDefaultDeps(), +): Promise { + const ctxOrErr = lookupSession(manager, params, "hover"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + const aborted = throwIfAborted(deps.signal); + if (aborted) return aborted; + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "hover"); + if (denied) return denied; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + + const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover"); + if (isRpcError(node)) return node; + + try { + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId); + if (scrollErr) return scrollErr; + } catch (err) { + return { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + }; + } + + const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId); + if (isRpcError(centre)) return centre; + + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "hover aborted" }; + } + + const modifiers = modifiersBitfield(params.modifiers); + const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y); + let automationBypassEnabled = false; + let hoverCompleted = false; + if (overlayBlocking && deps.bypassOverlay) { + try { + await deps.bypassOverlay(target.tabId, true); + automationBypassEnabled = true; + } catch (err) { + console.debug("[bsk interaction] overlay bypass enable failed", err); + } + } + + try { + await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: centre.x, + y: centre.y, + modifiers, + }); + const settleMs = params.settle_ms ?? DEFAULT_HOVER_SETTLE_MS; + if (settleMs > 0) { + await wait(settleMs, deps.signal); + } + hoverCompleted = true; + } catch (err) { + if (isAbortLikeError(err)) { + return { code: "cancelled", message: "hover aborted" }; + } + return { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + }; + } finally { + if ( + automationBypassEnabled && + deps.bypassOverlay && + (!deps.keepOverlayBypassAfterHover || !hoverCompleted) + ) { try { await deps.bypassOverlay(target.tabId, false); } catch (err) { diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 48fe799..bde04f9 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -5,7 +5,9 @@ import { type ActiveScopeBlock, + applyVomInteractionRecovery, type CondSurface, + isVomReferenceNode, renderVom, type VomNode, type VomScene, @@ -908,39 +910,28 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A return blocks; } -function labelForSurfaceMatch(node: VomNode): string { - return ( - cleanAttr(node.name) ?? - cleanAttr(node.text) ?? - cleanAttr(node.attrs?.["aria-label"]) ?? - cleanAttr(node.attrs?.title) ?? - "" - ); -} - function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] { const probes = captured.surfaceProbes ?? []; if (probes.length === 0) return []; const surfaces: CondSurface[] = []; - const nodesWithLabels = nodes - .map((node) => ({ node, key: normalizeProbeKey(labelForSurfaceMatch(node)) })) - .filter((entry) => entry.key.length > 0); + const recoveredNodes = applyVomInteractionRecovery(nodes); + const signals = capturedOnlySignals(captured.nodes); const used = new Set(); for (const probe of probes) { - const probeKey = normalizeProbeKey(probe.triggerLabel); - if (!probeKey || probe.subItems.length === 0) continue; - const match = - nodesWithLabels.find((entry) => entry.key === probeKey && !used.has(entry.node.id)) ?? - nodesWithLabels.find( - (entry) => - !used.has(entry.node.id) && - (probeKey.startsWith(entry.key) || entry.key.startsWith(probeKey)), - ); + if (probe.subItems.length === 0 || used.has(probe.triggerBackendNodeId)) continue; + const match = findSurfaceTriggerNode( + probe.triggerBackendNodeId, + recoveredNodes, + signals, + probe.triggerPoint, + ); if (!match) continue; - used.add(match.node.id); + if (used.has(match.id)) continue; + used.add(probe.triggerBackendNodeId); + used.add(match.id); surfaces.push({ - triggerId: match.node.id, + triggerId: match.id, triggerAction: probe.triggerAction, subItems: probe.subItems, }); @@ -948,6 +939,77 @@ function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel) return surfaces; } +function findSurfaceTriggerNode( + triggerBackendNodeId: number, + nodes: VomNode[], + signals: VomNodeDomSignals, + triggerPoint?: { x: number; y: number }, +): VomNode | undefined { + const renderedById = new Map(nodes.map((node) => [node.id, node])); + const exact = renderedById.get(triggerBackendNodeId); + if (exact && isSurfaceAttachableNode(exact)) return exact; + + const domDescendant = nodes.find( + (node) => + isSurfaceAttachableNode(node) && + (node.domParentId === triggerBackendNodeId || + node.domAncestorIds?.includes(triggerBackendNodeId) === true), + ); + if (domDescendant) return domDescendant; + + const queue = [...(signals.childrenByParentId.get(triggerBackendNodeId) ?? [])]; + while (queue.length > 0) { + const child = queue.shift() as CapturedNode; + const rendered = renderedById.get(child.backendNodeId); + if (rendered && isSurfaceAttachableNode(rendered)) return rendered; + queue.push(...(signals.childrenByParentId.get(child.backendNodeId) ?? [])); + } + + let current = signals.capturedByBackendId.get(triggerBackendNodeId); + while (current?.parentBackendNodeId !== null && current?.parentBackendNodeId !== undefined) { + const parent = renderedById.get(current.parentBackendNodeId); + if (parent && isSurfaceAttachableNode(parent)) return parent; + current = signals.capturedByBackendId.get(current.parentBackendNodeId); + } + + if (triggerPoint) { + return findSurfaceNodeByPoint(triggerPoint, nodes, signals); + } + + return undefined; +} + +function isSurfaceAttachableNode(node: VomNode): boolean { + return isVomReferenceNode(node); +} + +function findSurfaceNodeByPoint( + point: { x: number; y: number }, + nodes: VomNode[], + signals: VomNodeDomSignals, +): VomNode | undefined { + let best: { node: VomNode; score: number } | undefined; + for (const node of nodes) { + if (!isSurfaceAttachableNode(node)) continue; + const rect = node.rect ?? signals.capturedByBackendId.get(node.id)?.rect; + if (!rect) continue; + const contains = + point.x >= rect.x && + point.x <= rect.x + rect.w && + point.y >= rect.y && + point.y <= rect.y + rect.h; + const centerX = rect.x + rect.w / 2; + const centerY = rect.y + rect.h / 2; + const distance = Math.hypot(point.x - centerX, point.y - centerY); + if (!contains && distance > 40) continue; + const score = contains ? distance : distance + 1_000; + if (!best || score < best.score) { + best = { node, score }; + } + } + return best?.node; +} + function axNodeInOverlaySubtree( axNode: CdpAxNode, axById: Map, @@ -1072,6 +1134,8 @@ export interface SnapshotDeps { get(tabId: number): Promise; query(q: chrome.tabs.QueryInfo): Promise; }; + conditionalSurfaceProbe?: boolean; + hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise; } let defaultDeps: SnapshotDeps | null = null; @@ -1217,9 +1281,13 @@ async function captureForVom( cdp: CdpRunner, tabId: number, conditionalSurfaceProbe: boolean, + hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise, ): Promise { try { - return await captureViewModel(cdp, tabId, { conditionalSurfaceProbe }); + return await captureViewModel(cdp, tabId, { + conditionalSurfaceProbe, + hoverProbeBypassOverlay, + }); } catch { return fallbackCapturedViewModel(cdp, tabId); } @@ -1258,7 +1326,14 @@ async function handleVomObservation( {}, ); const axNodes = result.nodes ?? []; - const captured = await captureForVom(deps.cdp, target.tabId, conditionalSurfaceProbe); + const effectiveConditionalSurfaceProbe = + deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe; + const captured = await captureForVom( + deps.cdp, + target.tabId, + effectiveConditionalSurfaceProbe, + deps.hoverProbeBypassOverlay, + ); const scene = buildVomScene(axNodes, captured, { pageUrl: target.url }); const rendered = renderVom(scene, { maxDepth: params.max_depth, @@ -1275,6 +1350,19 @@ async function handleVomObservation( ref_count: rendered.refs.length, tab_id: target.tabId, truncated: rendered.truncated, + ...(toolName === "observe" && (params as ObserveParams).debug_surfaces + ? { + debug: { + surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({ + trigger_backend_node_id: probe.triggerBackendNodeId, + ...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}), + trigger_action: probe.triggerAction, + sub_items: probe.subItems, + ...(probe.confidence ? { confidence: probe.confidence } : {}), + })), + }, + } + : {}), }); } catch (err) { return { diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index 2fe1f2e..bc4cd2a 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -51,6 +51,156 @@ function fakeSnapshotReply() { }; } +function hoverTriggerSnapshotReply() { + const S = [ + "html", + "body", + "button", + "class", + "user-avatar", + "position", + "static", + "pointer-events", + "auto", + "cursor", + "pointer", + ]; + const i = (s: string) => S.indexOf(s); + return { + strings: S, + documents: [ + { + nodes: { + parentIndex: [-1, 0, 1], + nodeType: [1, 1, 1], + nodeName: [i("html"), i("body"), i("button")], + backendNodeId: [10, 11, 12], + attributes: [[], [], [i("class"), i("user-avatar")]], + }, + layout: { + nodeIndex: [1, 2], + styles: [ + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + ], + bounds: [ + [0, 0, 1000, 800], + [940, 16, 32, 32], + ], + paintOrders: [0, 1], + }, + }, + ], + }; +} + +function twoHoverTriggerSnapshotReply() { + const S = [ + "html", + "body", + "button", + "class", + "user-avatar", + "secondary-dropdown-trigger", + "position", + "static", + "pointer-events", + "auto", + "cursor", + "pointer", + ]; + const i = (s: string) => S.indexOf(s); + return { + strings: S, + documents: [ + { + nodes: { + parentIndex: [-1, 0, 1, 1], + nodeType: [1, 1, 1, 1], + nodeName: [i("html"), i("body"), i("button"), i("button")], + backendNodeId: [10, 11, 12, 13], + attributes: [ + [], + [], + [i("class"), i("user-avatar")], + [i("class"), i("secondary-dropdown-trigger")], + ], + }, + layout: { + nodeIndex: [1, 2, 3], + styles: [ + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + ], + bounds: [ + [0, 0, 1000, 800], + [940, 16, 32, 32], + [100, 120, 32, 32], + ], + paintOrders: [0, 1, 2], + }, + }, + ], + }; +} + +function nestedHoverTriggerSnapshotReply() { + const S = [ + "html", + "body", + "div", + "img", + "class", + "tg-avatar", + "tg-avatar__inner", + "tg-avatar__image", + "position", + "static", + "pointer-events", + "auto", + "cursor", + "pointer", + ]; + const i = (s: string) => S.indexOf(s); + return { + strings: S, + documents: [ + { + nodes: { + parentIndex: [-1, 0, 1, 2, 3], + nodeType: [1, 1, 1, 1, 1], + nodeName: [i("html"), i("body"), i("div"), i("div"), i("img")], + backendNodeId: [10, 11, 12, 13, 14], + attributes: [ + [], + [], + [i("class"), i("tg-avatar")], + [i("class"), i("tg-avatar__inner")], + [i("class"), i("tg-avatar__image")], + ], + }, + layout: { + nodeIndex: [1, 2, 3, 4], + styles: [ + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + ], + bounds: [ + [0, 0, 1000, 800], + [940, 16, 32, 32], + [942, 18, 28, 28], + [944, 20, 24, 24], + ], + paintOrders: [0, 1, 2, 3], + }, + }, + ], + }; +} + function makeCdp(snapshot: unknown) { return { send: vi.fn(async (_tab: number, method: string) => { @@ -126,11 +276,11 @@ describe("captureViewModel", () => { }); it("runs hover surface probes when explicitly enabled", async () => { - let runtimeCalls = 0; + let hoverStateCalls = 0; const cdp = { send: vi.fn(async (_tabId: number, method: string, params?: object) => { if (method === "DOMSnapshot.enable") return {}; - if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply(); + if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply(); if (method === "Page.getLayoutMetrics") { return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, @@ -141,22 +291,23 @@ describe("captureViewModel", () => { if (expression.includes("input,textarea,select")) { return { result: { value: { controls: [], childFrames: [] } } }; } - runtimeCalls += 1; - return runtimeCalls === 1 - ? { - result: { - value: [ - { - triggerSel: ".menu", - affectedSub: " .items", - label: "Products", - x: 50, - y: 20, - }, - ], - }, - } - : { result: { value: ["Shoes", "Bags"] } }; + if (expression.includes("document.styleSheets")) { + return { result: { value: [] } }; + } + if (expression.includes("querySelectorAll(selectors)")) { + hoverStateCalls += 1; + return hoverStateCalls === 1 + ? { result: { value: [] } } + : { + result: { + value: [ + { text: "My profile", role: "", tag: "a", x: 900, y: 60 }, + { text: "Sign out", role: "", tag: "a", x: 900, y: 90 }, + ], + }, + }; + } + throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`); } if (method === "Input.dispatchMouseEvent") return {}; throw new Error(`unexpected ${method}`); @@ -166,12 +317,18 @@ describe("captureViewModel", () => { const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true }); expect(result.surfaceProbes).toEqual([ - { triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] }, + { + triggerBackendNodeId: 12, + triggerPoint: { x: 956, y: 32 }, + triggerAction: "hover", + subItems: ["My profile", "Sign out"], + confidence: "high", + }, ]); expect(cdp.send).toHaveBeenCalledWith( 4, "Input.dispatchMouseEvent", - expect.objectContaining({ type: "mouseMoved", x: 50, y: 20 }), + expect.objectContaining({ type: "mouseMoved", x: 956, y: 32 }), ); expect(cdp.send).toHaveBeenCalledWith( 4, @@ -180,6 +337,91 @@ describe("captureViewModel", () => { ); }); + it("uses a fresh baseline for each hover candidate", async () => { + let hoverStateCalls = 0; + const cdp = { + send: vi.fn(async (_tabId: number, method: string, params?: object) => { + if (method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply(); + if (method === "Page.getLayoutMetrics") { + return { + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + }; + } + if (method === "Runtime.evaluate") { + const expression = (params as { expression?: string } | undefined)?.expression ?? ""; + if (expression.includes("input,textarea,select")) { + return { result: { value: { controls: [], childFrames: [] } } }; + } + if (expression.includes("document.styleSheets")) { + return { result: { value: [] } }; + } + if (expression.includes("querySelectorAll(selectors)")) { + hoverStateCalls += 1; + const visibleMenu = [ + { text: "My profile", role: "", tag: "a", x: 900, y: 60 }, + { text: "Sign out", role: "", tag: "a", x: 900, y: 90 }, + ]; + return hoverStateCalls === 1 + ? { result: { value: [] } } + : { result: { value: visibleMenu } }; + } + throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`); + } + if (method === "Input.dispatchMouseEvent") return {}; + throw new Error(`unexpected ${method}`); + }) as unknown as (tabId: number, method: string, params?: object) => Promise, + }; + + const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true }); + + expect(result.surfaceProbes).toEqual([ + expect.objectContaining({ + triggerBackendNodeId: 12, + subItems: ["My profile", "Sign out"], + }), + ]); + expect(hoverStateCalls).toBeGreaterThanOrEqual(4); + }); + + it("deduplicates nested hover candidates for one visual trigger", async () => { + let hoverMoves = 0; + const cdp = { + send: vi.fn(async (_tabId: number, method: string, params?: object) => { + if (method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply(); + if (method === "Page.getLayoutMetrics") { + return { + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + }; + } + if (method === "Runtime.evaluate") { + const expression = (params as { expression?: string } | undefined)?.expression ?? ""; + if (expression.includes("input,textarea,select")) { + return { result: { value: { controls: [], childFrames: [] } } }; + } + if (expression.includes("document.styleSheets")) { + return { result: { value: [] } }; + } + if (expression.includes("querySelectorAll(selectors)")) { + return { result: { value: [] } }; + } + throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`); + } + if (method === "Input.dispatchMouseEvent") { + const point = params as { x?: number; y?: number }; + if (point.x !== -10 && point.x !== 0) hoverMoves += 1; + return {}; + } + throw new Error(`unexpected ${method}`); + }) as unknown as (tabId: number, method: string, params?: object) => Promise, + }; + + await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true }); + + expect(hoverMoves).toBe(1); + }); + it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => { // DOMSnapshot inlines an open shadow root's content as descendants of the // host. The agent's WXT overlay host carries a fixed full-viewport diff --git a/apps/extension/src/tools/vom/capture.ts b/apps/extension/src/tools/vom/capture.ts index d8358f5..0f56577 100644 --- a/apps/extension/src/tools/vom/capture.ts +++ b/apps/extension/src/tools/vom/capture.ts @@ -5,6 +5,7 @@ // `styles` columns are [position, pointer-events, cursor] in that order. import type { Rect, Viewport } from "@browser-skill/vom"; +import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; import type { CdpRunner } from "../shared"; @@ -45,9 +46,11 @@ export interface CapturedNode { export type CapturedIframeNodes = Map; export interface CapturedSurfaceProbe { - triggerLabel: string; + triggerBackendNodeId: number; + triggerPoint?: { x: number; y: number }; triggerAction: "hover" | "focus" | string; subItems: string[]; + confidence?: "high" | "medium" | "low"; } export interface CapturedViewModel { @@ -61,6 +64,7 @@ export interface CapturedViewModel { export interface CaptureViewModelOptions { conditionalSurfaceProbe?: boolean; + hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise; } /** Sparse array format Chrome uses for infrequently-set per-node fields. */ @@ -225,11 +229,12 @@ interface RuntimeEvaluateReply { } interface HoverCandidate { - triggerSel: string; - affectedSub: string; - label: string; + backendNodeId: number; + label?: string; x: number; y: number; + score: number; + reasons: string[]; } interface CdpDomNode { @@ -322,17 +327,19 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set(reply: RuntimeEvaluateReply): T | undefined { return reply.result?.value as T | undefined; } -function hoverCssScanExpression(): string { +function hoverCssTriggerScanExpression(): string { return `(() => { const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"]; - const pairs = []; + const centres = []; const seenRules = new Set(); for (const sheet of Array.from(document.styleSheets)) { let rules; @@ -348,90 +355,73 @@ function hoverCssScanExpression(): string { const hoverIndex = part.indexOf(":hover"); if (hoverIndex < 0) continue; const triggerSel = part.slice(0, hoverIndex).trim(); - const affectedSub = part.slice(hoverIndex + 6).trim(); if (!triggerSel) continue; - const key = triggerSel + "||" + affectedSub; - if (seenRules.has(key)) continue; - seenRules.add(key); - pairs.push({ triggerSel, affectedSub }); + if (seenRules.has(triggerSel)) continue; + seenRules.add(triggerSel); + let elements; + try { elements = Array.from(document.querySelectorAll(triggerSel)); } catch { continue; } + for (const el of elements) { + if (!(el instanceof HTMLElement)) continue; + const rect = el.getBoundingClientRect(); + const style = getComputedStyle(el); + if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue; + centres.push({ x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }); + if (centres.length >= 24) return centres; + } } } } - - const candidates = []; - const seenLabels = new Set(); - const isUnsafeTrigger = (el) => { - if (!(el instanceof HTMLElement)) return true; - const tag = el.tagName.toLowerCase(); - if (["input", "textarea", "select", "option"].includes(tag)) return true; - if (el.isContentEditable) return true; - if (el.hasAttribute("disabled") || el.hasAttribute("inert")) return true; - if ((el.getAttribute("aria-disabled") || "").toLowerCase() === "true") return true; - return false; - }; - for (const pair of pairs) { - let elements; - try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; } - for (const el of elements) { - if (!(el instanceof Element)) continue; - if (isUnsafeTrigger(el)) continue; - const rect = el.getBoundingClientRect(); - const style = getComputedStyle(el); - if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue; - const label = (el.textContent || "").replace(/\\s+/g, " ").trim(); - if (!label || seenLabels.has(label)) continue; - seenLabels.add(label); - candidates.push({ - triggerSel: pair.triggerSel, - affectedSub: pair.affectedSub, - label, - x: rect.left + rect.width / 2, - y: rect.top + rect.height / 2, - }); - if (candidates.length >= ${MAX_HOVER_TRIGGERS}) return candidates; - } - } - return candidates; + return centres; })()`; } -function hoverCollectExpression(candidate: HoverCandidate): string { +interface HoverRuntimeItem { + text: string; + role: string; + tag: string; + x: number; + y: number; +} + +function hoverStateExpression(): string { return `(() => { - const triggerSel = ${JSON.stringify(candidate.triggerSel)}; - const rawAffectedSub = ${JSON.stringify(candidate.affectedSub)}; - const x = ${JSON.stringify(candidate.x)}; - const y = ${JSON.stringify(candidate.y)}; - const hit = document.elementFromPoint(x, y); - const trigger = hit instanceof Element ? hit.closest(triggerSel) : null; - if (!trigger) return []; - - const affectedSub = rawAffectedSub.replace(/^[>+~]\\s*/, "").trim(); - let targets = []; - if (!affectedSub) { - targets = [trigger]; - } else { - try { targets = Array.from(trigger.querySelectorAll(affectedSub)); } catch { targets = []; } - } const items = []; const seen = new Set(); - const push = (value) => { - const text = String(value || "").replace(/\\s+/g, " ").trim(); + const selectors = [ + "a", + "button", + "[role='menuitem']", + "[role='menuitemcheckbox']", + "[role='menuitemradio']", + "[role='option']", + "[role='tab']", + "[role='link']", + "[role='button']" + ].join(","); + const push = (el) => { + if (!(el instanceof HTMLElement)) return; + const style = getComputedStyle(el); + const rect = el.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) return; + if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") return; + const text = String( + el.getAttribute("aria-label") || + el.getAttribute("title") || + el.textContent || + "" + ).replace(/\\s+/g, " ").trim(); if (!text || seen.has(text)) return; seen.add(text); - items.push(text); + items.push({ + text, + role: (el.getAttribute("role") || "").toLowerCase(), + tag: el.tagName.toLowerCase(), + x: rect.left, + y: rect.top, + }); }; - for (const target of targets) { - if (!(target instanceof Element)) continue; - const style = getComputedStyle(target); - if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue; - const clickable = target.querySelectorAll('a, button, [role="menuitem"], [role="option"]'); - if (clickable.length > 0) { - for (const child of Array.from(clickable)) push(child.textContent); - } else { - push(target.textContent); - } - } - return items.slice(0, 12); + for (const el of Array.from(document.querySelectorAll(selectors))) push(el); + return items.slice(0, 400); })()`; } @@ -453,45 +443,217 @@ async function clearHover(cdp: CdpRunner, tabId: number): Promise { ); } -async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise { +function capturedText(node: CapturedNode): string | undefined { + const value = + node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? ""; + const clean = value.replace(/\s+/g, " ").trim(); + return clean || undefined; +} + +function hasGraphicDescendant( + node: CapturedNode, + childrenByParentId: Map, + depth = 0, +): boolean { + if (depth > 3) return false; + for (const child of childrenByParentId.get(node.backendNodeId) ?? []) { + const tag = child.tag.toLowerCase(); + if (["img", "svg", "use", "path", "i"].includes(tag)) return true; + if (hasGraphicDescendant(child, childrenByParentId, depth + 1)) return true; + } + return false; +} + +function roleOf(node: CapturedNode): string { + return (node.attrs.role ?? "").toLowerCase(); +} + +function scoreHoverCandidate( + node: CapturedNode, + childrenByParentId: Map, + cssHoverPoints: Array<{ x: number; y: number }>, +): HoverCandidate | null { + const rect = node.rect; + if (!rect) return null; + const label = capturedText(node); + const cssHoverMatch = cssHoverPoints.some( + (point) => + point.x >= rect.x && + point.x <= rect.x + rect.w && + point.y >= rect.y && + point.y <= rect.y + rect.h, + ); + const decision = evaluateHoverTrigger({ + tag: node.tag, + role: roleOf(node), + label, + attrs: node.attrs, + rect, + cursor: node.cursor, + pointerEvents: node.pointerEvents, + hasGraphicDescendant: hasGraphicDescendant(node, childrenByParentId), + cssHoverMatch, + }); + + if (!decision.eligible) return null; + return { + backendNodeId: node.backendNodeId, + label, + x: rect.x + rect.w / 2, + y: rect.y + rect.h / 2, + score: decision.score, + reasons: decision.reasons, + }; +} + +function buildHoverCandidates( + nodes: CapturedNode[], + cssHoverPoints: Array<{ x: number; y: number }>, +): HoverCandidate[] { + const childrenByParentId = new Map(); + const parentByBackendId = new Map(); + for (const node of nodes) { + parentByBackendId.set(node.backendNodeId, node.parentBackendNodeId); + if (node.parentBackendNodeId === null) continue; + const children = childrenByParentId.get(node.parentBackendNodeId) ?? []; + children.push(node); + childrenByParentId.set(node.parentBackendNodeId, children); + } + + const candidates = nodes + .map((node) => scoreHoverCandidate(node, childrenByParentId, cssHoverPoints)) + .filter((candidate): candidate is HoverCandidate => candidate !== null) + .sort((a, b) => b.score - a.score); + + const deduped: HoverCandidate[] = []; + const seen = new Set(); + for (const candidate of candidates) { + if (seen.has(candidate.backendNodeId)) continue; + if (deduped.some((existing) => sameHoverCluster(existing, candidate, parentByBackendId))) { + continue; + } + seen.add(candidate.backendNodeId); + deduped.push(candidate); + if (deduped.length >= MAX_HOVER_TRIGGERS) break; + } + return deduped; +} + +function sameHoverCluster( + a: HoverCandidate, + b: HoverCandidate, + parentByBackendId: Map, +): boolean { + if (Math.hypot(a.x - b.x, a.y - b.y) <= 8) return true; + return ( + isBackendAncestor(a.backendNodeId, b.backendNodeId, parentByBackendId) || + isBackendAncestor(b.backendNodeId, a.backendNodeId, parentByBackendId) + ); +} + +function isBackendAncestor( + ancestorId: number, + nodeId: number, + parentByBackendId: Map, +): boolean { + let parentId = parentByBackendId.get(nodeId); + let guard = 0; + while (parentId !== null && parentId !== undefined && guard < parentByBackendId.size) { + if (parentId === ancestorId) return true; + parentId = parentByBackendId.get(parentId); + guard += 1; + } + return false; +} + +function diffHoverItems(before: HoverRuntimeItem[], after: HoverRuntimeItem[]): string[] { + const beforeKeys = new Set(before.map((item) => item.text.toLowerCase())); + const out: string[] = []; + const seen = new Set(); + for (const item of after) { + const text = item.text.replace(/\s+/g, " ").trim(); + const key = text.toLowerCase(); + if (!text || beforeKeys.has(key) || seen.has(key)) continue; + seen.add(key); + out.push(text); + if (out.length >= MAX_HOVER_SUB_ITEMS) break; + } + return out; +} + +function confidenceForHover( + candidate: HoverCandidate, + subItems: string[], +): "high" | "medium" | "low" { + if (subItems.length >= 2 && candidate.score >= 80) return "high"; + if (subItems.length >= 2 || candidate.score >= 80) return "medium"; + return "low"; +} + +async function probeHoverSurfaces( + cdp: CdpRunner, + tabId: number, + nodes: CapturedNode[], + options: CaptureViewModelOptions, +): Promise { const started = Date.now(); try { - const scan = await cdp.send(tabId, "Runtime.evaluate", { - expression: hoverCssScanExpression(), + const cssScan = await cdp.send(tabId, "Runtime.evaluate", { + expression: hoverCssTriggerScanExpression(), returnByValue: true, }); - const candidates = runtimeValue(scan) ?? []; + const cssHoverPoints = runtimeValue>(cssScan) ?? []; + const candidates = buildHoverCandidates(nodes, cssHoverPoints); + if (candidates.length === 0) return []; + const results: CapturedSurfaceProbe[] = []; - const seen = new Set(); - for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) { - if (Date.now() - started > MAX_HOVER_PROBE_MS) break; - if (!candidate.label || seen.has(candidate.label)) continue; - try { - await cdp.send(tabId, "Input.dispatchMouseEvent", { - type: "mouseMoved", - x: candidate.x, - y: candidate.y, - }); - await wait(HOVER_SETTLE_MS); - const collected = await cdp.send(tabId, "Runtime.evaluate", { - expression: hoverCollectExpression(candidate), - returnByValue: true, - }); - const subItems = (runtimeValue(collected) ?? []) - .map((item) => item.replace(/\s+/g, " ").trim()) - .filter(Boolean); - if (subItems.length === 0) continue; - seen.add(candidate.label); - results.push({ - triggerLabel: candidate.label, - triggerAction: "hover", - subItems, - }); - } catch { - continue; - } finally { - await clearHover(cdp, tabId); + const seen = new Set(); + await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined); + try { + for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) { + if (Date.now() - started > MAX_HOVER_PROBE_MS) break; + if (results.length >= MAX_HOVER_SURFACES) break; + if (seen.has(candidate.backendNodeId)) continue; + try { + await clearHover(cdp, tabId); + await wait(HOVER_SETTLE_MS); + const baselineReply = await cdp.send(tabId, "Runtime.evaluate", { + expression: hoverStateExpression(), + returnByValue: true, + }); + const baselineItems = runtimeValue(baselineReply) ?? []; + + await cdp.send(tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: candidate.x, + y: candidate.y, + }); + await wait(HOVER_SETTLE_MS); + const collected = await cdp.send(tabId, "Runtime.evaluate", { + expression: hoverStateExpression(), + returnByValue: true, + }); + const subItems = diffHoverItems( + baselineItems, + runtimeValue(collected) ?? [], + ); + if (subItems.length === 0) continue; + seen.add(candidate.backendNodeId); + results.push({ + triggerBackendNodeId: candidate.backendNodeId, + triggerPoint: { x: candidate.x, y: candidate.y }, + triggerAction: "hover", + subItems, + confidence: confidenceForHover(candidate, subItems), + }); + } catch { + continue; + } finally { + await clearHover(cdp, tabId); + } } + } finally { + await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined); } return results; } catch (err) { @@ -797,7 +959,9 @@ export async function captureViewModel( await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]); - const surfaceProbes = options.conditionalSurfaceProbe ? await probeHoverSurfaces(cdp, tabId) : []; + const surfaceProbes = options.conditionalSurfaceProbe + ? await probeHoverSurfaces(cdp, tabId, nodes, options) + : []; return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds }; } diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index a335747..4533d7d 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -314,8 +314,21 @@ export interface SnapshotResult { dialogs?: JavaScriptDialogInfo[]; } -export type ObserveParams = SnapshotParams; -export type ObserveResult = SnapshotResult; +export interface ObserveParams extends SnapshotParams { + debug_surfaces?: boolean; +} + +export interface ObserveResult extends SnapshotResult { + debug?: { + surface_probes?: Array<{ + trigger_backend_node_id: number; + trigger_point?: { x: number; y: number }; + trigger_action: string; + sub_items: string[]; + confidence?: string; + }>; + }; +} export interface GetHtmlParams { session_id: string; @@ -410,6 +423,25 @@ export interface ClickResult { dialogs?: JavaScriptDialogInfo[]; } +export interface HoverParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + modifiers?: KeyModifier[]; + settle_ms?: number; + timeout_ms?: number; +} + +export interface HoverResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + x: number; + y: number; + dialogs?: JavaScriptDialogInfo[]; +} + export interface FillParams { session_id: string; value: string; @@ -668,6 +700,11 @@ export type DraftTraceStep = navigated_to?: string; page_url?: string; } + | { + op: "hover"; + target: TargetDescriptor; + page_url?: string; + } | { op: "fill"; target: TargetDescriptor; @@ -701,6 +738,7 @@ export type DraftTraceStep = export type Step = | ({ op: "navigate" } & StepCommon & { to: string }) | ({ op: "click" } & StepCommon & { target: TargetDescriptor }) + | ({ op: "hover" } & StepCommon & { target: TargetDescriptor }) | ({ op: "fill" } & StepCommon & { target: TargetDescriptor; value: string; diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 3c36961..9a1b6a8 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -67,22 +67,25 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe ``` bsk navigate --session -bsk snapshot --session → aria tree with @e1, @e2, … refs -bsk observe --session → semantic VOM view; may reveal hover/focus surfaces -bsk click @e3 --session → or bsk fill, bsk select, bsk press -bsk snapshot --session → again after navigation / DOM change +bsk observe --session → primary semantic VOM view; reveals hover/focus surfaces +bsk snapshot --session → static aria tree fallback when VOM is insufficient +bsk hover @e3 --session → reveal hover-triggered menus before re-observing/clicking +bsk click @e4 --session → or bsk fill, bsk select, bsk press +bsk observe --session → again after navigation / DOM change ``` **Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page. Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`). +When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover --session `, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action. + ## Observation priority -Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question: +Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question: -1. `bsk snapshot` — strict static page understanding and interaction planning -2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery +1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery +2. `bsk snapshot` — strict static accessibility tree fallback 3. `bsk get-html` — when hidden DOM, metadata, or markup details are required 4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab. @@ -186,6 +189,7 @@ bsk emulate --session --off | Command | Summary | |---------|---------| | `bsk click ` | Click element (`--button`, `--click-count`, `--modifiers`) | +| `bsk hover ` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) | | `bsk fill --value ` | Clear and type into input | | `bsk select --value ` | Set `` option(s) by `value` (repeat `--value` for multi-select) | | `bsk press ` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |