diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af9098518..df7409155 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -207,7 +207,7 @@ jobs: working-directory: apps/extension env: BSK_CLICK_CHROME: google-chrome - run: pnpm exec vitest run src/tools/__tests__/click.browser.test.ts src/tools/__tests__/observation-layers.browser.test.ts src/debug/__tests__/*.browser.test.ts + run: pnpm exec vitest run src/tools/__tests__/*click*.browser.test.ts src/tools/__tests__/observation-layers.browser.test.ts src/debug/__tests__/*.browser.test.ts node-scripts: name: Node script tests diff --git a/apps/extension/src/tools/__tests__/dom-multiclick.browser.test.ts b/apps/extension/src/tools/__tests__/dom-multiclick.browser.test.ts new file mode 100644 index 000000000..6822631aa --- /dev/null +++ b/apps/extension/src/tools/__tests__/dom-multiclick.browser.test.ts @@ -0,0 +1,131 @@ +// @vitest-environment node +// Opt in with BSK_CLICK_CHROME; the harness owns an isolated browser/profile. +import { describe, expect, it } from "vitest"; +import { type CdpDebuggerApi, ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import { SessionManager } from "@/session-manager/manager"; +import { handleClick } from "../interaction"; + +type Send = >( + method: string, + params?: object, + sessionId?: string, +) => Promise; + +async function browser( + run: (send: Send, cdp: ChromiumCdp, sessionId: () => string) => Promise, +) { + const { withChrome } = await import( + new URL( + "../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs", + import.meta.url, + ).href + ); + await withChrome( + { + executable: process.env.BSK_CLICK_CHROME, + deviceScale: 1, + zoom: 1, + startupTimeout: 30_000, + }, + async (send: Send) => { + const { targetId } = await send<{ targetId: string }>("Target.createTarget", { + url: "about:blank", + }); + let activeSession = ""; + const api: CdpDebuggerApi = { + attach: async () => { + const reply = await send<{ sessionId: string }>("Target.attachToTarget", { + targetId, + flatten: true, + }); + activeSession = reply.sessionId; + }, + detach: async () => { + await send("Target.detachFromTarget", { sessionId: activeSession }); + activeSession = ""; + }, + sendCommand: async (_target, method, params) => send(method, params, activeSession), + onEvent: { addListener() {}, removeListener() {} } as unknown as CdpDebuggerApi["onEvent"], + onDetach: { + addListener() {}, + removeListener() {}, + } as unknown as CdpDebuggerApi["onDetach"], + }; + const cdp = new ChromiumCdp(api); + try { + await run(send, cdp, () => activeSession); + } finally { + await cdp.detach(7); + } + }, + ); +} + +function manager() { + return new SessionManager({ + agentWindow: { + create: async () => ({ windowId: 100, initialTabIds: [7] }), + remove: async () => {}, + ensureActiveTab: async () => 7, + }, + }); +} +const tab = { id: 7, windowId: 100, active: true, url: "about:blank" } as chrome.tabs.Tab; +const tabsApi = { get: async () => tab, query: async () => [tab] }; + +describe.skipIf(!process.env.BSK_CLICK_CHROME)("DOM native click sequences", () => { + it("DOM double click delivers both click events, as a native double click does", async () => { + await browser(async (_send, cdp) => { + const sessions = manager(); + const ctx = await sessions.start("double-click-audit"); + await cdp.send(7, "Runtime.evaluate", { + expression: ` + document.body.innerHTML = ''; + window.auditEvents = []; + for (const type of ['mousedown', 'mouseup', 'click', 'dblclick']) { + document.querySelector('#target').addEventListener(type, event => { + auditEvents.push({type: event.type, detail: event.detail, trusted: event.isTrusted}); + }); + } + `, + }); + const result = await handleClick( + sessions, + { + session_id: ctx.sessionId, + tab_id: 7, + selector: "#target", + click_count: 2, + }, + { cdp, tabsApi }, + ); + expect(result).not.toHaveProperty("code"); + const actual = await cdp.send<{ result: { value: unknown[] } }>(7, "Runtime.evaluate", { + expression: "auditEvents", + returnByValue: true, + }); + // Positive control: a full CDP double-click sequence on the same element. + await cdp.send(7, "Runtime.evaluate", { expression: "auditEvents = []" }); + for (const clickCount of [1, 2]) { + for (const type of ["mousePressed", "mouseReleased"]) { + await cdp.send(7, "Input.dispatchMouseEvent", { + type, + x: 50, + y: 50, + button: "left", + clickCount, + }); + } + } + const control = await cdp.send<{ result: { value: unknown[] } }>(7, "Runtime.evaluate", { + expression: "auditEvents", + returnByValue: true, + }); + console.log( + "DOUBLE_CLICK_PROOF", + JSON.stringify({ actual: actual.result.value, control: control.result.value }), + ); + expect(actual.result.value).toEqual(control.result.value); + }); + }, 40_000); +}); diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index 74e6ebbe6..5d04d2023 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -148,7 +148,7 @@ describe("handleClick", () => { expect(fake.cdp.send).not.toHaveBeenCalled(); }); - it("clicks by ref, computes the quad centre, dispatches three mouse events", async () => { + it("double-clicks by ref with two consecutive native press/release pairs", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); ctx.refStore.set("e3", 1234, { tabId: 4 }); @@ -175,21 +175,59 @@ describe("handleClick", () => { expect(res.used_selector).toBeUndefined(); expect(res.x).toBeCloseTo(60); // (10 + 110) / 2 expect(res.y).toBeCloseTo(40); // (20 + 60) / 2 - // 1 scroll + 1 quad query + 3 mouse events = 5 CDP calls. const mouse = fake.sent.filter((c) => c.method === "Input.dispatchMouseEvent"); - expect(mouse).toHaveLength(3); - expect(mouse[0].params).toMatchObject({ type: "mouseMoved" }); - expect(mouse[1].params).toMatchObject({ - type: "mousePressed", - button: "left", - clickCount: 2, - modifiers: 2 | 8, - }); - expect(mouse[2].params).toMatchObject({ - type: "mouseReleased", - button: "left", - clickCount: 2, + expect(mouse.map((event) => event.params)).toEqual([ + { type: "mouseMoved", x: 60, y: 40, modifiers: 2 | 8 }, + ...[1, 2].flatMap((clickCount) => [ + { type: "mousePressed", x: 60, y: 40, button: "left", clickCount, modifiers: 2 | 8 }, + { type: "mouseReleased", x: 60, y: 40, button: "left", clickCount, modifiers: 2 | 8 }, + ]), + ]); + }); + + it.each([ + "cancel", + "overlay", + ])("stops the second DOM press after %s interrupts the first", async (reason) => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const controller = new AbortController(); + let released = false; + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), + "Runtime.evaluate": () => ({ + result: { value: released && reason === "overlay" ? "covered" : "absent" }, + }), + "Input.dispatchMouseEvent": (params) => { + if ((params as { type?: string })?.type === "mouseReleased") { + released = true; + if (reason === "cancel") controller.abort(); + } + return {}; + }, }); + fake.overlayHit.mockImplementation(async () => ({ + result: { value: released && reason === "overlay" ? "covered" : "absent" }, + })); + const result = await handleClick( + sm, + { session_id: "aa11", ref: "e3", click_count: 2 }, + { + cdp: fake.cdp, + tabsApi: fake.tabsApi, + signal: controller.signal, + }, + ); + expect(result).toMatchObject({ code: reason === "cancel" ? "cancelled" : "cdp_failed" }); + expect( + fake.sent.filter((c) => c.method === "Input.dispatchMouseEvent").map((c) => c.params), + ).toMatchObject([ + { type: "mouseMoved" }, + { type: "mousePressed", clickCount: 1 }, + { type: "mouseReleased", clickCount: 1 }, + ]); }); it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => { @@ -533,7 +571,7 @@ describe("handleClick", () => { expect(res).toMatchObject({ code: "invalid_params", - message: "click_count must be greater than zero", + message: "click_count must be a positive integer", data: { effect_state: "none" }, }); expect(res).not.toHaveProperty("data.reason"); diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index ab14e8e22..0a1af3298 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -510,8 +510,8 @@ export async function clickResolvedTarget( } const clickCount = params.click_count ?? 1; - if (clickCount < 1) { - return { code: "invalid_params", message: "click_count must be greater than zero" }; + if (!Number.isSafeInteger(clickCount) || clickCount < 1) { + return { code: "invalid_params", message: "click_count must be a positive integer" }; } const error = await withClickOverlay(target.tabId, centre, deps, (verifyOverlay) => dispatchClickAtPoint(target.tabId, centre, params, deps, verifyOverlay, undefined, markSent), @@ -527,7 +527,7 @@ export async function clickResolvedTarget( }); } -/** Shared mouse lifecycle; visual clicks additionally verify after move and emit full double clicks. */ +/** Shared mouse lifecycle; visual clicks additionally verify their captured target. */ async function dispatchClickAtPoint( tabId: number, point: { x: number; y: number }, @@ -574,10 +574,9 @@ async function dispatchClickAtPoint( const error = await beforePress(); if (error) return failure(error); } - const counts = beforePress - ? Array.from({ length: params.click_count ?? 1 }, (_, i) => i + 1) - : [count]; - for (count of counts) { + // clickCount describes this press in a consecutive sequence; it does not + // ask Chrome to synthesize the preceding clicks for DOM targets. + for (count = 1; count <= (params.click_count ?? 1); count++) { if (beforePress && count > 1) { const error = await beforePress(); if (error) return failure(error);