chore: respect iframe offset in input events (#2720)

Respect frame offset relative to the target.

---------

Co-authored-by: browser-automation-bot <[email protected]>

src/bidiMapper/modules/input/ActionDispatcher.ts

@@ -477,6 +477,34 @@ export class ActionDispatcher {
     } while (!last);
   }
 
+  async #getFrameOffset(): Promise<{x: number; y: number}> {
+    // https://github.com/w3c/webdriver/pull/1847 proposes dispatching events from
+    // the top-level browsing context. This implementation dispatches it on the top-most
+    // same-target frame, which is not top-level one in case of OOPiF.
+    // TODO: switch to the top-level browsing context.
+    try {
+      const {backendNodeId} =
+        await this.#context.cdpTarget.cdpClient.sendCommand(
+          'DOM.getFrameOwner',
+          {frameId: this.#context.id},
+        );
+      const {model: frameBoxModel} =
+        await this.#context.cdpTarget.cdpClient.sendCommand('DOM.getBoxModel', {
+          backendNodeId,
+        });
+      return {x: frameBoxModel.content[0]!, y: frameBoxModel.content[1]!};
+    } catch (e: any) {
+      if (
+        e.code === -32000 &&
+        e.message === 'Frame with the given id does not belong to the target.'
+      ) {
+        // Heuristic to determine if the browsing context is top-level in the target session.
+        return {x: 0, y: 0};
+      }
+      throw e;
+    }
+  }
+
   async #getCoordinateFromOrigin(
     origin: Input.Origin,
     offsetX: number,
@@ -486,23 +514,24 @@ export class ActionDispatcher {
   ) {
     let targetX: number;
     let targetY: number;
+    const frameOffset = await this.#getFrameOffset();
     switch (origin) {
       case 'viewport':
-        targetX = offsetX;
-        targetY = offsetY;
+        targetX = offsetX + frameOffset.x;
+        targetY = offsetY + frameOffset.y;
         break;
       case 'pointer':
-        targetX = startX + offsetX;
-        targetY = startY + offsetY;
+        targetX = startX + offsetX + frameOffset.x;
+        targetY = startY + offsetY + frameOffset.y;
         break;
       default: {
         const {x: posX, y: posY} = await getElementCenter(
           this.#context,
           origin.element,
         );
         // SAFETY: These can never be special numbers.
-        targetX = posX + offsetX;
-        targetY = posY + offsetY;
+        targetX = posX + offsetX + frameOffset.x;
+        targetY = posY + offsetY + frameOffset.y;
         break;
       }
     }

tests/conftest.py

@@ -410,10 +410,13 @@ async def activate_main_tab():
 
 
 @pytest.fixture
-def html(local_server_http):
+def html(local_server_http, local_server_http_another_host):
     """Return a factory for URL with the given content."""
-    def html(content=""):
-        return local_server_http.url_200(content=content)
+    def html(content="", same_origin=True):
+        if same_origin:
+            return local_server_http.url_200(content=content)
+        else:
+            return local_server_http_another_host.url_200(content=content)
 
     return html
 

tests/input/test_input.py

@@ -15,7 +15,8 @@
 
 import pytest
 from syrupy.filters import props
-from test_helpers import execute_command, goto_url
+from test_helpers import (AnyExtending, execute_command, goto_url,
+                          send_JSON_command, subscribe, wait_for_event)
 
 SET_FILES_HTML = """
 <input id=input type=file>
@@ -526,6 +527,106 @@ async def test_input_performActionsEmitsWheelEvents(websocket, context_id,
     assert result == snapshot(exclude=props("realm"))
 
 
+@pytest.mark.parametrize("same_origin", [True, False])
+@pytest.mark.asyncio
+async def test_click_iframe_context(websocket, context_id, html, same_origin,
+                                    read_sorted_messages):
+    # TODO: add test for double-nested iframes.
+    await subscribe(websocket, ["log.entryAdded"])
+
+    iframe_url = html(content="""
+        <script>
+            document.addEventListener('mousedown', function(event) {
+                document.body.textContent = `X: ${event.clientX}, Y: ${event.clientY}`;
+                console.log("mousedown event", event.clientX, event.clientY)
+            });
+            console.log("iframe loaded")
+        </script>
+        """,
+                      same_origin=same_origin)
+
+    # Parent page with an iFrame with margins and borders.
+    main_page_url = html(content=f"""
+        <iframe style="border: 10px solid red; margin-top: 20px; margin-left: 30px" src="{iframe_url}" />
+    """)
+
+    await send_JSON_command(
+        websocket, {
+            "method": "browsingContext.navigate",
+            "params": {
+                "url": main_page_url,
+                "context": context_id,
+                "wait": "none"
+            }
+        })
+
+    # Wait for the iframe to load. It cannot be guaranteed by the "wait"
+    # condition.
+    [_, frame_loaded_console_event] = await read_sorted_messages(2)
+    assert frame_loaded_console_event == AnyExtending({
+        "method": "log.entryAdded",
+        "params": {
+            'args': [{
+                'type': 'string',
+                'value': 'iframe loaded',
+            }, ],
+        }
+    })
+    iframe_id = frame_loaded_console_event["params"]["source"]["context"]
+
+    (X, Y) = (7, 13)
+    # Perform action with iframe as origin.
+    await send_JSON_command(
+        websocket, {
+            "method": "input.performActions",
+            "params": {
+                "context": iframe_id,
+                "actions": [{
+                    "type": "pointer",
+                    "id": "main_mouse",
+                    "actions": [{
+                        "origin": "pointer",
+                        "type": "pointerMove",
+                        "x": X,
+                        "y": Y,
+                    }, {
+                        "type": "pointerDown",
+                        "button": 0,
+                    }, {
+                        "type": "pointerUp",
+                        "button": 0,
+                    }, {
+                        "type": "pointerDown",
+                        "button": 1,
+                    }, {
+                        "type": "pointerUp",
+                        "button": 1,
+                    }]
+                }]
+            }
+        })
+
+    await wait_for_event(websocket, "log.entryAdded")
+
+    [mousedown_console_event] = await read_sorted_messages(1)
+    assert mousedown_console_event == AnyExtending({
+        "method": "log.entryAdded",
+        "params": {
+            'args': [
+                {
+                    'value': 'mousedown event',
+                },
+                {
+                    'value': X,
+                },
+                {
+                    'value': Y,
+                },
+            ],
+        }
+    })
+
+
 @pytest.mark.asyncio
 async def test_input_performActionsEmitsClickCountsByButton(
         websocket, context_id, html, activate_main_tab, snapshot):