Emulate Page.frameStartedNavigating CDP event

src/bidiMapper/BidiMapper.ts

@@ -22,7 +22,7 @@
  */
 export {BidiServer, MapperOptions} from './BidiServer.js';
 export type {CdpConnection} from '../cdp/CdpConnection.js';
-export type {CdpClient} from '../cdp/CdpClient.js';
+export type {CdpClient, ExtendedCdpClient} from '../cdp/CdpClient.js';
 export {EventEmitter} from '../utils/EventEmitter.js';
 export type {BidiTransport} from './BidiTransport.js';
 export {OutgoingMessage} from './OutgoingMessage.js';

src/bidiMapper/modules/cdp/CdpTarget.ts

@@ -17,7 +17,11 @@
  */
 import type {Protocol} from 'devtools-protocol';
 
-import type {CdpClient} from '../../../cdp/CdpClient.js';
+import {
+  type CdpClient,
+  type ExtendedCdpClient,
+  CdpClientWithEmulatedEventsWrapper,
+} from '../../../cdp/CdpClient.js';
 import {BiDiModule} from '../../../protocol/chromium-bidi.js';
 import type {ChromiumBidi, Session} from '../../../protocol/protocol.js';
 import {Deferred} from '../../../utils/Deferred.js';
@@ -41,6 +45,7 @@ interface FetchStages {
 export class CdpTarget {
   readonly #id: Protocol.Target.TargetID;
   readonly #cdpClient: CdpClient;
+  readonly #extendedCdpClient: ExtendedCdpClient;
   readonly #browserCdpClient: CdpClient;
   readonly #parentCdpClient: CdpClient;
   readonly #realmStorage: RealmStorage;
@@ -120,6 +125,9 @@ export class CdpTarget {
   ) {
     this.#id = targetId;
     this.#cdpClient = cdpClient;
+    this.#extendedCdpClient = new CdpClientWithEmulatedEventsWrapper(
+      cdpClient,
+    );
     this.#browserCdpClient = browserCdpClient;
     this.#parentCdpClient = parentCdpClient;
     this.#eventManager = eventManager;
@@ -141,8 +149,8 @@ export class CdpTarget {
     return this.#id;
   }
 
-  get cdpClient(): CdpClient {
-    return this.#cdpClient;
+  get cdpClient(): ExtendedCdpClient {
+    return this.#extendedCdpClient;
   }
 
   get parentCdpClient(): CdpClient {

src/bidiMapper/modules/context/BrowsingContextImpl.ts

@@ -450,13 +450,14 @@ export class BrowsingContextImpl {
 
     // Required to detect navigation started.
     // http://goto.google.com/webdriver:detect-navigation-started
-    this.#cdpTarget.cdpClient.on('Network.requestWillBeSent', (params) => {
+    this.#cdpTarget.cdpClient.on('Page.frameStartedNavigating', (params) => {
       if (
         this.isTopLevelContext() &&
         params.frameId !== undefined &&
         params.frameId !== this.id
       ) {
-        // `Network.requestWillBeSent`'s frameId can be missing for top level browsing context.
+        // `Page.frameStartedNavigating` uses `Network.requestWillBeSent` and `frameId`
+        // can be missing for top level browsing context.
         return;
       }
 
@@ -465,12 +466,7 @@ export class BrowsingContextImpl {
         return;
       }
 
-      if (params.loaderId !== params.requestId) {
-        // Navigation requests have `loaderId` equals to `requestId`.
-        return;
-      }
-
-      this.#navigationTracker.requestWillBeSent(params.requestId);
+      this.#navigationTracker.frameStartedNavigating(params.loaderId);
     });
 
     this.#cdpTarget.cdpClient.on('Page.lifecycleEvent', (params) => {

src/bidiMapper/modules/context/NavigationTracker.ts

@@ -35,7 +35,7 @@ export type NavigationEventName =
 class NavigationState {
   started = new Deferred<void>();
   finished = new Deferred<NavigationEventName>();
-  cdpNetworkRequestId?: string;
+  loaderId?: string;
   // Hack.
   startedByBeforeUnload = false;
 
@@ -91,12 +91,16 @@ export class NavigationTracker {
   }
 
   dispose() {
+    this.#logger?.(LogType.debug, 'dispose');
+
     this.#currentNavigation.finished.resolve(
       ChromiumBidi.BrowsingContext.EventNames.NavigationAborted,
     );
   }
 
   createOngoingNavigation(url: string): NavigationState {
+    this.#logger?.(LogType.debug, 'createOngoingNavigation');
+
     if (this.#ongoingNavigation !== undefined) {
       this.#ongoingNavigation.finished.resolve(
         ChromiumBidi.BrowsingContext.EventNames.NavigationAborted,
@@ -113,6 +117,10 @@ export class NavigationTracker {
   #setEventListeners(navigation: NavigationState) {
     void navigation.started
       .then(() => {
+        this.#logger?.(
+          LogType.debug,
+          `Navigation ${navigation.navigationId} started`,
+        );
         this.#eventManager.registerEvent(
           {
             type: 'event',
@@ -129,6 +137,10 @@ export class NavigationTracker {
       });
 
     void navigation.finished.then((eventName: NavigationEventName) => {
+      this.#logger?.(
+        LogType.debug,
+        `Navigation ${navigation.navigationId} finished with ${eventName}`,
+      );
       if (
         [
           ChromiumBidi.BrowsingContext.EventNames.FragmentNavigated,
@@ -151,12 +163,11 @@ export class NavigationTracker {
   }
 
   frameStartedLoading() {
-    if (this.#ongoingNavigation === undefined) {
-      this.#ongoingNavigation = this.createOngoingNavigation('UNKNOWN');
-    }
+    this.#logger?.(LogType.debug, 'Page.frameStartedLoading');
   }
 
   navigatedWithinDocument(url: string, navigationType: string) {
+    this.#logger?.(LogType.debug, 'Page.navigatedWithinDocument');
     this.#currentNavigation.url = url;
 
     if (navigationType === 'fragment') {
@@ -190,8 +201,10 @@ export class NavigationTracker {
     this.#ongoingNavigation = undefined;
   }
 
-  requestWillBeSent(cdpNetworkRequestId: string) {
-    if (this.#currentNavigation.cdpNetworkRequestId === cdpNetworkRequestId) {
+  frameStartedNavigating(loaderId: string) {
+    this.#logger?.(LogType.debug, `Page.frameStartedNavigating ${loaderId}`);
+
+    if (this.#currentNavigation.loaderId === loaderId) {
       // The same request can be due to redirect. Ignore if so.
       return;
     }
@@ -213,19 +226,26 @@ export class NavigationTracker {
 
     this.#ongoingNavigation.started.resolve();
     this.#currentNavigation = this.#ongoingNavigation;
-    this.#currentNavigation.cdpNetworkRequestId = cdpNetworkRequestId;
+    this.#currentNavigation.loaderId = loaderId;
     this.#ongoingNavigation = undefined;
   }
 
   frameNavigated(url: string) {
+    this.#logger?.(LogType.debug, `Page.frameNavigated ${url}`);
+    if (this.#ongoingNavigation !== undefined) {
+      // In some cases (`about:blank`) the `requestWillBeSent` is not emitted.
+      this.#currentNavigation = this.#ongoingNavigation;
+    }
     this.#currentNavigation.url = url;
   }
 
   frameRequestedNavigation(url: string) {
+    this.#logger?.(LogType.debug, `Page.frameRequestedNavigation ${url}`);
     this.createOngoingNavigation(url);
   }
 
   lifecycleEventLoad() {
+    this.#logger?.(LogType.debug, 'Page.lifecycleEvent:load');
     this.#currentNavigation.finished.resolve(
       ChromiumBidi.BrowsingContext.EventNames.Load,
     );

src/bidiMapper/modules/network/NetworkModuleMocks.spec.ts

@@ -21,6 +21,7 @@ import {
   CloseError,
   type CdpClient,
   type CdpEvents,
+  type ExtendedCdpClient,
 } from '../../../cdp/CdpClient.js';
 import {EventEmitter} from '../../../utils/EventEmitter.js';
 
@@ -29,7 +30,7 @@ export class MockCdpNetworkEvents {
   static defaultFrameId = '099A5216AF03AAFEC988F214B024DF08';
   static defaultUrl = 'http://www.google.com';
 
-  cdpClient: CdpClient;
+  cdpClient: ExtendedCdpClient;
 
   url: string;
   requestId: string;
@@ -39,7 +40,7 @@ export class MockCdpNetworkEvents {
   private type: Protocol.Network.ResourceType;
 
   constructor(
-    cdpClient: CdpClient,
+    cdpClient: ExtendedCdpClient,
     {
       requestId,
       fetchId,

src/bidiMapper/modules/network/NetworkStorage.spec.ts

@@ -17,7 +17,7 @@
  */
 import {expect} from 'chai';
 
-import type {CdpClient} from '../../../cdp/CdpClient.js';
+import type {CdpClient, ExtendedCdpClient} from '../../../cdp/CdpClient.js';
 import {ChromiumBidi, Network} from '../../../protocol/protocol.js';
 import {ProcessingQueue} from '../../../utils/ProcessingQueue.js';
 import type {OutgoingMessage} from '../../OutgoingMessage.js';
@@ -48,7 +48,7 @@ describe('NetworkStorage', () => {
   ][] = [];
   let eventManager!: EventManager;
   let networkStorage!: NetworkStorage;
-  let cdpClient!: CdpClient;
+  let cdpClient!: ExtendedCdpClient;
   let processingQueue!: ProcessingQueue<OutgoingMessage>;
 
   // TODO: Better way of getting Events

src/bidiMapper/modules/script/Realm.ts

@@ -16,7 +16,7 @@
  */
 import {Protocol} from 'devtools-protocol';
 
-import type {CdpClient} from '../../../cdp/CdpClient.js';
+import type {ExtendedCdpClient} from '../../../cdp/CdpClient.js';
 import {
   ChromiumBidi,
   NoSuchHandleException,
@@ -32,7 +32,7 @@ import {ChannelProxy} from './ChannelProxy.js';
 import type {RealmStorage} from './RealmStorage.js';
 
 export abstract class Realm {
-  readonly #cdpClient: CdpClient;
+  readonly #cdpClient: ExtendedCdpClient;
   readonly #eventManager: EventManager;
   readonly #executionContextId: Protocol.Runtime.ExecutionContextId;
   readonly #logger?: LoggerFn;
@@ -41,7 +41,7 @@ export abstract class Realm {
   readonly #realmStorage: RealmStorage;
 
   constructor(
-    cdpClient: CdpClient,
+    cdpClient: ExtendedCdpClient,
     eventManager: EventManager,
     executionContextId: Protocol.Runtime.ExecutionContextId,
     logger: LoggerFn | undefined,
@@ -177,7 +177,7 @@ export abstract class Realm {
     };
   }
 
-  get cdpClient(): CdpClient {
+  get cdpClient(): ExtendedCdpClient {
     return this.#cdpClient;
   }
 

src/bidiMapper/modules/script/WindowRealm.ts

@@ -17,7 +17,7 @@
 
 import type {Protocol} from 'devtools-protocol';
 
-import type {CdpClient} from '../../../cdp/CdpClient.js';
+import type {ExtendedCdpClient} from '../../../cdp/CdpClient.js';
 import {
   NoSuchNodeException,
   UnknownErrorException,
@@ -42,7 +42,7 @@ export class WindowRealm extends Realm {
   constructor(
     browsingContextId: BrowsingContext.BrowsingContext,
     browsingContextStorage: BrowsingContextStorage,
-    cdpClient: CdpClient,
+    cdpClient: ExtendedCdpClient,
     eventManager: EventManager,
     executionContextId: Protocol.Runtime.ExecutionContextId,
     logger: LoggerFn | undefined,

src/bidiMapper/modules/script/WorkerRealm.ts

@@ -17,7 +17,7 @@
 
 import type {Protocol} from 'devtools-protocol';
 
-import type {CdpClient} from '../../../cdp/CdpClient.js';
+import type {ExtendedCdpClient} from '../../../cdp/CdpClient.js';
 import type {Script} from '../../../protocol/protocol.js';
 import type {LoggerFn} from '../../../utils/log.js';
 import type {BrowsingContextImpl} from '../context/BrowsingContextImpl.js';
@@ -38,7 +38,7 @@ export class WorkerRealm extends Realm {
   readonly #ownerRealms: Realm[];
 
   constructor(
-    cdpClient: CdpClient,
+    cdpClient: ExtendedCdpClient,
     eventManager: EventManager,
     executionContextId: Protocol.Runtime.ExecutionContextId,
     logger: LoggerFn | undefined,

src/cdp/CdpClient.ts

@@ -17,6 +17,7 @@
 
 import type {Protocol} from 'devtools-protocol';
 import type {ProtocolMapping} from 'devtools-protocol/types/protocol-mapping.js';
+import type {EventType} from 'mitt';
 
 import {EventEmitter} from '../utils/EventEmitter.js';
 
@@ -26,10 +27,79 @@ export type CdpEvents = {
   [Property in keyof ProtocolMapping.Events]: ProtocolMapping.Events[Property][0];
 };
 
+/**
+ * Emulated CDP events. These events are not native to the CDP but are synthesized by the
+ * BiDi mapper for convenience and compatibility. They are intended to simplify handling
+ * certain scenarios.
+ */
+type EmulatedEvents = {
+  /**
+   * Emulated CDP event emitted right before the `Network.requestWillBeSent` event
+   * indicating that a new navigation is about to start.
+   *
+   * http://go/webdriver:detect-navigation-started#bookmark=id.64balpqrmadv
+   */
+  'Page.frameStartedNavigating': {
+    loaderId: Protocol.Network.LoaderId;
+    url: string;
+    // Frame id can be omitted for the top-level frame.
+    frameId?: Protocol.Page.FrameId;
+  };
+};
+
 /** A error that will be thrown if/when the connection is closed. */
 export class CloseError extends Error {}
 
-export interface CdpClient extends EventEmitter<CdpEvents> {
+/**
+ * CDP client with additional emulated events.
+ */
+export interface ExtendedCdpClient extends CdpClientBase<EmulatedEvents> {}
+
+export class CdpClientWithEmulatedEventsWrapper
+  extends EventEmitter<EmulatedEvents & CdpEvents>
+  implements ExtendedCdpClient
+{
+  readonly #cdpClient: CdpClient;
+  constructor(cdpClient: CdpClient) {
+    super();
+    this.#cdpClient = cdpClient;
+    cdpClient.on('*', (event, params) => {
+      // We may encounter uses for EventEmitter other than CDP events,
+      // which we want to skip.
+      if (event === 'Network.requestWillBeSent') {
+        const eventParams = params as Protocol.Network.RequestWillBeSentEvent;
+        if (eventParams.loaderId === eventParams.requestId) {
+          this.emit('Page.frameStartedNavigating', {
+            loaderId: eventParams.loaderId,
+            url: eventParams.request.url,
+            frameId: eventParams.frameId,
+          });
+        }
+      }
+
+      this.emit(event, params);
+    });
+  }
+
+  get sessionId(): Protocol.Target.SessionID | undefined {
+    return this.#cdpClient.sessionId;
+  }
+
+  isCloseError(error: unknown): boolean {
+    return this.#cdpClient.isCloseError(error);
+  }
+
+  sendCommand<CdpMethod extends keyof ProtocolMapping.Commands>(
+    method: CdpMethod,
+    params?: ProtocolMapping.Commands[CdpMethod]['paramsType'][0],
+  ): Promise<ProtocolMapping.Commands[CdpMethod]['returnType']> {
+    return this.#cdpClient.sendCommand(method, params);
+  }
+}
+
+export interface CdpClientBase<
+  CdpEventExtensions extends Record<EventType, unknown>,
+> extends EventEmitter<CdpEvents & CdpEventExtensions> {
   /** Unique session identifier. */
   sessionId: Protocol.Target.SessionID | undefined;
 
@@ -55,6 +125,8 @@ export interface CdpClient extends EventEmitter<CdpEvents> {
   ): Promise<ProtocolMapping.Commands[CdpMethod]['returnType']>;
 }
 
+export interface CdpClient extends CdpClientBase<{}> {}
+
 /** Represents a high-level CDP connection to the browser. */
 export class MapperCdpClient
   extends EventEmitter<CdpEvents>