Skip to content

Commit

Permalink
fix: ensure that we capture service worker requests
Browse files Browse the repository at this point in the history
  • Loading branch information
ryanthemanuel committed Dec 13, 2023
1 parent 687dc63 commit a7cedfe
Show file tree
Hide file tree
Showing 35 changed files with 1,321 additions and 60 deletions.
20 changes: 18 additions & 2 deletions packages/proxy/lib/http/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import type { RemoteStates } from '@packages/server/lib/remote_states'
import type { CookieJar, SerializableAutomationCookie } from '@packages/server/lib/util/cookies'
import type { ResourceTypeAndCredentialManager } from '@packages/server/lib/util/resourceTypeAndCredentialManager'
import type { ProtocolManagerShape } from '@packages/types'
import type Protocol from 'devtools-protocol'

function getRandomColorFn () {
return chalk.hex(`#${Number(
Expand Down Expand Up @@ -429,10 +430,13 @@ export class Http {
}
}

reset () {
reset (options: { fullReset?: boolean } = {}) {
this.buffers.reset()
this.setAUTUrl(undefined)
this.preRequests.reset()

if (options.fullReset) {
this.preRequests.reset()
}
}

setBuffer (buffer) {
Expand All @@ -451,6 +455,18 @@ export class Http {
this.preRequests.addPendingUrlWithoutPreRequest(url)
}

updateServiceWorkerRegistrations (data: Protocol.ServiceWorker.WorkerRegistrationUpdatedEvent) {
this.preRequests.updateServiceWorkerRegistrations(data)
}

updateServiceWorkerVersions (data: Protocol.ServiceWorker.WorkerVersionUpdatedEvent) {
this.preRequests.updateServiceWorkerVersions(data)
}

updateServiceWorkerClientSideRegistrations (data: { scriptURL: string, initiatorURL: string }) {
this.preRequests.updateServiceWorkerClientSideRegistrations(data)
}

setProtocolManager (protocolManager: ProtocolManagerShape) {
this.protocolManager = protocolManager
this.preRequests.setProtocolManager(protocolManager)
Expand Down
53 changes: 47 additions & 6 deletions packages/proxy/lib/http/util/prerequests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import type {
} from '@packages/proxy'
import type { ProtocolManagerShape } from '@packages/types'
import Debug from 'debug'
import { ServiceWorkerManager } from './service-worker-manager'
import type Protocol from 'devtools-protocol'

const debug = Debug('cypress:proxy:http:util:prerequests')
const debugVerbose = Debug('cypress-verbose:proxy:http:util:prerequests')
Expand Down Expand Up @@ -108,6 +110,7 @@ export class PreRequests {
pendingUrlsWithoutPreRequests = new QueueMap<PendingUrlWithoutPreRequest>()
sweepIntervalTimer: NodeJS.Timeout
protocolManager?: ProtocolManagerShape
serviceWorkerManager: ServiceWorkerManager = new ServiceWorkerManager()

constructor (
requestTimeout = 2000,
Expand Down Expand Up @@ -147,8 +150,23 @@ export class PreRequests {
}

addPending (browserPreRequest: BrowserPreRequest) {
metrics.browserPreRequestsReceived++
const key = `${browserPreRequest.method}-${decodeURI(browserPreRequest.url)}`

// The initial request that loads the service worker does not always get sent to CDP. Thus, we need to explicitly ignore it. We determine
// it's the service worker request via the `service-worker` header
if (browserPreRequest.headers?.['Service-Worker'] === 'script') {
debugVerbose('Ignoring service worker script since we are not guaranteed to receive it', key)

return
}

if (this.serviceWorkerManager.processBrowserPreRequest(browserPreRequest)) {
debugVerbose('Not correlating request since it is fully controlled by service worker and the correlation will happen within the service worker', key)

return
}

metrics.browserPreRequestsReceived++
const pendingRequest = this.pendingRequests.shift(key)

if (pendingRequest) {
Expand Down Expand Up @@ -220,11 +238,10 @@ export class PreRequests {
}

get (req: CypressIncomingRequest, ctxDebug, callback: GetPreRequestCb) {
// The initial request that loads the service worker does not get sent to CDP and it happens prior
// to the service worker target being added. Thus, we need to explicitly ignore it. We determine
// it's the service worker request via the `sec-fetch-dest` header
if (req.headers['sec-fetch-dest'] === 'serviceworker') {
ctxDebug('Ignoring request with sec-fetch-dest: serviceworker', req.proxiedUrl)
// The initial request that loads the service worker does not always get sent to CDP. Thus, we need to explicitly ignore it. We determine
// it's the service worker request via the `service-worker` header
if (req.headers?.['service-worker'] === 'script') {
ctxDebug('Ignoring service worker script since we are not guaranteed to receive it', req.proxiedUrl)

callback({
noPreRequestExpected: true,
Expand All @@ -242,6 +259,7 @@ export class PreRequests {
if (pendingPreRequest) {
metrics.immediatelyMatchedRequests++
ctxDebug('Incoming request %s matches known pre-request: %o', key, pendingPreRequest)

callback({
browserPreRequest: {
...pendingPreRequest.browserPreRequest,
Expand Down Expand Up @@ -305,7 +323,30 @@ export class PreRequests {
delete pendingRequest.callback
}

updateServiceWorkerRegistrations (data: Protocol.ServiceWorker.WorkerRegistrationUpdatedEvent) {
data.registrations.forEach((registration) => {
if (registration.isDeleted) {
this.serviceWorkerManager.unregisterServiceWorker({ registrationId: registration.registrationId })
} else {
this.serviceWorkerManager.registerServiceWorker({ registrationId: registration.registrationId, scopeURL: registration.scopeURL })
}
})
}

updateServiceWorkerVersions (data: Protocol.ServiceWorker.WorkerVersionUpdatedEvent) {
data.versions.forEach((version) => {
if (version.status === 'activated') {
this.serviceWorkerManager.addActivatedServiceWorker({ registrationId: version.registrationId, scriptURL: version.scriptURL })
}
})
}

updateServiceWorkerClientSideRegistrations (data: { scriptURL: string, initiatorURL: string }) {
this.serviceWorkerManager.addInitiatorToServiceWorker({ scriptURL: data.scriptURL, initiatorURL: data.initiatorURL })
}

reset () {
this.serviceWorkerManager = new ServiceWorkerManager()
this.pendingPreRequests = new QueueMap<PendingPreRequest>()

// Clear out the pending requests timeout callbacks first then clear the queue
Expand Down
155 changes: 155 additions & 0 deletions packages/proxy/lib/http/util/service-worker-manager.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
import type { BrowserPreRequest } from '../../types'

type ServiceWorkerRegistration = {
registrationId: string
scopeURL: string
activatedServiceWorker?: ServiceWorker
}

type ServiceWorker = {
registrationId: string
scriptURL: string
initiatorURL?: string
controlledURLs: Set<string>
}

type RegisterServiceWorkerOptions = {
registrationId: string
scopeURL: string
}

type UnregisterServiceWorkerOptions = {
registrationId: string
}

type AddActivatedServiceWorkerOptions = {
registrationId: string
scriptURL: string
}

type AddInitiatorToServiceWorkerOptions = {
scriptURL: string
initiatorURL: string
}

/**
* Manages service worker registrations and their controlled URLs.
*
* The basic lifecycle is as follows:
*
* 1. A service worker is registered via `registerServiceWorker`.
* 2. The service worker is activated via `addActivatedServiceWorker`.
*
* At some point while 1 and 2 are happening:
*
* 3. We receive a message from the browser that a service worker has been registered with the `addInitiatorToServiceWorker` method.
*
* At this point, when the manager tries to process a browser pre-request, it will check if the request is controlled by a service worker.
* It determines it is controlled by a service worker if:
*
* 1. The document URL for the browser pre-request matches the initiator URL for the service worker.
* 2. The request URL is within the scope of the service worker or the request URL's initiator is controlled by the service worker.
*/
export class ServiceWorkerManager {
private serviceWorkerRegistrations: Map<string, ServiceWorkerRegistration> = new Map<string, ServiceWorkerRegistration>()
private pendingInitiators: Map<string, string> = new Map<string, string>()

/**
* Registers the given service worker with the given scope. Will not overwrite an existing registration.
*/
registerServiceWorker ({ registrationId, scopeURL }: RegisterServiceWorkerOptions) {
// Only register service workers if they haven't already been registered
if (this.serviceWorkerRegistrations.has(registrationId) && this.serviceWorkerRegistrations.get(registrationId)?.scopeURL === scopeURL) {
return
}

this.serviceWorkerRegistrations.set(registrationId, {
registrationId,
scopeURL,
})
}

/**
* Unregisters the service worker with the given registration ID.
*/
unregisterServiceWorker ({ registrationId }: UnregisterServiceWorkerOptions) {
this.serviceWorkerRegistrations.delete(registrationId)
}

/**
* Adds an activated service worker to the manager.
*/
addActivatedServiceWorker ({ registrationId, scriptURL }: AddActivatedServiceWorkerOptions) {
const registration = this.serviceWorkerRegistrations.get(registrationId)

if (registration) {
const initiatorURL = this.pendingInitiators.get(scriptURL)

registration.activatedServiceWorker = {
registrationId,
scriptURL,
controlledURLs: new Set<string>(),
initiatorURL: initiatorURL || registration.activatedServiceWorker?.initiatorURL,
}

this.pendingInitiators.delete(scriptURL)
}
}

/**
* Adds an initiator URL to a service worker. If the service worker has not yet been activated, the initiator URL is added to a pending list and will
* be added to the service worker when it is activated.
*/
addInitiatorToServiceWorker ({ scriptURL, initiatorURL }: AddInitiatorToServiceWorkerOptions) {
let initiatorAdded = false

this.serviceWorkerRegistrations.forEach((registration) => {
if (registration.activatedServiceWorker && registration.activatedServiceWorker.scriptURL === scriptURL) {
registration.activatedServiceWorker.initiatorURL = initiatorURL

initiatorAdded = true
}
})

if (!initiatorAdded) {
this.pendingInitiators.set(scriptURL, initiatorURL)
}
}

/**
* Processes a browser pre-request to determine if it is controlled by a service worker. If it is, the service worker's controlled URLs are updated with the given request URL.
*
* @param browserPreRequest The browser pre-request to process.
* @returns `true` if the request is controlled by a service worker, `false` otherwise.
*/
processBrowserPreRequest (browserPreRequest: BrowserPreRequest) {
if (browserPreRequest.initiator?.type === 'preload') {
return false
}

let requestControlledByServiceWorker = false

this.serviceWorkerRegistrations.forEach((registration) => {
const activatedServiceWorker = registration.activatedServiceWorker
const paramlessDocumentURL = browserPreRequest.documentURL.split('?')[0]

if (!activatedServiceWorker || activatedServiceWorker.initiatorURL !== paramlessDocumentURL) {
return
}

const paramlessURL = browserPreRequest.url.split('?')[0]
const paramlessInitiatorURL = browserPreRequest.initiator?.url?.split('?')[0]
const paramlessCallStackURL = browserPreRequest.initiator?.stack?.callFrames[0]?.url?.split('?')[0]
const urlIsControlled = paramlessURL.startsWith(registration.scopeURL)
const initiatorUrlIsControlled = paramlessInitiatorURL && activatedServiceWorker.controlledURLs?.has(paramlessInitiatorURL)
const topStackUrlIsControlled = paramlessCallStackURL && activatedServiceWorker.controlledURLs?.has(paramlessCallStackURL)

if (urlIsControlled || initiatorUrlIsControlled || topStackUrlIsControlled) {
activatedServiceWorker.controlledURLs.add(paramlessURL)
requestControlledByServiceWorker = true
}
})

return requestControlledByServiceWorker
}
}
17 changes: 15 additions & 2 deletions packages/proxy/lib/network-proxy.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { telemetry } from '@packages/telemetry'
import { Http, ServerCtx } from './http'
import type { BrowserPreRequest } from './types'
import type Protocol from 'devtools-protocol'

export class NetworkProxy {
http: Http
Expand All @@ -21,6 +22,18 @@ export class NetworkProxy {
this.http.addPendingUrlWithoutPreRequest(url)
}

updateServiceWorkerRegistrations (data: Protocol.ServiceWorker.WorkerRegistrationUpdatedEvent) {
this.http.updateServiceWorkerRegistrations(data)
}

updateServiceWorkerVersions (data: Protocol.ServiceWorker.WorkerVersionUpdatedEvent) {
this.http.updateServiceWorkerVersions(data)
}

updateServiceWorkerClientSideRegistrations (data: { scriptURL: string, initiatorURL: string }) {
this.http.updateServiceWorkerClientSideRegistrations(data)
}

handleHttpRequest (req, res) {
const span = telemetry.startSpan({
name: 'network:proxy:handleHttpRequest',
Expand All @@ -46,8 +59,8 @@ export class NetworkProxy {
this.http.setBuffer(buffer)
}

reset () {
this.http.reset()
reset (options: { fullReset?: boolean } = {}) {
this.http.reset(options)
}

setProtocolManager (protocolManager) {
Expand Down
3 changes: 3 additions & 0 deletions packages/proxy/lib/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import type { Request, Response } from 'express'
import type { ProxyTimings } from '@packages/types'
import type { ResourceType } from '@packages/net-stubbing'
import type { BackendRoute } from '@packages/net-stubbing/lib/server/types'
import type { Protocol } from 'devtools-protocol'

/**
* An incoming request to the Cypress web server.
Expand Down Expand Up @@ -63,6 +64,8 @@ export type BrowserPreRequest = {
resourceType: ResourceType
originalResourceType: string | undefined
errorHandled?: boolean
initiator?: Protocol.Network.Initiator
documentURL: string
cdpRequestWillBeSentTimestamp: number
cdpRequestWillBeSentReceivedTimestamp: number
}
Expand Down
1 change: 1 addition & 0 deletions packages/proxy/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
"@packages/server": "0.0.0-development",
"@types/express": "4.17.2",
"@types/supertest": "2.0.10",
"devtools-protocol": "0.0.927104",
"express": "4.17.3",
"supertest": "6.0.1",
"typescript": "^4.7.4"
Expand Down
Loading

0 comments on commit a7cedfe

Please sign in to comment.