From 1a6f87922cb4722792b5ead43de7d32feaf3ca19 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Fri, 6 Oct 2023 14:45:33 -0400 Subject: [PATCH] fix: report results correctly when the browser crashes mid-test (#27786) * report correct number of failures if browser crashes in the middle of a test suite * report failure correctly when browser crashes during test * refactor crash handling * correct exit option check, clean up debug * exit on success if exit option !== false * use default stats when reporter stats are unavailable * fix error messaging * move reporter types to an intermediate .ts file, to be combined with reporter.js when migrated * debug tab close test in ci * move debug env from pkg to ci yml * set debug env in spec * fix pckg * adds some logging to cri-client * remove event emit logging from project-base * revert snapshot for tab close system test * fixes console output for no exit on success * changelog * changelog wsp * cleanup * clean up tests * refactor to more straightforward control flow * rm export for unused type * correct tab close snapshot for ci * new system test for mid-test config crash * update snapshots --- cli/CHANGELOG.md | 1 + packages/server/lib/modes/run.ts | 93 +- packages/server/lib/project-base.ts | 8 +- packages/server/lib/types/reporter.ts | 48 + .../lib/util/graceful_crash_handling.ts | 115 + .../browser_crash_handling_spec.js | 50 +- system-tests/__snapshots__/record_spec.js | 2893 +++++++++-------- .../cypress-with-project-id.config.js | 15 + .../cypress/e2e/simple_multiple.cy.js | 13 + .../e2e/cypress/e2e/chrome_tab_crash.cy.js | 15 +- system-tests/test/record_spec.js | 45 +- 11 files changed, 1790 insertions(+), 1506 deletions(-) create mode 100644 packages/server/lib/types/reporter.ts create mode 100644 packages/server/lib/util/graceful_crash_handling.ts create mode 100644 system-tests/projects/config-with-crashing-plugin/cypress-with-project-id.config.js create mode 100644 system-tests/projects/config-with-crashing-plugin/cypress/e2e/simple_multiple.cy.js diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 83c36686124b..3b7f8fa6e726 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -8,6 +8,7 @@ _Released 10/03/2023 (PENDING)_ - Fixed an issue where requests were correlated in the wrong order in the proxy. This could cause an issue where the wrong request is used for `cy.intercept` or assets (e.g. stylesheets or images) may not properly be available in Test Replay. Addressed in [#27892](https://github.com/cypress-io/cypress/pull/27892). - Fixed an issue where a crashed Chrome renderer can cause the Test Replay recorder to hang. Addressed in [#27909](https://github.com/cypress-io/cypress/pull/27909). - Fixed an issue where multiple responses yielded from calls to `cy.wait()` would sometimes be out of order. Fixes [#27337](https://github.com/cypress-io/cypress/issues/27337). +- Enables test replay for executed specs in runs that have a spec that causes a browser crash. Addressed in [#27786](https://github.com/cypress-io/cypress/pull/27786) ## 13.3.0 diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index 95558d72a2ee..2dbee21338a6 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -21,12 +21,13 @@ import random from '../util/random' import system from '../util/system' import chromePolicyCheck from '../util/chrome_policy_check' import type { SpecWithRelativeRoot, SpecFile, TestingType, OpenProjectLaunchOpts, FoundBrowser, BrowserVideoController, VideoRecording, ProcessOptions } from '@packages/types' -import type { Cfg } from '../project-base' +import type { Cfg, ProjectBase } from '../project-base' import type { Browser } from '../browsers/types' import * as printResults from '../util/print-run' import type { ProtocolManager } from '../cloud/protocol' import { telemetry } from '@packages/telemetry' import { CypressRunResult, createPublicBrowser, createPublicConfig, createPublicRunResults, createPublicSpec, createPublicSpecResults } from './results' +import { EarlyExitTerminator } from '../util/graceful_crash_handling' type SetScreenshotMetadata = (data: TakeScreenshotProps) => void type ScreenshotMetadata = ReturnType @@ -36,18 +37,13 @@ type BeforeSpecRun = any type AfterSpecRun = any type Project = NonNullable> -let exitEarly = (err) => { - debug('set early exit error: %s', err.stack) - - earlyExitErr = err -} -let earlyExitErr: Error let currentSetScreenshotMetadata: SetScreenshotMetadata const debug = Debug('cypress:server:run') - const DELAY_TO_LET_VIDEO_FINISH_MS = 1000 +let earlyExitTerminator = new EarlyExitTerminator() + const relativeSpecPattern = (projectRoot, pattern) => { if (typeof pattern === 'string') { return pattern.replace(`${projectRoot}/`, '') @@ -411,53 +407,30 @@ function launchBrowser (options: { browser: Browser, spec: SpecWithRelativeRoot, return openProject.launch(browser, spec, browserOpts) } -function listenForProjectEnd (project, exit): Bluebird { +async function listenForProjectEnd (project: ProjectBase, exit: boolean): Promise { if (globalThis.CY_TEST_MOCK?.listenForProjectEnd) return Bluebird.resolve(globalThis.CY_TEST_MOCK.listenForProjectEnd) - return new Bluebird((resolve, reject) => { - if (exit === false) { - resolve = () => { + // if exit is false, we need to intercept the resolution of tests - whether + // an early exit with intermediate results, or a full run. + return new Promise((resolve, reject) => { + Promise.race([ + new Promise((res) => { + project.once('end', (results) => { + debug('project ended with results %O', results) + res(results) + }) + }), + earlyExitTerminator.waitForEarlyExit(project, exit), + ]).then((results) => { + if (exit === false) { + // eslint-disable-next-line no-console console.log('not exiting due to options.exit being false') + } else { + resolve(results) } - } - - const onEarlyExit = function (err) { - if (err.isFatalApiErr) { - return reject(err) - } - - console.log('') - errors.log(err) - - const obj = { - error: errors.stripAnsi(err.message), - stats: { - failures: 1, - tests: 0, - passes: 0, - pending: 0, - suites: 0, - skipped: 0, - wallClockDuration: 0, - wallClockStartedAt: new Date().toJSON(), - wallClockEndedAt: new Date().toJSON(), - }, - } - - return resolve(obj) - } - - project.once('end', (results) => resolve(results)) - - // if we already received a reason to exit early, go ahead and do it - if (earlyExitErr) { - return onEarlyExit(earlyExitErr) - } - - // otherwise override exitEarly so we exit as soon as there is a reason - exitEarly = (err) => { - onEarlyExit(err) - } + }).catch((err) => { + reject(err) + }) }) } @@ -730,6 +703,13 @@ async function waitForTestsToFinishRunning (options: { project: Project, screens results.video = null } + // the early exit terminator persists between specs, + // so if this spec crashed, the next one will report as + // a crash too unless it is reset. Would like to not rely + // on closure, but threading through fn props via options is also not + // great. + earlyExitTerminator = new EarlyExitTerminator() + return results } @@ -1005,7 +985,11 @@ async function ready (options: ReadyOptions) { // this needs to be a closure over `exitEarly` and not a reference // because `exitEarly` gets overwritten in `listenForProjectEnd` // TODO: refactor this so we don't need to extend options - const onError = options.onError = (err) => exitEarly(err) + + const onError = options.onError = (err) => { + debug('onError') + earlyExitTerminator.exitEarly(err) + } // alias and coerce to null let specPatternFromCli = options.spec || null @@ -1140,6 +1124,7 @@ async function ready (options: ReadyOptions) { } export async function run (options, loading: Promise) { + debug('run start') // Check if running as electron process if (require('../util/electron-app').isRunningAsElectronProcess({ debug })) { const app = require('electron').app @@ -1161,6 +1146,8 @@ export async function run (options, loading: Promise) { try { return ready(options) } catch (e) { - return exitEarly(e) + debug('caught outer error', e) + + return earlyExitTerminator.exitEarly(e) } } diff --git a/packages/server/lib/project-base.ts b/packages/server/lib/project-base.ts index f9ac70da24e9..e8172918c40c 100644 --- a/packages/server/lib/project-base.ts +++ b/packages/server/lib/project-base.ts @@ -376,7 +376,6 @@ export class ProjectBase extends EE { }, onMocha: async (event, runnable) => { - debug('onMocha', event) // bail if we dont have a // reporter instance if (!reporterInstance) { @@ -385,7 +384,12 @@ export class ProjectBase extends EE { reporterInstance.emit(event, runnable) - if (event === 'end') { + if (event === 'test:before:run') { + this.emit('test:before:run', { + runnable, + previousResults: reporterInstance?.results() || {}, + }) + } else if (event === 'end') { const [stats = {}] = await Promise.all([ (reporterInstance != null ? reporterInstance.end() : undefined), this.server.end(), diff --git a/packages/server/lib/types/reporter.ts b/packages/server/lib/types/reporter.ts new file mode 100644 index 000000000000..f09d52464afb --- /dev/null +++ b/packages/server/lib/types/reporter.ts @@ -0,0 +1,48 @@ +interface ReporterTestAttempt { + state: 'skipped' | 'failed' | 'passed' + error: any + timings: any + failedFromHookId: any + wallClockStartedAt: Date + wallClockDuration: number + videoTimestamp: any +} +interface ReporterTest { + testId: string + title: string[] + state: 'skipped' | 'passed' | 'failed' + body: string + displayError: any + attempts: ReporterTestAttempt[] +} + +export interface BaseReporterResults { + error?: string + stats: { + failures: number + tests: number + passes: number + pending: number + suites: number + skipped: number + wallClockDuration: number + wallClockStartedAt: string + wallClockEndedAt: string + } +} + +export interface ReporterResults extends BaseReporterResults { + reporter: string + reporterStats: { + suites: number + tests: number + passes: number + pending: number + failures: number + start: string + end: string + duration: number + } + hooks: any[] + tests: ReporterTest[] +} diff --git a/packages/server/lib/util/graceful_crash_handling.ts b/packages/server/lib/util/graceful_crash_handling.ts new file mode 100644 index 000000000000..38d42718fc49 --- /dev/null +++ b/packages/server/lib/util/graceful_crash_handling.ts @@ -0,0 +1,115 @@ +import type { ProjectBase } from '../project-base' +import type { BaseReporterResults, ReporterResults } from '../types/reporter' +import * as errors from '../errors' +import Debug from 'debug' +import pDefer, { DeferredPromise } from 'p-defer' + +const debug = Debug('cypress:util:crash_handling') + +const patchRunResultsAfterCrash = (error: Error, reporterResults: ReporterResults, mostRecentRunnable: any): ReporterResults => { + const endTime: number = reporterResults?.stats?.wallClockEndedAt ? Date.parse(reporterResults?.stats?.wallClockEndedAt) : new Date().getTime() + const wallClockDuration = reporterResults?.stats?.wallClockStartedAt ? + endTime - Date.parse(reporterResults.stats.wallClockStartedAt) : 0 + const endTimeStamp = new Date(endTime).toJSON() + + // in crash situations, the most recent report will not have the triggering test + // so the results are manually patched, which produces the expected exit=1 and + // terminal output indicating the failed test + return { + ...reporterResults, + stats: { + ...reporterResults?.stats, + wallClockEndedAt: endTimeStamp, + wallClockDuration, + failures: (reporterResults?.stats?.failures ?? 0) + 1, + skipped: (reporterResults?.stats?.skipped ?? 1) - 1, + }, + reporterStats: { + ...reporterResults?.reporterStats, + tests: (reporterResults?.reporterStats?.tests ?? 0) + 1, // crashed test does not increment this value + end: reporterResults?.reporterStats?.end || endTimeStamp, + duration: wallClockDuration, + failures: (reporterResults?.reporterStats?.failures ?? 0) + 1, + }, + tests: (reporterResults?.tests || []).map((test) => { + if (test.testId === mostRecentRunnable.id) { + return { + ...test, + state: 'failed', + attempts: [ + ...test.attempts.slice(0, -1), + { + ...test.attempts[test.attempts.length - 1], + state: 'failed', + }, + ], + } + } + + return test + }), + error: errors.stripAnsi(error.message), + } +} + +const defaultStats = (error: Error): BaseReporterResults => { + return { + error: errors.stripAnsi(error.message), + stats: { + failures: 1, + tests: 0, + passes: 0, + pending: 0, + suites: 0, + skipped: 0, + wallClockDuration: 0, + wallClockStartedAt: new Date().toJSON(), + wallClockEndedAt: new Date().toJSON(), + }, + } +} + +export class EarlyExitTerminator { + private terminator: DeferredPromise + + private pendingRunnable: any + private intermediateStats: ReporterResults | undefined + + constructor () { + this.terminator = pDefer() + } + + waitForEarlyExit (project: ProjectBase, exit?: boolean) { + debug('waiting for early exit') + + project.on('test:before:run', ({ + runnable, + previousResults, + }) => { + debug('preparing to run test, previous stats reported as %O', previousResults) + + this.intermediateStats = previousResults + this.pendingRunnable = runnable + }) + + return this.terminator.promise + } + + exitEarly (error) { + if (error.isFatalApiErr) { + this.terminator.reject(error) + + return + } + + // eslint-disable-next-line no-console + console.log('') + errors.log(error) + + const runResults: BaseReporterResults = (this.intermediateStats && this.pendingRunnable) ? + patchRunResultsAfterCrash(error, this.intermediateStats, this.pendingRunnable) : + defaultStats(error) + + this.terminator.resolve(runResults) + } +} diff --git a/system-tests/__snapshots__/browser_crash_handling_spec.js b/system-tests/__snapshots__/browser_crash_handling_spec.js index 56fc095e8eec..e73a03aebb52 100644 --- a/system-tests/__snapshots__/browser_crash_handling_spec.js +++ b/system-tests/__snapshots__/browser_crash_handling_spec.js @@ -17,7 +17,8 @@ exports['Browser Crash Handling / when the tab crashes in chrome / fails'] = ` Running: chrome_tab_crash.cy.js (1 of 2) - 1) navigates to about /html + a test suite with a browser crash + ✓ navigates to about:blank We detected that the Chrome Renderer process just crashed. @@ -37,24 +38,18 @@ https://on.cypress.io/renderer-process-crashed (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 0 │ - │ Passing: 0 │ + │ Tests: 2 │ + │ Passing: 1 │ │ Failing: 1 │ │ Pending: 0 │ │ Skipped: 0 │ - │ Screenshots: 1 │ + │ Screenshots: 0 │ │ Video: false │ │ Duration: X seconds │ │ Spec Ran: chrome_tab_crash.cy.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/chrome_tab_crash.cy.js/navigates to about html (1280x720) - (failed).png - - ──────────────────────────────────────────────────────────────────────────────────────────────────── Running: simple.cy.js (2 of 2) @@ -87,11 +82,11 @@ https://on.cypress.io/renderer-process-crashed Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ chrome_tab_crash.cy.js XX:XX - - 1 - - │ + │ ✖ chrome_tab_crash.cy.js XX:XX 2 1 1 - - │ ├────────────────────────────────────────────────────────────────────────────────────────────────┤ │ ✔ simple.cy.js XX:XX 1 1 - - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✖ 1 of 2 failed (50%) XX:XX 1 1 1 - - + ✖ 1 of 2 failed (50%) XX:XX 3 2 1 - - ` @@ -115,7 +110,8 @@ exports['Browser Crash Handling / when the tab crashes in electron / fails'] = ` Running: chrome_tab_crash.cy.js (1 of 2) - 1) navigates to about /html + a test suite with a browser crash + ✓ navigates to about:blank We detected that the Electron Renderer process just crashed. @@ -135,24 +131,18 @@ https://on.cypress.io/renderer-process-crashed (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 0 │ - │ Passing: 0 │ + │ Tests: 2 │ + │ Passing: 1 │ │ Failing: 1 │ │ Pending: 0 │ │ Skipped: 0 │ - │ Screenshots: 1 │ + │ Screenshots: 0 │ │ Video: false │ │ Duration: X seconds │ │ Spec Ran: chrome_tab_crash.cy.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/chrome_tab_crash.cy.js/navigates to about html (1280x720) - (failed).png - - ──────────────────────────────────────────────────────────────────────────────────────────────────── Running: simple.cy.js (2 of 2) @@ -185,11 +175,11 @@ https://on.cypress.io/renderer-process-crashed Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ chrome_tab_crash.cy.js XX:XX - - 1 - - │ + │ ✖ chrome_tab_crash.cy.js XX:XX 2 1 1 - - │ ├────────────────────────────────────────────────────────────────────────────────────────────────┤ │ ✔ simple.cy.js XX:XX 1 1 - - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✖ 1 of 2 failed (50%) XX:XX 1 1 1 - - + ✖ 1 of 2 failed (50%) XX:XX 3 2 1 - - ` @@ -229,7 +219,7 @@ This can happen for many different reasons: (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 0 │ + │ Tests: 1 │ │ Passing: 0 │ │ Failing: 1 │ │ Pending: 0 │ @@ -273,11 +263,11 @@ This can happen for many different reasons: Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ chrome_process_crash.cy.js XX:XX - - 1 - - │ + │ ✖ chrome_process_crash.cy.js XX:XX 1 - 1 - - │ ├────────────────────────────────────────────────────────────────────────────────────────────────┤ │ ✔ simple.cy.js XX:XX 1 1 - - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✖ 1 of 2 failed (50%) XX:XX 1 1 1 - - + ✖ 1 of 2 failed (50%) XX:XX 2 1 1 - - ` @@ -317,7 +307,7 @@ This can happen for many different reasons: (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 0 │ + │ Tests: 1 │ │ Passing: 0 │ │ Failing: 1 │ │ Pending: 0 │ @@ -371,11 +361,11 @@ This can happen for many different reasons: Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ chrome_process_crash.cy.js XX:XX - - 1 - - │ + │ ✖ chrome_process_crash.cy.js XX:XX 1 - 1 - - │ ├────────────────────────────────────────────────────────────────────────────────────────────────┤ │ ✔ simple.cy.js XX:XX 1 1 - - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✖ 1 of 2 failed (50%) XX:XX 1 1 1 - - + ✖ 1 of 2 failed (50%) XX:XX 2 1 1 - - ` diff --git a/system-tests/__snapshots__/record_spec.js b/system-tests/__snapshots__/record_spec.js index a518ba71b934..d811fa8f189c 100644 --- a/system-tests/__snapshots__/record_spec.js +++ b/system-tests/__snapshots__/record_spec.js @@ -272,931 +272,6 @@ We dynamically generated a new test to display this failure. Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 -` - -exports['e2e record api interaction errors project 404 errors and exits 1'] = ` -We could not find a Cypress Cloud project with the projectId: pid123 - -This projectId came from your cypress-with-project-id.config.js file or an environment variable. - -Please log into Cypress Cloud and find your project. - -We will list the correct projectId in the 'Settings' tab. - -Alternatively, you can create a new project directly from within the Cypress app. - -https://on.cypress.io/cloud - -` - -exports['e2e record api interaction errors update instance stdout warns but proceeds 1'] = ` - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass.cy.js) │ - │ Searched: cypress/e2e/record_pass* │ - │ Params: Tag: false, Group: false, Parallel: false │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: record_pass.cy.js (1 of 1) - Estimated: X second(s) - - - record pass - ✓ passes - - is pending - - - 1 passing - 1 pending - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 2 │ - │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: false │ - │ Duration: X seconds │ - │ Estimated: X second(s) │ - │ Spec Ran: record_pass.cy.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - - - (Uploading Cloud Artifacts) - - - Video - Nothing to upload - - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings - - (Uploaded Cloud Artifacts) - - - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png -Warning: We encountered an error communicating with our servers. - -This run will proceed, but will not be recorded. - -This error will not affect or change the exit code. - -StatusCodeError: 500 - "Internal Server Error" - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - - - -─────────────────────────────────────────────────────────────────────────────────────────────────────── - - Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 - - -` - -exports['e2e record recordKey errors and exits without recordKey 1'] = ` -You passed the --record flag but did not provide us your Record Key. - -You can pass us your Record Key like this: - - $ cypress run --record --key - -You can also set the key as an environment variable with the name: CYPRESS_RECORD_KEY - -https://on.cypress.io/how-do-i-record-runs - -` - -exports['e2e record projectId errors and exits without projectId 1'] = ` -You passed the --record flag but this project has not been setup to record. - -This project is missing the projectId inside of: cypress.config.js - -We cannot uniquely identify this project without this id. - -You need to setup this project to record. This will generate a unique projectId. - -Alternatively if you omit the --record flag this project will run without recording. - -https://on.cypress.io/recording-project-runs - -` - -exports['e2e record api interaction errors recordKey and projectId errors and exits on 401 1'] = ` -Your Record Key f858a...ee7e1 is not valid with this projectId: pid123 - -It may have been recently revoked by you or another user. - -Please log into Cypress Cloud to see the valid Record Keys. - -https://on.cypress.io/dashboard/projects/pid123 - -` - -exports['e2e record api interaction errors uploading assets warns but proceeds 1'] = ` - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass.cy.js) │ - │ Searched: cypress/e2e/record_pass* │ - │ Params: Tag: false, Group: false, Parallel: false │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: record_pass.cy.js (1 of 1) - Estimated: X second(s) - - - record pass - ✓ passes - - is pending - - - 1 passing - 1 pending - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 2 │ - │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: true │ - │ Duration: X seconds │ - │ Estimated: X second(s) │ - │ Spec Ran: record_pass.cy.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - - - (Video) - - - Video output: /XXX/XXX/XXX/cypress/videos/record_pass.cy.js.mp4 - - - (Uploading Cloud Artifacts) - - - Video - 1 kB /XXX/XXX/XXX/cypress/videos/record_pass.cy.js.mp4 - - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings - - (Uploaded Cloud Artifacts) - - - Video - Failed Uploading 1/2 /XXX/XXX/XXX/cypress/videos/record_pass.cy.js.mp4 - 500 - "Internal Server Error" - - Screenshot - Failed Uploading 2/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - 500 - "Internal Server Error" - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - - - -─────────────────────────────────────────────────────────────────────────────────────────────────────── - - Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 - - -` - -exports['e2e record misconfiguration errors and exits when no specs found 1'] = ` -Can't run because no spec files were found. - -We searched for specs matching this glob pattern: - - > /foo/bar/.projects/e2e/cypress/e2e/notfound/** - -` - -exports['e2e record recordKey warns but does not exit when is forked pr 1'] = ` -Warning: It looks like you are trying to record this run from a forked PR. - -The Record Key is missing. Your CI provider is likely not passing private environment variables to builds from forks. - -These results will not be recorded. - -This error will not affect or change the exit code. - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass.cy.js) │ - │ Searched: cypress/e2e/record_pass* │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: record_pass.cy.js (1 of 1) - - - record pass - ✓ passes - - is pending - - - 1 passing - 1 pending - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 2 │ - │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: false │ - │ Duration: X seconds │ - │ Spec Ran: record_pass.cy.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - - - -` - -exports['e2e record api interaction errors create run 422 errors and exits when group name is in use 1'] = ` -You passed the --group flag, but this group name has already been used for this run. - -The existing run is: https://cloud.cypress.io/runs/12345 - -The --group flag you passed was: e2e-tests - -If you are trying to parallelize this run, then also pass the --parallel flag, else pass a different group name. - -https://on.cypress.io/run-group-name-not-unique - -` - -exports['e2e record api interaction errors create run unknown 422 errors and exits when there is an unknown 422 response 1'] = ` -We encountered an unexpected error communicating with our servers. - -StatusCodeError: 422 - -{ - "code": "SOMETHING_UNKNOWN", - "message": "An unknown message here from the server." -} - -There is likely something wrong with the request. - -The --tag flag you passed was: nightly -The --group flag you passed was: e2e-tests -The --parallel flag you passed was: true -The --ciBuildId flag you passed was: ciBuildId123 - -` - -exports['e2e record api interaction errors create run 500 does not proceed and exits with error when parallelizing 1'] = ` -We encountered an unexpected error communicating with our servers. - -StatusCodeError: 500 - "Internal Server Error" - -Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. - -The --group flag you passed was: foo -The --ciBuildId flag you passed was: ciBuildId123 - -` - -exports['e2e record api interaction errors create instance 500 does not proceed and exits with error when parallelizing and creating instance 1'] = ` - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass.cy.js) │ - │ Searched: cypress/e2e/record_pass* │ - │ Params: Tag: nightly, Group: foo, Parallel: true │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - -We encountered an unexpected error communicating with our servers. - -StatusCodeError: 500 - "Internal Server Error" - -Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. - -The --group flag you passed was: foo -The --ciBuildId flag you passed was: ciBuildId123 - -` - -exports['e2e record api interaction errors update instance 500 does not proceed and exits with error when parallelizing and updating instance 1'] = ` - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass.cy.js) │ - │ Searched: cypress/e2e/record_pass* │ - │ Params: Tag: nightly, Group: foo, Parallel: true │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: record_pass.cy.js (1 of 1) - Estimated: X second(s) - - - record pass - ✓ passes - - is pending - - - 1 passing - 1 pending - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 2 │ - │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: false │ - │ Duration: X seconds │ - │ Estimated: X second(s) │ - │ Spec Ran: record_pass.cy.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - -We encountered an unexpected error communicating with our servers. - -StatusCodeError: 500 - "Internal Server Error" - -Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. - -The --group flag you passed was: foo -The --ciBuildId flag you passed was: ciBuildId123 - -` - -exports['e2e record api interaction errors api retries on error warns and does not create or update instances 1'] = ` -We encountered an unexpected error communicating with our servers. - -StatusCodeError: 500 - "Internal Server Error" - -We will retry 3 more times in X second(s)... - -We encountered an unexpected error communicating with our servers. - -StatusCodeError: 500 - "Internal Server Error" - -We will retry 2 more times in X second(s)... - -We encountered an unexpected error communicating with our servers. - -StatusCodeError: 500 - "Internal Server Error" - -We will retry 1 more time in X second(s)... - - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass.cy.js) │ - │ Searched: cypress/e2e/record_pass* │ - │ Params: Tag: nightly, Group: foo, Parallel: true │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - -We encountered an unexpected error communicating with our servers. - -StatusCodeError: 500 - "Internal Server Error" - -We will retry 3 more times in X second(s)... - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: record_pass.cy.js (1 of 1) - Estimated: X second(s) - - - record pass - ✓ passes - - is pending - - - 1 passing - 1 pending - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 2 │ - │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: false │ - │ Duration: X seconds │ - │ Estimated: X second(s) │ - │ Spec Ran: record_pass.cy.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - - - (Uploading Cloud Artifacts) - - - Video - Nothing to upload - - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings - - (Uploaded Cloud Artifacts) - - - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - - - -─────────────────────────────────────────────────────────────────────────────────────────────────────── - - Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 - - -` - -exports['e2e record recordKey warns but does not exit when is forked pr and parallel 1'] = ` -Warning: It looks like you are trying to record this run from a forked PR. - -The Record Key is missing. Your CI provider is likely not passing private environment variables to builds from forks. - -These results will not be recorded. - -This error will not affect or change the exit code. - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass.cy.js) │ - │ Searched: cypress/e2e/record_pass* │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: record_pass.cy.js (1 of 1) - - - record pass - ✓ passes - - is pending - - - 1 passing - 1 pending - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 2 │ - │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: false │ - │ Duration: X seconds │ - │ Spec Ran: record_pass.cy.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - - - -` - -exports['e2e record api interaction errors create run 402 - free plan exceeds monthly private tests errors and exits when on free plan and over recorded runs limit 1'] = ` -You've exceeded the limit of private test results under your free plan this month. The limit is 500 private test results. - -To continue recording tests this month you must upgrade your account. Please visit your billing to upgrade to another billing plan. - -https://on.cypress.io/dashboard/organizations/org-id-1234/billing - -` - -exports['e2e record api interaction errors create run 402 - parallel feature not available in plan errors and exits when attempting parallel run when not available in plan 1'] = ` -Parallelization is not included under your current billing plan. - -To run your tests in parallel, please visit your billing and upgrade to another plan with parallelization. - -https://on.cypress.io/dashboard/organizations/org-id-1234/billing - -` - -exports['e2e record api interaction errors create run 402 - unknown error errors and exits when there\'s an unknown 402 error 1'] = ` -We encountered an unexpected error communicating with our servers. - -StatusCodeError: 402 - -{ - "error": "Something went wrong" -} - -There is likely something wrong with the request. - -The --tag flag you passed was: - -` - -exports['e2e record api interaction errors create run 402 - free plan exceeds monthly tests errors and exits when on free plan and over recorded tests limit 1'] = ` -You've exceeded the limit of test results under your free plan this month. The limit is 500 test results. - -To continue recording tests this month you must upgrade your account. Please visit your billing to upgrade to another billing plan. - -https://on.cypress.io/dashboard/organizations/org-id-1234/billing - -` - -exports['e2e record api interaction errors create run 402 - grouping feature not available in plan errors and exits when attempting parallel run when not available in plan 1'] = ` -Grouping is not included under your current billing plan. - -To run your tests with groups, please visit your billing and upgrade to another plan with grouping. - -https://on.cypress.io/dashboard/organizations/org-id-1234/billing - -` - -exports['e2e record api interaction warnings create run warnings grace period - grouping feature warns when using parallel feature 1'] = ` -Grouping is not included under your free plan. - -Your plan is now in a grace period, which means your tests will still run with groups until 2999-12-31. Please upgrade your plan to continue running your tests with groups in the future. - -https://on.cypress.io/dashboard/organizations/org-id-1234/billing - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass.cy.js) │ - │ Searched: cypress/e2e/record_pass* │ - │ Params: Tag: false, Group: false, Parallel: false │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: record_pass.cy.js (1 of 1) - Estimated: X second(s) - - - record pass - ✓ passes - - is pending - - - 1 passing - 1 pending - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 2 │ - │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: false │ - │ Duration: X seconds │ - │ Estimated: X second(s) │ - │ Spec Ran: record_pass.cy.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - - - (Uploading Cloud Artifacts) - - - Video - Nothing to upload - - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - (Uploaded Cloud Artifacts) - - - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - - - -─────────────────────────────────────────────────────────────────────────────────────────────────────── - - Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 - - -` - -exports['e2e record api interaction warnings create run warnings grace period - parallel feature warns when using parallel feature 1'] = ` -Parallelization is not included under your free plan. - -Your plan is now in a grace period, which means your tests will still run in parallel until 2999-12-31. Please upgrade your plan to continue running your tests in parallel in the future. - -https://on.cypress.io/dashboard/organizations/org-id-1234/billing - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass.cy.js) │ - │ Searched: cypress/e2e/record_pass* │ - │ Params: Tag: false, Group: false, Parallel: false │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: record_pass.cy.js (1 of 1) - Estimated: X second(s) - - - record pass - ✓ passes - - is pending - - - 1 passing - 1 pending - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 2 │ - │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: false │ - │ Duration: X seconds │ - │ Estimated: X second(s) │ - │ Spec Ran: record_pass.cy.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - - - (Uploading Cloud Artifacts) - - - Video - Nothing to upload - - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - (Uploaded Cloud Artifacts) - - - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - - - -─────────────────────────────────────────────────────────────────────────────────────────────────────── - - Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 - - -` - -exports['e2e record api interaction warnings create run warnings unknown warning warns with unknown warning code 1'] = ` -Warning from Cypress Cloud: You are almost out of time - -Details: - -{ - "code": "OUT_OF_TIME", - "hadTime": 1000, - "name": "OutOfTime", - "spentTime": 999 -} - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass.cy.js) │ - │ Searched: cypress/e2e/record_pass* │ - │ Params: Tag: false, Group: false, Parallel: false │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: record_pass.cy.js (1 of 1) - Estimated: X second(s) - - - record pass - ✓ passes - - is pending - - - 1 passing - 1 pending - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 2 │ - │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: false │ - │ Duration: X seconds │ - │ Estimated: X second(s) │ - │ Spec Ran: record_pass.cy.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - - - (Uploading Cloud Artifacts) - - - Video - Nothing to upload - - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings - - (Uploaded Cloud Artifacts) - - - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - - - -─────────────────────────────────────────────────────────────────────────────────────────────────────── - - Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 - - ` exports['e2e record passing passes 2'] = [ @@ -1209,8 +284,8 @@ exports['e2e record passing passes 2'] = [ 'skipped': 1, 'failures': 1, 'wallClockStartedAt': '2018-02-01T20:14:19.323Z', - 'wallClockEndedAt': '2018-02-01T20:14:19.323Z', 'wallClockDuration': 1234, + 'wallClockEndedAt': '2018-02-01T20:14:19.323Z', }, 'tests': [ { @@ -1305,8 +380,8 @@ exports['e2e record passing passes 2'] = [ 'skipped': 0, 'failures': 0, 'wallClockStartedAt': '2018-02-01T20:14:19.323Z', - 'wallClockEndedAt': '2018-02-01T20:14:19.323Z', 'wallClockDuration': 1234, + 'wallClockEndedAt': '2018-02-01T20:14:19.323Z', }, 'tests': [ { @@ -1385,8 +460,8 @@ exports['e2e record passing passes 2'] = [ 'skipped': 0, 'failures': 1, 'wallClockStartedAt': '2018-02-01T20:14:19.323Z', - 'wallClockEndedAt': '2018-02-01T20:14:19.323Z', 'wallClockDuration': 1234, + 'wallClockEndedAt': '2018-02-01T20:14:19.323Z', }, 'tests': [ { @@ -1455,12 +530,662 @@ exports['e2e record passing passes 2'] = [ }, ] -exports['e2e record api interaction warnings create run warnings grace period - over private tests limit warns when over private test results 1'] = ` -You've exceeded the limit of private test results under your free plan this month. The limit is 500 private test results. +exports['e2e record misconfiguration errors and exits when no specs found 1'] = ` +Can't run because no spec files were found. + +We searched for specs matching this glob pattern: + + > /foo/bar/.projects/e2e/cypress/e2e/notfound/** + +` + +exports['e2e record misconfiguration errors and exits when no browser found 1'] = ` +Can't run because you've entered an invalid browser name. + +Browser: browserDoesNotExist was not found on your system or is not supported by Cypress. + +Cypress supports the following browsers: + - electron + - chrome + - chromium + - chrome:canary + - edge + - firefox + +You can also use a custom browser: https://on.cypress.io/customize-browsers + +Available browsers found on your system are: +- browser1 +- browser2 +- browser3 +` + +exports['e2e record empty specs succeeds when empty spec file 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 2 found (empty_suite.cy.js, empty.cy.js) │ + │ Searched: cypress/e2e/empty_suite.cy.js, cypress/e2e/empty.cy.js │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: empty_suite.cy.js (1 of 2) + Estimated: X second(s) + + + 0 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 0 │ + │ Passing: 0 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: empty_suite.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - Nothing to upload + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: empty.cy.js (2 of 2) + Estimated: X second(s) + + + 0 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 0 │ + │ Passing: 0 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: empty.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - Nothing to upload + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ empty_suite.cy.js XX:XX - - - - - │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✔ empty.cy.js XX:XX - - - - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX - - - - - + + +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 + + +` + +exports['e2e record projectId errors and exits without projectId 1'] = ` +You passed the --record flag but this project has not been setup to record. + +This project is missing the projectId inside of: cypress.config.js + +We cannot uniquely identify this project without this id. + +You need to setup this project to record. This will generate a unique projectId. + +Alternatively if you omit the --record flag this project will run without recording. + +https://on.cypress.io/recording-project-runs + +` + +exports['e2e record quiet mode respects quiet mode 1'] = ` + + + record pass + ✓ passes + - is pending + + + 1 passing + 1 pending + + +` + +exports['e2e record recordKey errors and exits without recordKey 1'] = ` +You passed the --record flag but did not provide us your Record Key. + +You can pass us your Record Key like this: + + $ cypress run --record --key + +You can also set the key as an environment variable with the name: CYPRESS_RECORD_KEY + +https://on.cypress.io/how-do-i-record-runs + +` + +exports['e2e record recordKey warns but does not exit when is forked pr 1'] = ` +Warning: It looks like you are trying to record this run from a forked PR. + +The Record Key is missing. Your CI provider is likely not passing private environment variables to builds from forks. + +These results will not be recorded. + +This error will not affect or change the exit code. + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: record_pass.cy.js (1 of 1) + + + record pass + ✓ passes + - is pending + + + 1 passing + 1 pending + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 1 │ + │ Skipped: 0 │ + │ Screenshots: 1 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: record_pass.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Screenshots) + + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 2 1 - 1 - + + +` + +exports['e2e record recordKey warns but does not exit when is forked pr and parallel 1'] = ` +Warning: It looks like you are trying to record this run from a forked PR. + +The Record Key is missing. Your CI provider is likely not passing private environment variables to builds from forks. + +These results will not be recorded. + +This error will not affect or change the exit code. + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: record_pass.cy.js (1 of 1) + + + record pass + ✓ passes + - is pending + + + 1 passing + 1 pending + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 1 │ + │ Skipped: 0 │ + │ Screenshots: 1 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: record_pass.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Screenshots) + + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 2 1 - 1 - + + +` + +exports['e2e record api skips specs records tests and exits without executing 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 2 found (a_record_instantfail.cy.js, b_record.cy.js) │ + │ Searched: cypress/e2e/a_record_instantfail.cy.js, cypress/e2e/b_record.cy.js │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: a_record_instantfail.cy.js (1 of 2) + Estimated: X second(s) + + This spec and its tests were skipped because the run has been canceled. + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: b_record.cy.js (2 of 2) + Estimated: X second(s) + + + b spec + ✓ b test + + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: b_record.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - Nothing to upload + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ - a_record_instantfail.cy.js SKIPPED - - - - - │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✔ b_record.cy.js XX:XX 1 1 - - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + - The run was canceled XX:XX 1 1 - - - + + +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 + + + Exiting with non-zero exit code because the run was canceled. + +` + +exports['e2e record api skips specs records tests and exits without executing in parallel 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 2 found (a_record_instantfail.cy.js, b_record.cy.js) │ + │ Searched: cypress/e2e/a_record_instantfail.cy.js, cypress/e2e/b_record.cy.js │ + │ Params: Tag: false, Group: abc, Parallel: true │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + -Your plan is now in a grace period, which means your tests will still be recorded until 2999-12-31. Please upgrade your plan to continue recording tests on Cypress Cloud in the future. +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: a_record_instantfail.cy.js (1 of 2) + Estimated: X second(s) + + This spec and its tests were skipped because the run has been canceled. + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: b_record.cy.js (2 of 2) + Estimated: X second(s) + + + b spec + ✓ b test + + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: b_record.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - Nothing to upload + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ - a_record_instantfail.cy.js SKIPPED - - - - - │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✔ b_record.cy.js XX:XX 1 1 - - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + - The run was canceled XX:XX 1 1 - - - + + +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 + + + Exiting with non-zero exit code because the run was canceled. + +` + +exports['e2e record video recording when video=false does not upload when not enabled 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: record_pass.cy.js (1 of 1) + Estimated: X second(s) + + + record pass +plugin stdout + ✓ passes + - is pending + + + 1 passing + 1 pending + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 1 │ + │ Skipped: 0 │ + │ Screenshots: 1 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: record_pass.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Screenshots) + + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) + + + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 2 1 - 1 - + + +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 + + +` + +exports['e2e record api interaction errors recordKey and projectId errors and exits on 401 1'] = ` +Your Record Key f858a...ee7e1 is not valid with this projectId: pid123 + +It may have been recently revoked by you or another user. + +Please log into Cypress Cloud to see the valid Record Keys. + +https://on.cypress.io/dashboard/projects/pid123 + +` + +exports['e2e record api interaction errors project 404 errors and exits 1'] = ` +We could not find a Cypress Cloud project with the projectId: pid123 + +This projectId came from your cypress-with-project-id.config.js file or an environment variable. + +Please log into Cypress Cloud and find your project. + +We will list the correct projectId in the 'Settings' tab. + +Alternatively, you can create a new project directly from within the Cypress app. + +https://on.cypress.io/cloud + +` + +exports['e2e record api interaction errors create run 500 errors and exits 1'] = ` +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 500 - "Internal Server Error" + +` + +exports['e2e record api interaction errors create run 500 when grouping without parallelization errors and exits 1'] = ` +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 500 - "Internal Server Error" + +The --group flag you passed was: foo +The --ciBuildId flag you passed was: ciBuildId123 + +` + +exports['e2e record api interaction errors create run 500 does not proceed and exits with error when parallelizing 1'] = ` +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 500 - "Internal Server Error" + +Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. + +The --group flag you passed was: foo +The --ciBuildId flag you passed was: ciBuildId123 + +` + +exports['e2e record api interaction errors create instance 500 does not proceed and exits with error when parallelizing and creating instance 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ + │ Params: Tag: nightly, Group: foo, Parallel: true │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 500 - "Internal Server Error" + +Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. + +The --group flag you passed was: foo +The --ciBuildId flag you passed was: ciBuildId123 + +` + +exports['e2e record api interaction errors create instance 500 without parallelization - does not proceed 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 2 found (a_record.cy.js, b_record.cy.js) │ + │ Searched: cypress/e2e/a_record.cy.js, cypress/e2e/b_record.cy.js │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 500 - "Internal Server Error" + +` -https://on.cypress.io/dashboard/organizations/org-id-1234/billing +exports['e2e record api interaction errors update instance 500 does not proceed and exits with error when parallelizing and updating instance 1'] = ` ==================================================================================================== @@ -1471,7 +1196,7 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing │ Browser: FooBrowser 88 │ │ Specs: 1 found (record_pass.cy.js) │ │ Searched: cypress/e2e/record_pass* │ - │ Params: Tag: false, Group: false, Parallel: false │ + │ Params: Tag: nightly, Group: foo, Parallel: true │ │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ @@ -1511,43 +1236,232 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) +We encountered an unexpected error communicating with our servers. - (Uploading Cloud Artifacts) +StatusCodeError: 500 - "Internal Server Error" - - Video - Nothing to upload - - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png +Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. - (Uploaded Cloud Artifacts) +The --group flag you passed was: foo +The --ciBuildId flag you passed was: ciBuildId123 - - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png +` + +exports['e2e record api interaction errors create run 422 errors and exits when group name is in use 1'] = ` +You passed the --group flag, but this group name has already been used for this run. + +The existing run is: https://cloud.cypress.io/runs/12345 + +The --group flag you passed was: e2e-tests + +If you are trying to parallelize this run, then also pass the --parallel flag, else pass a different group name. + +https://on.cypress.io/run-group-name-not-unique + +` + +exports['e2e record api interaction errors create run 412 errors and exits when request schema is invalid 1'] = ` +Recording this run failed. The request was invalid. + +Request Validation Error + +Errors: + +[ + "ci is the wrong type, saw null, expected object", + "commit is the wrong type, saw null, expected object", + "platform is the wrong type, saw null, expected object" +] + +Request Sent: + +{ + "ci": null, + "specs": [ + "cypress/e2e/record_pass.cy.js" + ], + "commit": null, + "group": null, + "platform": null, + "parallel": null, + "ciBuildId": null, + "projectId": "pid123", + "recordKey": "f85...7e1", + "specPattern": "cypress/e2e/record_pass*", + "tags": [ + "" + ], + "testingType": "e2e", + "runnerCapabilities": { + "dynamicSpecsInSerialMode": true, + "skipSpecAction": true, + "protocolMountVersion": 2 + } +} + +` + +exports['e2e record api interaction errors create run unknown 422 errors and exits when there is an unknown 422 response 1'] = ` +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 422 + +{ + "code": "SOMETHING_UNKNOWN", + "message": "An unknown message here from the server." +} + +There is likely something wrong with the request. + +The --tag flag you passed was: nightly +The --group flag you passed was: e2e-tests +The --parallel flag you passed was: true +The --ciBuildId flag you passed was: ciBuildId123 + +` + +exports['e2e record api interaction errors create run 402 - free plan exceeds monthly private tests errors and exits when on free plan and over recorded runs limit 1'] = ` +You've exceeded the limit of private test results under your free plan this month. The limit is 500 private test results. + +To continue recording tests this month you must upgrade your account. Please visit your billing to upgrade to another billing plan. + +https://on.cypress.io/dashboard/organizations/org-id-1234/billing + +` + +exports['e2e record api interaction errors create run 402 - free plan exceeds monthly tests errors and exits when on free plan and over recorded tests limit 1'] = ` +You've exceeded the limit of test results under your free plan this month. The limit is 500 test results. + +To continue recording tests this month you must upgrade your account. Please visit your billing to upgrade to another billing plan. + +https://on.cypress.io/dashboard/organizations/org-id-1234/billing + +` + +exports['e2e record api interaction errors create run 402 - parallel feature not available in plan errors and exits when attempting parallel run when not available in plan 1'] = ` +Parallelization is not included under your current billing plan. + +To run your tests in parallel, please visit your billing and upgrade to another plan with parallelization. + +https://on.cypress.io/dashboard/organizations/org-id-1234/billing + +` + +exports['e2e record api interaction errors create run 402 - grouping feature not available in plan errors and exits when attempting parallel run when not available in plan 1'] = ` +Grouping is not included under your current billing plan. + +To run your tests with groups, please visit your billing and upgrade to another plan with grouping. + +https://on.cypress.io/dashboard/organizations/org-id-1234/billing + +` + +exports['e2e record api interaction errors create run 402 - unknown error errors and exits when there\'s an unknown 402 error 1'] = ` +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 402 + +{ + "error": "Something went wrong" +} + +There is likely something wrong with the request. + +The --tag flag you passed was: + +` + +exports['e2e record api interaction errors create run 402 - auto cancel not available in plan errors and exits when auto cancel not available in plan 1'] = ` +Auto Cancellation is not included under your current billing plan. + +To enable this service, please visit your billing and upgrade to another plan with Auto Cancellation. + +https://on.cypress.io/dashboard/organizations/org-id-1234/billing + +` + +exports['e2e record api interaction errors create instance errors and exits on createInstance error 1'] = ` ==================================================================================================== - (Run Finished) + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (a_record_instantfail.cy.js) │ + │ Searched: cypress/e2e/a_record_instantfail.cy.js │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 500 - "Internal Server Error" + +` + +exports['e2e record api interaction errors postInstanceTests without parallelization errors and exits 1'] = ` + +==================================================================================================== + + (Run Starting) - Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 2 found (a_record.cy.js, b_record.cy.js) │ + │ Searched: cypress/e2e/a_record.cy.js, cypress/e2e/b_record.cy.js │ + │ Params: Tag: false, Group: foo, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - -─────────────────────────────────────────────────────────────────────────────────────────────────────── - - Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: a_record.cy.js (1 of 2) + Estimated: X second(s) +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 500 - "Internal Server Error" +The --group flag you passed was: foo +The --ciBuildId flag you passed was: 1 ` -exports['e2e record api interaction warnings create run warnings grace period - over tests limit warns when over test results 1'] = ` -You've exceeded the limit of test results under your free plan this month. The limit is 500 test results. +exports['e2e record api interaction errors postInstanceTests with parallelization errors and exits 1'] = ` -Your plan is now in a grace period, which means you will have the full benefits of your current plan until 2999-12-31. +==================================================================================================== -Please visit your billing to upgrade your plan. + (Run Starting) -https://on.cypress.io/dashboard/organizations/org-id-1234/billing + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 2 found (a_record.cy.js, b_record.cy.js) │ + │ Searched: cypress/e2e/a_record.cy.js, cypress/e2e/b_record.cy.js │ + │ Params: Tag: false, Group: foo, Parallel: true │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: a_record.cy.js (1 of 2) + Estimated: X second(s) +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 500 - "Internal Server Error" + +Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. + +The --group flag you passed was: foo +The --ciBuildId flag you passed was: ciBuildId123 + +` + +exports['e2e record api interaction errors postInstanceResults errors and exits in serial 1'] = ` ==================================================================================================== @@ -1596,43 +1510,15 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing (Screenshots) - - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - - - (Uploading Cloud Artifacts) - - - Video - Nothing to upload - - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - (Uploaded Cloud Artifacts) - - - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - - - -─────────────────────────────────────────────────────────────────────────────────────────────────────── - - Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 - + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) -` +We encountered an unexpected error communicating with our servers. -exports['e2e record api interaction warnings create run warnings paid plan - over private tests limit warns when over private test results 1'] = ` -You've exceeded the limit of test results under your current billing plan this month. The limit is 500 private test results. +StatusCodeError: 500 - "Internal Server Error" -To continue getting the full benefits of your current plan, please visit your billing to upgrade. +` -https://on.cypress.io/dashboard/organizations/org-id-1234/billing +exports['e2e record api interaction errors update instance stdout warns but proceeds 1'] = ` ==================================================================================================== @@ -1688,10 +1574,18 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing - Video - Nothing to upload - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings (Uploaded Cloud Artifacts) - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png +Warning: We encountered an error communicating with our servers. + +This run will proceed, but will not be recorded. + +This error will not affect or change the exit code. + +StatusCodeError: 500 - "Internal Server Error" ==================================================================================================== @@ -1712,12 +1606,7 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing ` -exports['e2e record api interaction warnings create run warnings paid plan - over tests limit warns when over test results 1'] = ` -You've exceeded the limit of test results under your current billing plan this month. The limit is 500 test results. - -To continue getting the full benefits of your current plan, please visit your billing to upgrade. - -https://on.cypress.io/dashboard/organizations/org-id-1234/billing +exports['e2e record api interaction errors uploading assets warns but proceeds 1'] = ` ==================================================================================================== @@ -1757,7 +1646,7 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing │ Pending: 1 │ │ Skipped: 0 │ │ Screenshots: 1 │ - │ Video: false │ + │ Video: true │ │ Duration: X seconds │ │ Estimated: X second(s) │ │ Spec Ran: record_pass.cy.js │ @@ -1769,14 +1658,21 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) + (Video) + + - Video output: /XXX/XXX/XXX/cypress/videos/record_pass.cy.js.mp4 + + (Uploading Cloud Artifacts) - - Video - Nothing to upload + - Video - 1 kB /XXX/XXX/XXX/cypress/videos/record_pass.cy.js.mp4 - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings (Uploaded Cloud Artifacts) - - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Video - Failed Uploading 1/2 /XXX/XXX/XXX/cypress/videos/record_pass.cy.js.mp4 - 500 - "Internal Server Error" + - Screenshot - Failed Uploading 2/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - 500 - "Internal Server Error" ==================================================================================================== @@ -1797,12 +1693,25 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing ` -exports['e2e record api interaction warnings create run warnings free plan - over tests limit v2 warns when over test results 1'] = ` -You've exceeded the limit of test results under your free billing plan this month. The limit is 500 test results. +exports['e2e record api interaction errors api retries on error warns and does not create or update instances 1'] = ` +We encountered an unexpected error communicating with our servers. -To continue getting the full benefits of your current plan, please visit your billing to upgrade. +StatusCodeError: 500 - "Internal Server Error" + +We will retry 3 more times in X second(s)... + +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 500 - "Internal Server Error" + +We will retry 2 more times in X second(s)... + +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 500 - "Internal Server Error" + +We will retry 1 more time in X second(s)... -https://on.cypress.io/dashboard/organizations/org-id-1234/billing ==================================================================================================== @@ -1813,10 +1722,16 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing │ Browser: FooBrowser 88 │ │ Specs: 1 found (record_pass.cy.js) │ │ Searched: cypress/e2e/record_pass* │ - │ Params: Tag: false, Group: false, Parallel: false │ + │ Params: Tag: nightly, Group: foo, Parallel: true │ │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 500 - "Internal Server Error" + +We will retry 3 more times in X second(s)... + ──────────────────────────────────────────────────────────────────────────────────────────────────── @@ -1858,6 +1773,7 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing - Video - Nothing to upload - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings (Uploaded Cloud Artifacts) @@ -1882,117 +1798,118 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing ` -exports['e2e record api interaction errors create run 500 errors and exits 1'] = ` +exports['e2e record api interaction errors sendPreflight [F1] socket errors fails after retrying 1'] = ` We encountered an unexpected error communicating with our servers. -StatusCodeError: 500 - "Internal Server Error" +RequestError: Error: socket hang up + +We will retry 1 more time in X second(s)... + +We encountered an unexpected error communicating with our servers. + +RequestError: Error: socket hang up + +Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. + +The --group flag you passed was: foo +The --ciBuildId flag you passed was: ciBuildId123 ` -exports['e2e record api interaction errors create run 500 when grouping without parallelization errors and exits 1'] = ` +exports['e2e record api interaction errors sendPreflight [F1] 500 status code errors with empty body fails after retrying 1'] = ` +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 500 - "Internal Server Error" + +We will retry 1 more time in X second(s)... + We encountered an unexpected error communicating with our servers. StatusCodeError: 500 - "Internal Server Error" +Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. + The --group flag you passed was: foo The --ciBuildId flag you passed was: ciBuildId123 ` -exports['e2e record api interaction errors create instance 500 without parallelization - does not proceed 1'] = ` - -==================================================================================================== +exports['e2e record api interaction errors sendPreflight [F1] 500 status code errors with body fails after retrying 1'] = ` +We encountered an unexpected error communicating with our servers. - (Run Starting) +StatusCodeError: 500 - "Internal Server Error" - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 2 found (a_record.cy.js, b_record.cy.js) │ - │ Searched: cypress/e2e/a_record.cy.js, cypress/e2e/b_record.cy.js │ - │ Params: Tag: false, Group: false, Parallel: false │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ +We will retry 1 more time in X second(s)... We encountered an unexpected error communicating with our servers. StatusCodeError: 500 - "Internal Server Error" +Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. + +The --group flag you passed was: foo +The --ciBuildId flag you passed was: ciBuildId123 + ` -exports['e2e record api interaction errors create instance errors and exits on createInstance error 1'] = ` +exports['e2e record api interaction errors sendPreflight [F2] 404 status code with JSON body fails without retrying 1'] = ` +We could not find a Cypress Cloud project with the projectId: pid123 -==================================================================================================== +This projectId came from your cypress-with-project-id.config.js file or an environment variable. - (Run Starting) +Please log into Cypress Cloud and find your project. - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (a_record_instantfail.cy.js) │ - │ Searched: cypress/e2e/a_record_instantfail.cy.js │ - │ Params: Tag: false, Group: false, Parallel: false │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ +We will list the correct projectId in the 'Settings' tab. -We encountered an unexpected error communicating with our servers. +Alternatively, you can create a new project directly from within the Cypress app. -StatusCodeError: 500 - "Internal Server Error" +https://on.cypress.io/cloud ` -exports['e2e record api interaction errors postInstanceTests without parallelization errors and exits 1'] = ` +exports['e2e record api interaction errors sendPreflight [F2] 404 status code with empty body fails without retrying 1'] = ` +We could not find a Cypress Cloud project with the projectId: pid123 -==================================================================================================== +This projectId came from your cypress-with-project-id.config.js file or an environment variable. - (Run Starting) +Please log into Cypress Cloud and find your project. - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 2 found (a_record.cy.js, b_record.cy.js) │ - │ Searched: cypress/e2e/a_record.cy.js, cypress/e2e/b_record.cy.js │ - │ Params: Tag: false, Group: foo, Parallel: false │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ +We will list the correct projectId in the 'Settings' tab. +Alternatively, you can create a new project directly from within the Cypress app. -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: a_record.cy.js (1 of 2) - Estimated: X second(s) +https://on.cypress.io/cloud + +` + +exports['e2e record api interaction errors sendPreflight [F3] 422 status code with invalid decryption fails without retrying 1'] = ` We encountered an unexpected error communicating with our servers. -StatusCodeError: 500 - "Internal Server Error" +DecryptionError: JWE Recipients missing or incorrect type + +Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. The --group flag you passed was: foo -The --ciBuildId flag you passed was: 1 +The --ciBuildId flag you passed was: ciBuildId123 ` -exports['e2e record api interaction errors postInstanceTests with parallelization errors and exits 1'] = ` +exports['e2e record api interaction errors sendPreflight [F3] 201 status code with invalid decryption fails without retrying 1'] = ` +We encountered an unexpected error communicating with our servers. -==================================================================================================== +DecryptionError: JWE Recipients missing or incorrect type - (Run Starting) +Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 2 found (a_record.cy.js, b_record.cy.js) │ - │ Searched: cypress/e2e/a_record.cy.js, cypress/e2e/b_record.cy.js │ - │ Params: Tag: false, Group: foo, Parallel: true │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ +The --group flag you passed was: foo +The --ciBuildId flag you passed was: ciBuildId123 +` -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: a_record.cy.js (1 of 2) - Estimated: X second(s) +exports['e2e record api interaction errors sendPreflight [F3] 200 status code with empty body fails without retrying 1'] = ` We encountered an unexpected error communicating with our servers. -StatusCodeError: 500 - "Internal Server Error" +DecryptionError: General JWE must be an object Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. @@ -2001,7 +1918,50 @@ The --ciBuildId flag you passed was: ciBuildId123 ` -exports['e2e record api interaction errors postInstanceResults errors and exits in serial 1'] = ` +exports['e2e record api interaction errors sendPreflight [F4] 412 status code with valid decryption fails without retrying 1'] = ` +Recording this run failed. The request was invalid. + +Recording is not working + +Errors: + +[ + "attempted to send invalid data" +] + +Request Sent: + +{ + "projectId": "cy12345" +} + +` + +exports['e2e record api interaction errors sendPreflight [F5] 422 status code with valid decryption on createRun errors and exits when group name is in use 1'] = ` +You passed the --group flag, but this group name has already been used for this run. + +The existing run is: https://cloud.cypress.io/runs/12345 + +The --group flag you passed was: e2e-tests + +If you are trying to parallelize this run, then also pass the --parallel flag, else pass a different group name. + +https://on.cypress.io/run-group-name-not-unique + +` + +exports['e2e record api interaction errors sendPreflight [W1] warning message renders preflight warning messages prior to run warnings 1'] = ` +Warning from Cypress Cloud: + +---------------------------------------------------------------------- +This feature will not be supported soon, please check with Cypress to learn more: https://on.cypress.io/ +---------------------------------------------------------------------- + +You've exceeded the limit of private test results under your free plan this month. The limit is 500 private test results. + +Your plan is now in a grace period, which means your tests will still be recorded until 2999-12-31. Please upgrade your plan to continue recording tests on Cypress Cloud in the future. + +https://on.cypress.io/dashboard/organizations/org-id-1234/billing ==================================================================================================== @@ -2012,7 +1972,7 @@ exports['e2e record api interaction errors postInstanceResults errors and exits │ Browser: FooBrowser 88 │ │ Specs: 1 found (record_pass.cy.js) │ │ Searched: cypress/e2e/record_pass* │ - │ Params: Tag: false, Group: false, Parallel: false │ + │ Params: Tag: nightly, Group: foo, Parallel: true │ │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ @@ -2048,17 +2008,45 @@ exports['e2e record api interaction errors postInstanceResults errors and exits └────────────────────────────────────────────────────────────────────────────────────────────────┘ - (Screenshots) + (Screenshots) + + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) + + + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 2 1 - 1 - - - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) -We encountered an unexpected error communicating with our servers. +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 -StatusCodeError: 500 - "Internal Server Error" ` -exports['e2e record api skips specs records tests and exits without executing 1'] = ` +exports['e2e record api interaction warnings create run warnings grace period - over private tests limit warns when over private test results 1'] = ` +You've exceeded the limit of private test results under your free plan this month. The limit is 500 private test results. + +Your plan is now in a grace period, which means your tests will still be recorded until 2999-12-31. Please upgrade your plan to continue recording tests on Cypress Cloud in the future. + +https://on.cypress.io/dashboard/organizations/org-id-1234/billing ==================================================================================================== @@ -2067,8 +2055,8 @@ exports['e2e record api skips specs records tests and exits without executing 1' ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Cypress: 1.2.3 │ │ Browser: FooBrowser 88 │ - │ Specs: 2 found (a_record_instantfail.cy.js, b_record.cy.js) │ - │ Searched: cypress/e2e/a_record_instantfail.cy.js, cypress/e2e/b_record.cy.js │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ │ Params: Tag: false, Group: false, Parallel: false │ │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ @@ -2076,45 +2064,48 @@ exports['e2e record api skips specs records tests and exits without executing 1' ──────────────────────────────────────────────────────────────────────────────────────────────────── - Running: a_record_instantfail.cy.js (1 of 2) - Estimated: X second(s) - - This spec and its tests were skipped because the run has been canceled. - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: b_record.cy.js (2 of 2) + Running: record_pass.cy.js (1 of 1) Estimated: X second(s) - b spec - ✓ b test + record pass + ✓ passes + - is pending 1 passing + 1 pending (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 1 │ + │ Tests: 2 │ │ Passing: 1 │ │ Failing: 0 │ - │ Pending: 0 │ + │ Pending: 1 │ │ Skipped: 0 │ - │ Screenshots: 0 │ + │ Screenshots: 1 │ │ Video: false │ │ Duration: X seconds │ │ Estimated: X second(s) │ - │ Spec Ran: b_record.cy.js │ + │ Spec Ran: record_pass.cy.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ + (Screenshots) + + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) + + (Uploading Cloud Artifacts) - Video - Nothing to upload - - Screenshot - Nothing to upload - - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png ==================================================================================================== @@ -2123,11 +2114,9 @@ exports['e2e record api skips specs records tests and exits without executing 1' Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ - a_record_instantfail.cy.js SKIPPED - - - - - │ - ├────────────────────────────────────────────────────────────────────────────────────────────────┤ - │ ✔ b_record.cy.js XX:XX 1 1 - - - │ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - The run was canceled XX:XX 1 1 - - - + ✔ All specs passed! XX:XX 2 1 - 1 - ─────────────────────────────────────────────────────────────────────────────────────────────────────── @@ -2135,11 +2124,16 @@ exports['e2e record api skips specs records tests and exits without executing 1' Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 - Exiting with non-zero exit code because the run was canceled. - ` -exports['e2e record api skips specs records tests and exits without executing in parallel 1'] = ` +exports['e2e record api interaction warnings create run warnings grace period - over tests limit warns when over test results 1'] = ` +You've exceeded the limit of test results under your free plan this month. The limit is 500 test results. + +Your plan is now in a grace period, which means you will have the full benefits of your current plan until 2999-12-31. + +Please visit your billing to upgrade your plan. + +https://on.cypress.io/dashboard/organizations/org-id-1234/billing ==================================================================================================== @@ -2148,54 +2142,57 @@ exports['e2e record api skips specs records tests and exits without executing in ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Cypress: 1.2.3 │ │ Browser: FooBrowser 88 │ - │ Specs: 2 found (a_record_instantfail.cy.js, b_record.cy.js) │ - │ Searched: cypress/e2e/a_record_instantfail.cy.js, cypress/e2e/b_record.cy.js │ - │ Params: Tag: false, Group: abc, Parallel: true │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ + │ Params: Tag: false, Group: false, Parallel: false │ │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ ──────────────────────────────────────────────────────────────────────────────────────────────────── - Running: a_record_instantfail.cy.js (1 of 2) - Estimated: X second(s) - - This spec and its tests were skipped because the run has been canceled. - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: b_record.cy.js (2 of 2) + Running: record_pass.cy.js (1 of 1) Estimated: X second(s) - b spec - ✓ b test + record pass + ✓ passes + - is pending 1 passing + 1 pending (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 1 │ + │ Tests: 2 │ │ Passing: 1 │ │ Failing: 0 │ - │ Pending: 0 │ + │ Pending: 1 │ │ Skipped: 0 │ - │ Screenshots: 0 │ + │ Screenshots: 1 │ │ Video: false │ │ Duration: X seconds │ │ Estimated: X second(s) │ - │ Spec Ran: b_record.cy.js │ + │ Spec Ran: record_pass.cy.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ + (Screenshots) + + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) + + (Uploading Cloud Artifacts) - Video - Nothing to upload - - Screenshot - Nothing to upload - - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png ==================================================================================================== @@ -2204,11 +2201,9 @@ exports['e2e record api skips specs records tests and exits without executing in Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ - a_record_instantfail.cy.js SKIPPED - - - - - │ - ├────────────────────────────────────────────────────────────────────────────────────────────────┤ - │ ✔ b_record.cy.js XX:XX 1 1 - - - │ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - The run was canceled XX:XX 1 1 - - - + ✔ All specs passed! XX:XX 2 1 - 1 - ─────────────────────────────────────────────────────────────────────────────────────────────────────── @@ -2216,11 +2211,14 @@ exports['e2e record api skips specs records tests and exits without executing in Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 - Exiting with non-zero exit code because the run was canceled. - ` -exports['e2e record empty specs succeeds when empty spec file 1'] = ` +exports['e2e record api interaction warnings create run warnings grace period - parallel feature warns when using parallel feature 1'] = ` +Parallelization is not included under your free plan. + +Your plan is now in a grace period, which means your tests will still run in parallel until 2999-12-31. Please upgrade your plan to continue running your tests in parallel in the future. + +https://on.cypress.io/dashboard/organizations/org-id-1234/billing ==================================================================================================== @@ -2229,8 +2227,8 @@ exports['e2e record empty specs succeeds when empty spec file 1'] = ` ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Cypress: 1.2.3 │ │ Browser: FooBrowser 88 │ - │ Specs: 2 found (empty_suite.cy.js, empty.cy.js) │ - │ Searched: cypress/e2e/empty_suite.cy.js, cypress/e2e/empty.cy.js │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ │ Params: Tag: false, Group: false, Parallel: false │ │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ @@ -2238,65 +2236,48 @@ exports['e2e record empty specs succeeds when empty spec file 1'] = ` ──────────────────────────────────────────────────────────────────────────────────────────────────── - Running: empty_suite.cy.js (1 of 2) + Running: record_pass.cy.js (1 of 1) Estimated: X second(s) - 0 passing + record pass + ✓ passes + - is pending + + + 1 passing + 1 pending (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 0 │ - │ Passing: 0 │ + │ Tests: 2 │ + │ Passing: 1 │ │ Failing: 0 │ - │ Pending: 0 │ + │ Pending: 1 │ │ Skipped: 0 │ - │ Screenshots: 0 │ + │ Screenshots: 1 │ │ Video: false │ │ Duration: X seconds │ │ Estimated: X second(s) │ - │ Spec Ran: empty_suite.cy.js │ + │ Spec Ran: record_pass.cy.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - (Uploading Cloud Artifacts) - - - Video - Nothing to upload - - Screenshot - Nothing to upload - - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: empty.cy.js (2 of 2) - Estimated: X second(s) - - - 0 passing - - - (Results) + (Screenshots) - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 0 │ - │ Passing: 0 │ - │ Failing: 0 │ - │ Pending: 0 │ - │ Skipped: 0 │ - │ Screenshots: 0 │ - │ Video: false │ - │ Duration: X seconds │ - │ Estimated: X second(s) │ - │ Spec Ran: empty.cy.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) (Uploading Cloud Artifacts) - Video - Nothing to upload - - Screenshot - Nothing to upload - - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png ==================================================================================================== @@ -2305,11 +2286,9 @@ exports['e2e record empty specs succeeds when empty spec file 1'] = ` Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ empty_suite.cy.js XX:XX - - - - - │ - ├────────────────────────────────────────────────────────────────────────────────────────────────┤ - │ ✔ empty.cy.js XX:XX - - - - - │ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX - - - - - + ✔ All specs passed! XX:XX 2 1 - 1 - ─────────────────────────────────────────────────────────────────────────────────────────────────────── @@ -2319,62 +2298,12 @@ exports['e2e record empty specs succeeds when empty spec file 1'] = ` ` -exports['e2e record quiet mode respects quiet mode 1'] = ` - - - record pass - ✓ passes - - is pending - - - 1 passing - 1 pending - - -` - -exports['e2e record api interaction errors create run 412 errors and exits when request schema is invalid 1'] = ` -Recording this run failed. The request was invalid. - -Request Validation Error - -Errors: - -[ - "ci is the wrong type, saw null, expected object", - "commit is the wrong type, saw null, expected object", - "platform is the wrong type, saw null, expected object" -] - -Request Sent: - -{ - "ci": null, - "specs": [ - "cypress/e2e/record_pass.cy.js" - ], - "commit": null, - "group": null, - "platform": null, - "parallel": null, - "ciBuildId": null, - "projectId": "pid123", - "recordKey": "f85...7e1", - "specPattern": "cypress/e2e/record_pass*", - "tags": [ - "" - ], - "testingType": "e2e", - "runnerCapabilities": { - "dynamicSpecsInSerialMode": true, - "skipSpecAction": true, - "protocolMountVersion": 2 - } -} +exports['e2e record api interaction warnings create run warnings grace period - grouping feature warns when using parallel feature 1'] = ` +Grouping is not included under your free plan. -` +Your plan is now in a grace period, which means your tests will still run with groups until 2999-12-31. Please upgrade your plan to continue running your tests with groups in the future. -exports['e2e record video recording when video=false does not upload when not enabled 1'] = ` +https://on.cypress.io/dashboard/organizations/org-id-1234/billing ==================================================================================================== @@ -2397,7 +2326,6 @@ exports['e2e record video recording when video=false does not upload when not en record pass -plugin stdout ✓ passes - is pending @@ -2431,7 +2359,6 @@ plugin stdout - Video - Nothing to upload - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings (Uploaded Cloud Artifacts) @@ -2456,137 +2383,180 @@ plugin stdout ` -exports['e2e record api interaction errors create run 402 - auto cancel not available in plan errors and exits when auto cancel not available in plan 1'] = ` -Auto Cancellation is not included under your current billing plan. +exports['e2e record api interaction warnings create run warnings paid plan - over private tests limit warns when over private test results 1'] = ` +You've exceeded the limit of test results under your current billing plan this month. The limit is 500 private test results. -To enable this service, please visit your billing and upgrade to another plan with Auto Cancellation. +To continue getting the full benefits of your current plan, please visit your billing to upgrade. https://on.cypress.io/dashboard/organizations/org-id-1234/billing -` +==================================================================================================== -exports['e2e record misconfiguration errors and exits when no browser found 1'] = ` -Can't run because you've entered an invalid browser name. + (Run Starting) -Browser: browserDoesNotExist was not found on your system or is not supported by Cypress. + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ -Cypress supports the following browsers: - - electron - - chrome - - chromium - - chrome:canary - - edge - - firefox -You can also use a custom browser: https://on.cypress.io/customize-browsers +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: record_pass.cy.js (1 of 1) + Estimated: X second(s) -Available browsers found on your system are: -- browser1 -- browser2 -- browser3 -` -exports['e2e record api interaction errors sendPreflight [F1] 500 status code errors with empty body fails after retrying 1'] = ` -We encountered an unexpected error communicating with our servers. + record pass + ✓ passes + - is pending -StatusCodeError: 500 - "Internal Server Error" -We will retry 1 more time in X second(s)... + 1 passing + 1 pending -We encountered an unexpected error communicating with our servers. -StatusCodeError: 500 - "Internal Server Error" + (Results) -Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 1 │ + │ Skipped: 0 │ + │ Screenshots: 1 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: record_pass.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ -The --group flag you passed was: foo -The --ciBuildId flag you passed was: ciBuildId123 -` + (Screenshots) -exports['e2e record api interaction errors sendPreflight [F2] 404 status code with JSON body fails without retrying 1'] = ` -We could not find a Cypress Cloud project with the projectId: pid123 + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) -This projectId came from your cypress-with-project-id.config.js file or an environment variable. -Please log into Cypress Cloud and find your project. + (Uploading Cloud Artifacts) -We will list the correct projectId in the 'Settings' tab. + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png -Alternatively, you can create a new project directly from within the Cypress app. + (Uploaded Cloud Artifacts) -https://on.cypress.io/cloud + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png -` +==================================================================================================== -exports['e2e record api interaction errors sendPreflight [F2] 404 status code with empty body fails without retrying 1'] = ` -We could not find a Cypress Cloud project with the projectId: pid123 + (Run Finished) -This projectId came from your cypress-with-project-id.config.js file or an environment variable. -Please log into Cypress Cloud and find your project. + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 2 1 - 1 - -We will list the correct projectId in the 'Settings' tab. -Alternatively, you can create a new project directly from within the Cypress app. +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 -https://on.cypress.io/cloud ` -exports['e2e record api interaction errors sendPreflight [F3] 201 status code with invalid decryption fails without retrying 1'] = ` -We encountered an unexpected error communicating with our servers. +exports['e2e record api interaction warnings create run warnings paid plan - over tests limit warns when over test results 1'] = ` +You've exceeded the limit of test results under your current billing plan this month. The limit is 500 test results. -DecryptionError: JWE Recipients missing or incorrect type +To continue getting the full benefits of your current plan, please visit your billing to upgrade. -Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. +https://on.cypress.io/dashboard/organizations/org-id-1234/billing -The --group flag you passed was: foo -The --ciBuildId flag you passed was: ciBuildId123 +==================================================================================================== -` + (Run Starting) -exports['e2e record api interaction errors sendPreflight [F3] 200 status code with empty body fails without retrying 1'] = ` -We encountered an unexpected error communicating with our servers. + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ -DecryptionError: General JWE must be an object -Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: record_pass.cy.js (1 of 1) + Estimated: X second(s) -The --group flag you passed was: foo -The --ciBuildId flag you passed was: ciBuildId123 -` + record pass + ✓ passes + - is pending -exports['e2e record api interaction errors sendPreflight [F4] 412 status code with valid decryption fails without retrying 1'] = ` -Recording this run failed. The request was invalid. -Recording is not working + 1 passing + 1 pending -Errors: -[ - "attempted to send invalid data" -] + (Results) -Request Sent: + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 1 │ + │ Skipped: 0 │ + │ Screenshots: 1 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: record_pass.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ -{ - "projectId": "cy12345" -} -` + (Screenshots) -exports['e2e record api interaction errors sendPreflight [W1] warning message renders preflight warning messages prior to run warnings 1'] = ` -Warning from Cypress Cloud: + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) ----------------------------------------------------------------------- -This feature will not be supported soon, please check with Cypress to learn more: https://on.cypress.io/ ----------------------------------------------------------------------- -You've exceeded the limit of private test results under your free plan this month. The limit is 500 private test results. + (Uploading Cloud Artifacts) -Your plan is now in a grace period, which means your tests will still be recorded until 2999-12-31. Please upgrade your plan to continue recording tests on Cypress Cloud in the future. + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + + (Uploaded Cloud Artifacts) + + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 2 1 - 1 - + + +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 + + +` + +exports['e2e record api interaction warnings create run warnings free plan - over tests limit v2 warns when over test results 1'] = ` +You've exceeded the limit of test results under your free billing plan this month. The limit is 500 test results. + +To continue getting the full benefits of your current plan, please visit your billing to upgrade. https://on.cypress.io/dashboard/organizations/org-id-1234/billing @@ -2599,7 +2569,7 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing │ Browser: FooBrowser 88 │ │ Specs: 1 found (record_pass.cy.js) │ │ Searched: cypress/e2e/record_pass* │ - │ Params: Tag: nightly, Group: foo, Parallel: true │ + │ Params: Tag: false, Group: false, Parallel: false │ │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ @@ -2668,64 +2638,94 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing ` -exports['e2e record api interaction errors sendPreflight [F1] socket errors fails after retrying 1'] = ` -We encountered an unexpected error communicating with our servers. +exports['e2e record api interaction warnings create run warnings unknown warning warns with unknown warning code 1'] = ` +Warning from Cypress Cloud: You are almost out of time -RequestError: Error: socket hang up +Details: -We will retry 1 more time in X second(s)... +{ + "code": "OUT_OF_TIME", + "hadTime": 1000, + "name": "OutOfTime", + "spentTime": 999 +} -We encountered an unexpected error communicating with our servers. +==================================================================================================== -RequestError: Error: socket hang up + (Run Starting) -Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ -The --group flag you passed was: foo -The --ciBuildId flag you passed was: ciBuildId123 -` +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: record_pass.cy.js (1 of 1) + Estimated: X second(s) -exports['e2e record api interaction errors sendPreflight [F3] 422 status code with invalid decryption fails without retrying 1'] = ` -We encountered an unexpected error communicating with our servers. -DecryptionError: JWE Recipients missing or incorrect type + record pass + ✓ passes + - is pending -Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. -The --group flag you passed was: foo -The --ciBuildId flag you passed was: ciBuildId123 + 1 passing + 1 pending + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 1 │ + │ Skipped: 0 │ + │ Screenshots: 1 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: record_pass.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + -` + (Screenshots) -exports['e2e record api interaction errors sendPreflight [F1] 500 status code errors with body fails after retrying 1'] = ` -We encountered an unexpected error communicating with our servers. + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) -StatusCodeError: 500 - "Internal Server Error" -We will retry 1 more time in X second(s)... + (Uploading Cloud Artifacts) -We encountered an unexpected error communicating with our servers. + - Video - Nothing to upload + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Nothing to upload - Test Replay is disabled for this project. Enable Test Replay in Cloud project settings -StatusCodeError: 500 - "Internal Server Error" + (Uploaded Cloud Artifacts) -Because you passed the --parallel flag, this run cannot proceed because it requires a valid response from our servers. + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png -The --group flag you passed was: foo -The --ciBuildId flag you passed was: ciBuildId123 +==================================================================================================== -` + (Run Finished) -exports['e2e record api interaction errors sendPreflight [F5] 422 status code with valid decryption on createRun errors and exits when group name is in use 1'] = ` -You passed the --group flag, but this group name has already been used for this run. -The existing run is: https://cloud.cypress.io/runs/12345 + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 2 1 - 1 - -The --group flag you passed was: e2e-tests -If you are trying to parallelize this run, then also pass the --parallel flag, else pass a different group name. +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 -https://on.cypress.io/run-group-name-not-unique ` @@ -2892,13 +2892,7 @@ exports['e2e record capture-protocol enabled passing retrieves the capture proto ` -exports['capture-protocol api errors fetch script 500 continues 1'] = ` -We encountered an unexpected error communicating with our servers. - -StatusCodeError: 500 - "500 - Internal Server Error" - -We will retry 1 more time in X second(s)... - +exports['e2e record capture-protocol enabled when the tab crashes in chrome posts accurate test results 1'] = ` ==================================================================================================== @@ -2907,8 +2901,8 @@ We will retry 1 more time in X second(s)... ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Cypress: 1.2.3 │ │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass.cy.js) │ - │ Searched: cypress/e2e/record_pass* │ + │ Specs: 2 found (chrome_tab_crash.cy.js, record_pass.cy.js) │ + │ Searched: cypress/e2e/chrome_tab_crash*, cypress/e2e/record_pass* │ │ Params: Tag: false, Group: false, Parallel: false │ │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ @@ -2916,7 +2910,57 @@ We will retry 1 more time in X second(s)... ──────────────────────────────────────────────────────────────────────────────────────────────────── - Running: record_pass.cy.js (1 of 1) + Running: chrome_tab_crash.cy.js (1 of 2) + Estimated: X second(s) + + + a test suite with a browser crash + ✓ navigates to about:blank + +We detected that the Chrome Renderer process just crashed. + +We have failed the current spec but will continue running the next spec. + +This can happen for a number of different reasons. + +If you're running lots of tests on a memory intense application. + - Try increasing the CPU/memory on the machine you're running on. + - Try enabling experimentalMemoryManagement in your config file. + - Try lowering numTestsKeptInMemory in your config file during 'cypress open'. + +You can learn more here: + +https://on.cypress.io/renderer-process-crashed + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: chrome_tab_crash.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - Nothing to upload + - Test Replay - 1 kB + + (Uploaded Cloud Artifacts) + + - Test Replay - Done Uploading 1 kB 1/1 + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: record_pass.cy.js (2 of 2) Estimated: X second(s) @@ -2954,11 +2998,12 @@ We will retry 1 more time in X second(s)... - Video - Nothing to upload - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - Failed Capturing - Error downloading capture code: 500 - "500 - Internal Server Error" + - Test Replay (Uploaded Cloud Artifacts) - - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Screenshot - Done Uploading 1 kB 1/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Done Uploading 1 kB 2/2 ==================================================================================================== @@ -2967,9 +3012,11 @@ We will retry 1 more time in X second(s)... Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ chrome_tab_crash.cy.js XX:XX 2 1 1 - - │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - + ✖ 1 of 2 failed (50%) XX:XX 4 2 1 1 - ─────────────────────────────────────────────────────────────────────────────────────────────────────── @@ -2979,7 +3026,7 @@ We will retry 1 more time in X second(s)... ` -exports['capture-protocol api errors error report 500 continues 1'] = ` +exports['e2e record capture-protocol enabled when there is an async error thrown from config file posts accurate test results 1'] = ` ==================================================================================================== @@ -2988,8 +3035,8 @@ exports['capture-protocol api errors error report 500 continues 1'] = ` ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Cypress: 1.2.3 │ │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass.cy.js) │ - │ Searched: cypress/e2e/record_pass* │ + │ Specs: 1 found (simple_multiple.cy.js) │ + │ Searched: cypress/e2e/simple_multiple.cy.js │ │ Params: Tag: false, Group: false, Parallel: false │ │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ @@ -2997,50 +3044,45 @@ exports['capture-protocol api errors error report 500 continues 1'] = ` ──────────────────────────────────────────────────────────────────────────────────────────────────── - Running: record_pass.cy.js (1 of 1) + Running: simple_multiple.cy.js (1 of 1) Estimated: X second(s) - record pass - ✓ passes - - is pending + suite + ✓ is true +Your configFile threw an error from: cypress-with-project-id.config.js - 1 passing - 1 pending +We stopped running your tests because your config file crashed. +Error: Async error from plugins file + [stack trace lines] (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Tests: 2 │ │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ + │ Failing: 1 │ + │ Pending: 0 │ │ Skipped: 0 │ - │ Screenshots: 1 │ + │ Screenshots: 0 │ │ Video: false │ │ Duration: X seconds │ │ Estimated: X second(s) │ - │ Spec Ran: record_pass.cy.js │ + │ Spec Ran: simple_multiple.cy.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - - (Uploading Cloud Artifacts) - Video - Nothing to upload - - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Screenshot - Nothing to upload - Test Replay - 1 kB (Uploaded Cloud Artifacts) - - Screenshot - Done Uploading 1 kB 1/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - Done Uploading 1 kB 2/2 + - Test Replay - Done Uploading 1 kB 1/1 ==================================================================================================== @@ -3049,9 +3091,9 @@ exports['capture-protocol api errors error report 500 continues 1'] = ` Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + │ ✖ simple_multiple.cy.js XX:XX 2 1 1 - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - + ✖ 1 of 1 failed (100%) XX:XX 2 1 1 - - ─────────────────────────────────────────────────────────────────────────────────────────────────────── @@ -3224,7 +3266,7 @@ exports['e2e record capture-protocol enabled protocol runtime errors error initi ` -exports['e2e record capture-protocol enabled protocol runtime errors non-fatal error encountered during protocol capture reports the error to the protocol error endpoint 1'] = ` +exports['e2e record capture-protocol enabled protocol runtime errors error in protocol beforeSpec displays the error and reports the fatal error to the cloud via artifacts 1'] = ` ==================================================================================================== @@ -3280,12 +3322,11 @@ exports['e2e record capture-protocol enabled protocol runtime errors non-fatal e - Video - Nothing to upload - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - 1 kB + - Test Replay - Failed Capturing - Error in beforeSpec (Uploaded Cloud Artifacts) - - Screenshot - Done Uploading 1 kB 1/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - Done Uploading 1 kB 2/2 + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png ==================================================================================================== @@ -3306,7 +3347,7 @@ exports['e2e record capture-protocol enabled protocol runtime errors non-fatal e ` -exports['e2e record capture-protocol enabled protocol runtime errors error in protocol beforeSpec displays the error and reports the fatal error to the cloud via artifacts 1'] = ` +exports['e2e record capture-protocol enabled protocol runtime errors error in protocol beforeTest displays the error and reports the fatal error to the cloud via artifacts 1'] = ` ==================================================================================================== @@ -3362,7 +3403,7 @@ exports['e2e record capture-protocol enabled protocol runtime errors error in pr - Video - Nothing to upload - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - Failed Capturing - Error in beforeSpec + - Test Replay - Failed Capturing - error in beforeTest (Uploaded Cloud Artifacts) @@ -3387,7 +3428,7 @@ exports['e2e record capture-protocol enabled protocol runtime errors error in pr ` -exports['e2e record capture-protocol enabled protocol runtime errors error in protocol beforeTest displays the error and reports the fatal error to the cloud via artifacts 1'] = ` +exports['e2e record capture-protocol enabled protocol runtime errors non-fatal error encountered during protocol capture reports the error to the protocol error endpoint 1'] = ` ==================================================================================================== @@ -3443,11 +3484,12 @@ exports['e2e record capture-protocol enabled protocol runtime errors error in pr - Video - Nothing to upload - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - Failed Capturing - error in beforeTest + - Test Replay - 1 kB (Uploaded Cloud Artifacts) - - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Screenshot - Done Uploading 1 kB 1/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Done Uploading 1 kB 2/2 ==================================================================================================== @@ -3468,7 +3510,7 @@ exports['e2e record capture-protocol enabled protocol runtime errors error in pr ` -exports['capture-protocol api errors upload 500 - retries 7 times and succeeds on the last call continues 1'] = ` +exports['capture-protocol api errors upload 500 - retries 8 times and fails continues 1'] = ` ==================================================================================================== @@ -3524,12 +3566,12 @@ exports['capture-protocol api errors upload 500 - retries 7 times and succeeds o - Video - Nothing to upload - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - 1 kB + - Test Replay (Uploaded Cloud Artifacts) - Screenshot - Done Uploading 1 kB 1/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - Done Uploading 1 kB 2/2 + - Test Replay - Failed Uploading 2/2 - Internal Server Error ==================================================================================================== @@ -3550,7 +3592,7 @@ exports['capture-protocol api errors upload 500 - retries 7 times and succeeds o ` -exports['capture-protocol api errors upload 500 - retries 8 times and fails continues 1'] = ` +exports['capture-protocol api errors upload 500 - retries 7 times and succeeds on the last call continues 1'] = ` ==================================================================================================== @@ -3606,12 +3648,12 @@ exports['capture-protocol api errors upload 500 - retries 8 times and fails cont - Video - Nothing to upload - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay + - Test Replay - 1 kB (Uploaded Cloud Artifacts) - Screenshot - Done Uploading 1 kB 1/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - Failed Uploading 2/2 - Internal Server Error + - Test Replay - Done Uploading 1 kB 2/2 ==================================================================================================== @@ -3632,7 +3674,13 @@ exports['capture-protocol api errors upload 500 - retries 8 times and fails cont ` -exports['e2e record capture-protocol enabled crashing does not hang when the tab crashes & continues with next spec 1'] = ` +exports['capture-protocol api errors fetch script 500 continues 1'] = ` +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 500 - "500 - Internal Server Error" + +We will retry 1 more time in X second(s)... + ==================================================================================================== @@ -3641,8 +3689,8 @@ exports['e2e record capture-protocol enabled crashing does not hang when the tab ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Cypress: 1.2.3 │ │ Browser: FooBrowser 88 │ - │ Specs: 2 found (chrome_tab_crash.cy.js, record_pass.cy.js) │ - │ Searched: cypress/e2e/chrome_tab_crash*, cypress/e2e/record_pass* │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ │ Params: Tag: false, Group: false, Parallel: false │ │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ @@ -3650,63 +3698,88 @@ exports['e2e record capture-protocol enabled crashing does not hang when the tab ──────────────────────────────────────────────────────────────────────────────────────────────────── - Running: chrome_tab_crash.cy.js (1 of 2) + Running: record_pass.cy.js (1 of 1) Estimated: X second(s) - 1) navigates to about /html - -We detected that the Chrome Renderer process just crashed. - -We have failed the current spec but will continue running the next spec. - -This can happen for a number of different reasons. + record pass + ✓ passes + - is pending -If you're running lots of tests on a memory intense application. - - Try increasing the CPU/memory on the machine you're running on. - - Try enabling experimentalMemoryManagement in your config file. - - Try lowering numTestsKeptInMemory in your config file during 'cypress open'. -You can learn more here: + 1 passing + 1 pending -https://on.cypress.io/renderer-process-crashed (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 0 │ - │ Passing: 0 │ - │ Failing: 1 │ - │ Pending: 0 │ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 1 │ │ Skipped: 0 │ │ Screenshots: 1 │ │ Video: false │ │ Duration: X seconds │ │ Estimated: X second(s) │ - │ Spec Ran: chrome_tab_crash.cy.js │ + │ Spec Ran: record_pass.cy.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ (Screenshots) - - /XXX/XXX/XXX/cypress/screenshots/chrome_tab_crash.cy.js/navigates to about html (1000x660) - (failed).png + - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) (Uploading Cloud Artifacts) - Video - Nothing to upload - - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/chrome_tab_crash.cy.js/navigates to about html (failed).png - - Test Replay - 1 kB + - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + - Test Replay - Failed Capturing - Error downloading capture code: 500 - "500 - Internal Server Error" (Uploaded Cloud Artifacts) - - Screenshot - Done Uploading 1 kB 1/2 /XXX/XXX/XXX/cypress/screenshots/chrome_tab_crash.cy.js/navigates to about html (failed).png - - Test Replay - Done Uploading 1 kB 2/2 + - Screenshot - Done Uploading 1 kB 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✔ All specs passed! XX:XX 2 1 - 1 - + + +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 + + +` + +exports['capture-protocol api errors error report 500 continues 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 1 found (record_pass.cy.js) │ + │ Searched: cypress/e2e/record_pass* │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ──────────────────────────────────────────────────────────────────────────────────────────────────── - Running: record_pass.cy.js (2 of 2) + Running: record_pass.cy.js (1 of 1) Estimated: X second(s) @@ -3744,7 +3817,7 @@ https://on.cypress.io/renderer-process-crashed - Video - Nothing to upload - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay + - Test Replay - 1 kB (Uploaded Cloud Artifacts) @@ -3758,11 +3831,9 @@ https://on.cypress.io/renderer-process-crashed Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ chrome_tab_crash.cy.js XX:XX - - 1 - - │ - ├────────────────────────────────────────────────────────────────────────────────────────────────┤ │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✖ 1 of 2 failed (50%) XX:XX 2 1 1 1 - + ✔ All specs passed! XX:XX 2 1 - 1 - ─────────────────────────────────────────────────────────────────────────────────────────────────────── diff --git a/system-tests/projects/config-with-crashing-plugin/cypress-with-project-id.config.js b/system-tests/projects/config-with-crashing-plugin/cypress-with-project-id.config.js new file mode 100644 index 000000000000..135e9da2ecc1 --- /dev/null +++ b/system-tests/projects/config-with-crashing-plugin/cypress-with-project-id.config.js @@ -0,0 +1,15 @@ +module.exports = { + 'projectId': 'pid123', + 'e2e': { + 'supportFile': false, + setupNodeEvents (on, config) { + on('before:spec', () => { + setTimeout(() => { + throw new Error('Async error from plugins file') + }, 1000) + }) + + return config + }, + }, +} diff --git a/system-tests/projects/config-with-crashing-plugin/cypress/e2e/simple_multiple.cy.js b/system-tests/projects/config-with-crashing-plugin/cypress/e2e/simple_multiple.cy.js new file mode 100644 index 000000000000..8c710948ce29 --- /dev/null +++ b/system-tests/projects/config-with-crashing-plugin/cypress/e2e/simple_multiple.cy.js @@ -0,0 +1,13 @@ +describe('suite', function () { + it('is true', () => { + expect(true).to.be.true + cy.wait(150) + }) + + it('is still true', () => { + // the config should crash before this test completes; + // this is a long wait in order to improve predictability + cy.wait(10000) + expect(true).to.be.true + }) +}) diff --git a/system-tests/projects/e2e/cypress/e2e/chrome_tab_crash.cy.js b/system-tests/projects/e2e/cypress/e2e/chrome_tab_crash.cy.js index 3fdc3b581c42..893f2a751a81 100644 --- a/system-tests/projects/e2e/cypress/e2e/chrome_tab_crash.cy.js +++ b/system-tests/projects/e2e/cypress/e2e/chrome_tab_crash.cy.js @@ -1,9 +1,10 @@ -it('navigates to about /html', () => { - cy.visit('/html') - cy.contains('Herman Melville') -}) +describe('a test suite with a browser crash', function () { + it('navigates to about:blank', () => { + cy.visit('/index.html') + }) -it('crashes the chrome tab', () => { - Cypress.automation('remote:debugger:protocol', { command: 'Page.navigate', params: { url: 'chrome://crash', transitionType: 'typed' } }) - cy.visit('localhost') + it('crashes the chrome tab', () => { + Cypress.automation('remote:debugger:protocol', { command: 'Page.navigate', params: { url: 'chrome://crash', transitionType: 'typed' } }) + cy.visit('localhost') + }) }) diff --git a/system-tests/test/record_spec.js b/system-tests/test/record_spec.js index 8e4a435a2c5b..9233b727b567 100644 --- a/system-tests/test/record_spec.js +++ b/system-tests/test/record_spec.js @@ -2308,17 +2308,56 @@ describe('e2e record', () => { }) }) - describe('crashing', () => { + describe('when the tab crashes in chrome', () => { enableCaptureProtocol() - it('does not hang when the tab crashes & continues with next spec', function () { + it('posts accurate test results', function () { return systemTests.exec(this, { - browser: 'chrome', key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', configFile: 'cypress-with-project-id.config.js', + browser: 'chrome', spec: 'chrome_tab_crash*,record_pass*', record: true, snapshot: true, expectedExitCode: 1, + }).then(() => { + const requests = getRequests() + const postResultsRequest = requests.find((r) => r.url === `POST /instances/${instanceId}/results`) + + expect(postResultsRequest.body.exception).to.include('Chrome Renderer process just crashed') + expect(postResultsRequest.body.tests).to.have.length(2) + expect(postResultsRequest.body.stats.suites).to.equal(1) + expect(postResultsRequest.body.stats.tests).to.equal(2) + expect(postResultsRequest.body.stats.passes).to.equal(1) + expect(postResultsRequest.body.stats.failures).to.equal(1) + expect(postResultsRequest.body.stats.skipped).to.equal(0) + }) + }) + }) + + describe('when there is an async error thrown from config file', () => { + enableCaptureProtocol() + it('posts accurate test results', function () { + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + browser: 'chrome', + project: 'config-with-crashing-plugin', + spec: 'simple_multiple.cy.js', + configFile: 'cypress-with-project-id.config.js', + record: true, + snapshot: true, + expectedExitCode: 1, + }).then(() => { + const requests = getRequests() + const postResultsRequest = requests.find((r) => r.url === `POST /instances/${instanceId}/results`) + + console.log(postResultsRequest) + expect(postResultsRequest?.body.exception).to.include('Your configFile threw an error') + expect(postResultsRequest?.body.tests).to.have.length(2) + expect(postResultsRequest?.body.stats.suites).to.equal(1) + expect(postResultsRequest?.body.stats.tests).to.equal(2) + expect(postResultsRequest?.body.stats.passes).to.equal(1) + expect(postResultsRequest?.body.stats.failures).to.equal(1) + expect(postResultsRequest?.body.stats.skipped).to.equal(0) }) }) })