diff --git a/packages/driver/src/cy/testConfigOverrides.ts b/packages/driver/src/cy/testConfigOverrides.ts index 92097243f2aa..b1009a635d9e 100644 --- a/packages/driver/src/cy/testConfigOverrides.ts +++ b/packages/driver/src/cy/testConfigOverrides.ts @@ -167,6 +167,18 @@ export class TestConfigOverride { unverifiedTestConfig: [], } + // TODO: remove when experimental overriding is supported + const experimentalRetryCfgKeys = [ + 'experimentalStrategy', 'experimentalOptions', + ] + + const experimentalRetryConfigAttempted = Object.keys(resolvedTestConfig.unverifiedTestConfig?.retries || {}) + .some((v) => experimentalRetryCfgKeys.includes(v)) + + if (experimentalRetryConfigAttempted) { + throw new Error('Cannot set experimental retries on individual tests') + } + if (Object.keys(resolvedTestConfig.unverifiedTestConfig).length > 0) { this.restoreTestConfigFn = mutateConfiguration(resolvedTestConfig, config, env) } diff --git a/system-tests/projects/experimental-retries/cypress-experimental-retries.config.js b/system-tests/projects/experimental-retries/cypress-experimental-retries.config.js new file mode 100644 index 000000000000..910f7cbbb4a4 --- /dev/null +++ b/system-tests/projects/experimental-retries/cypress-experimental-retries.config.js @@ -0,0 +1,23 @@ +module.exports = { + e2e: { + supportFile: false, + setupNodeEvents (on, config) { + // in the case the tests needed to be debugged: + + // on('before:browser:launch', (browser, launchOptions) => { + // launchOptions.args.push('--auto-open-devtools-for-tabs') + + // return launchOptions + // }) + }, + }, + retries: { + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + runMode: true, + openMode: true, + experimentalOptions: { + maxRetries: 3, + passesRequired: 1, + }, + }, +} diff --git a/system-tests/projects/experimental-retries/cypress-legacy-retries.config.js b/system-tests/projects/experimental-retries/cypress-legacy-retries.config.js new file mode 100644 index 000000000000..d5f1b46c2d38 --- /dev/null +++ b/system-tests/projects/experimental-retries/cypress-legacy-retries.config.js @@ -0,0 +1,18 @@ +module.exports = { + e2e: { + supportFile: false, + setupNodeEvents (on, config) { + // in the case the tests needed to be debugged: + + // on('before:browser:launch', (browser, launchOptions) => { + // launchOptions.args.push('--auto-open-devtools-for-tabs') + + // return launchOptions + // }) + }, + }, + retries: { + runMode: 0, + openMode: 0, + }, +} diff --git a/system-tests/projects/experimental-retries/cypress/e2e/override-with-experimental-retries.cy.js b/system-tests/projects/experimental-retries/cypress/e2e/override-with-experimental-retries.cy.js new file mode 100644 index 000000000000..bd38d32d2a16 --- /dev/null +++ b/system-tests/projects/experimental-retries/cypress/e2e/override-with-experimental-retries.cy.js @@ -0,0 +1,29 @@ +describe('overriding legacy retries with experimental retries', () => { + const experimentalStrategy = 'detect-flake-and-pass-on-threshold' + const openMode = false + const runMode = true + const maxRetries = 3 + const passesRequired = 1 + + describe('at the describe level', { + retries: { + experimentalStrategy, + openMode, + runMode, + experimentalOptions: { + maxRetries, + passesRequired, + }, + }, + }, () => { + it('sets the config', () => { + const retries = Cypress.config('retries') + + expect(retries.experimentalStrategy).to.eq(experimentalStrategy) + expect(retries.experimentalOptions?.maxRetries).to.eq(maxRetries) + expect(retries.experimentalOptions?.passesRequired).to.eq(passesRequired) + expect(retries.runMode).to.eq(runMode) + expect(retries.openMode).to.eq(openMode) + }) + }) +}) diff --git a/system-tests/projects/experimental-retries/cypress/e2e/override-with-legacy-retries.cy.js b/system-tests/projects/experimental-retries/cypress/e2e/override-with-legacy-retries.cy.js new file mode 100644 index 000000000000..c9b8c26c1363 --- /dev/null +++ b/system-tests/projects/experimental-retries/cypress/e2e/override-with-legacy-retries.cy.js @@ -0,0 +1,16 @@ +describe('overriding experimental retries with legacy retries', () => { + const openMode = 1 + const runMode = 3 + + it('sets the config', { + retries: { + openMode, + runMode, + }, + }, () => { + const retries = Cypress.config('retries') + + expect(retries.runMode).to.eq(runMode) + expect(retries.openMode).to.eq(openMode) + }) +}) diff --git a/system-tests/test/testConfigOverrides_spec.ts b/system-tests/test/testConfigOverrides_spec.ts index a97b2ab48453..bfafcdc705f4 100644 --- a/system-tests/test/testConfigOverrides_spec.ts +++ b/system-tests/test/testConfigOverrides_spec.ts @@ -83,4 +83,22 @@ describe('testConfigOverrides', () => { expectedExitCode: 1, }) }) + + describe('experimental retries specific behavior', () => { + systemTests.it('fails when attempting to set experimental retries as override', { + spec: 'override-with-experimental-retries.cy.js', + project: 'experimental-retries', + configFile: 'cypress-legacy-retries.config.js', + expectedExitCode: 1, + browser: '!webkit', + }) + + systemTests.it('succeeds when setting legacy retries as an override to experimental retries', { + spec: 'override-with-legacy-retries.cy.js', + project: 'experimental-retries', + configFile: 'cypress-experimental-retries.config.js', + expectedExitCode: 0, + browser: '!webkit', + }) + }) })