chore: ensure legacy retry overrides work; reject exp. retries overri…

…des (#28045)

packages/driver/src/cy/testConfigOverrides.ts

@@ -167,6 +167,18 @@ export class TestConfigOverride {
       unverifiedTestConfig: [],
     }
 
+    // TODO: remove when experimental overriding is supported
+    const experimentalRetryCfgKeys = [
+      'experimentalStrategy', 'experimentalOptions',
+    ]
+
+    const experimentalRetryConfigAttempted = Object.keys(resolvedTestConfig.unverifiedTestConfig?.retries || {})
+    .some((v) => experimentalRetryCfgKeys.includes(v))
+
+    if (experimentalRetryConfigAttempted) {
+      throw new Error('Cannot set experimental retries on individual tests')
+    }
+
     if (Object.keys(resolvedTestConfig.unverifiedTestConfig).length > 0) {
       this.restoreTestConfigFn = mutateConfiguration(resolvedTestConfig, config, env)
     }

system-tests/projects/experimental-retries/cypress-experimental-retries.config.js

@@ -0,0 +1,23 @@
+module.exports = {
+  e2e: {
+    supportFile: false,
+    setupNodeEvents (on, config) {
+      // in the case the tests needed to be debugged:
+
+      // on('before:browser:launch', (browser, launchOptions) => {
+      //   launchOptions.args.push('--auto-open-devtools-for-tabs')
+
+      //   return launchOptions
+      // })
+    },
+  },
+  retries: {
+    experimentalStrategy: 'detect-flake-and-pass-on-threshold',
+    runMode: true,
+    openMode: true,
+    experimentalOptions: {
+      maxRetries: 3,
+      passesRequired: 1,
+    },
+  },
+}

system-tests/projects/experimental-retries/cypress-legacy-retries.config.js

@@ -0,0 +1,18 @@
+module.exports = {
+  e2e: {
+    supportFile: false,
+    setupNodeEvents (on, config) {
+      // in the case the tests needed to be debugged:
+
+      // on('before:browser:launch', (browser, launchOptions) => {
+      //   launchOptions.args.push('--auto-open-devtools-for-tabs')
+
+      //   return launchOptions
+      // })
+    },
+  },
+  retries: {
+    runMode: 0,
+    openMode: 0,
+  },
+}

system-tests/projects/experimental-retries/cypress/e2e/override-with-experimental-retries.cy.js

@@ -0,0 +1,29 @@
+describe('overriding legacy retries with experimental retries', () => {
+  const experimentalStrategy = 'detect-flake-and-pass-on-threshold'
+  const openMode = false
+  const runMode = true
+  const maxRetries = 3
+  const passesRequired = 1
+
+  describe('at the describe level', {
+    retries: {
+      experimentalStrategy,
+      openMode,
+      runMode,
+      experimentalOptions: {
+        maxRetries,
+        passesRequired,
+      },
+    },
+  }, () => {
+    it('sets the config', () => {
+      const retries = Cypress.config('retries')
+
+      expect(retries.experimentalStrategy).to.eq(experimentalStrategy)
+      expect(retries.experimentalOptions?.maxRetries).to.eq(maxRetries)
+      expect(retries.experimentalOptions?.passesRequired).to.eq(passesRequired)
+      expect(retries.runMode).to.eq(runMode)
+      expect(retries.openMode).to.eq(openMode)
+    })
+  })
+})

system-tests/projects/experimental-retries/cypress/e2e/override-with-legacy-retries.cy.js

@@ -0,0 +1,16 @@
+describe('overriding experimental retries with legacy retries', () => {
+  const openMode = 1
+  const runMode = 3
+
+  it('sets the config', {
+    retries: {
+      openMode,
+      runMode,
+    },
+  }, () => {
+    const retries = Cypress.config('retries')
+
+    expect(retries.runMode).to.eq(runMode)
+    expect(retries.openMode).to.eq(openMode)
+  })
+})

system-tests/test/testConfigOverrides_spec.ts

@@ -83,4 +83,22 @@ describe('testConfigOverrides', () => {
       expectedExitCode: 1,
     })
   })
+
+  describe('experimental retries specific behavior', () => {
+    systemTests.it('fails when attempting to set experimental retries as override', {
+      spec: 'override-with-experimental-retries.cy.js',
+      project: 'experimental-retries',
+      configFile: 'cypress-legacy-retries.config.js',
+      expectedExitCode: 1,
+      browser: '!webkit',
+    })
+
+    systemTests.it('succeeds when setting legacy retries as an override to experimental retries', {
+      spec: 'override-with-legacy-retries.cy.js',
+      project: 'experimental-retries',
+      configFile: 'cypress-experimental-retries.config.js',
+      expectedExitCode: 0,
+      browser: '!webkit',
+    })
+  })
 })