From 4a3dd4b500b0c9a9d8328e1470a4b403f528841a Mon Sep 17 00:00:00 2001
From: Cacie Prins <cacie@cypress.io>
Date: Thu, 12 Oct 2023 10:38:47 -0400
Subject: [PATCH] chore: ensure legacy retry overrides work; reject exp.
 retries overrides

---
 packages/driver/src/cy/testConfigOverrides.ts | 12 ++++++++
 .../cypress-experimental-retries.config.js    | 23 +++++++++++++++
 .../cypress-legacy-retries.config.js          | 18 ++++++++++++
 .../override-with-experimental-retries.cy.js  | 29 +++++++++++++++++++
 .../e2e/override-with-legacy-retries.cy.js    | 16 ++++++++++
 system-tests/test/testConfigOverrides_spec.ts | 18 ++++++++++++
 6 files changed, 116 insertions(+)
 create mode 100644 system-tests/projects/experimental-retries/cypress-experimental-retries.config.js
 create mode 100644 system-tests/projects/experimental-retries/cypress-legacy-retries.config.js
 create mode 100644 system-tests/projects/experimental-retries/cypress/e2e/override-with-experimental-retries.cy.js
 create mode 100644 system-tests/projects/experimental-retries/cypress/e2e/override-with-legacy-retries.cy.js

diff --git a/packages/driver/src/cy/testConfigOverrides.ts b/packages/driver/src/cy/testConfigOverrides.ts
index 92097243f2aa..b1009a635d9e 100644
--- a/packages/driver/src/cy/testConfigOverrides.ts
+++ b/packages/driver/src/cy/testConfigOverrides.ts
@@ -167,6 +167,18 @@ export class TestConfigOverride {
       unverifiedTestConfig: [],
     }
 
+    // TODO: remove when experimental overriding is supported
+    const experimentalRetryCfgKeys = [
+      'experimentalStrategy', 'experimentalOptions',
+    ]
+
+    const experimentalRetryConfigAttempted = Object.keys(resolvedTestConfig.unverifiedTestConfig?.retries || {})
+    .some((v) => experimentalRetryCfgKeys.includes(v))
+
+    if (experimentalRetryConfigAttempted) {
+      throw new Error('Cannot set experimental retries on individual tests')
+    }
+
     if (Object.keys(resolvedTestConfig.unverifiedTestConfig).length > 0) {
       this.restoreTestConfigFn = mutateConfiguration(resolvedTestConfig, config, env)
     }
diff --git a/system-tests/projects/experimental-retries/cypress-experimental-retries.config.js b/system-tests/projects/experimental-retries/cypress-experimental-retries.config.js
new file mode 100644
index 000000000000..910f7cbbb4a4
--- /dev/null
+++ b/system-tests/projects/experimental-retries/cypress-experimental-retries.config.js
@@ -0,0 +1,23 @@
+module.exports = {
+  e2e: {
+    supportFile: false,
+    setupNodeEvents (on, config) {
+      // in the case the tests needed to be debugged:
+
+      // on('before:browser:launch', (browser, launchOptions) => {
+      //   launchOptions.args.push('--auto-open-devtools-for-tabs')
+
+      //   return launchOptions
+      // })
+    },
+  },
+  retries: {
+    experimentalStrategy: 'detect-flake-and-pass-on-threshold',
+    runMode: true,
+    openMode: true,
+    experimentalOptions: {
+      maxRetries: 3,
+      passesRequired: 1,
+    },
+  },
+}
diff --git a/system-tests/projects/experimental-retries/cypress-legacy-retries.config.js b/system-tests/projects/experimental-retries/cypress-legacy-retries.config.js
new file mode 100644
index 000000000000..d5f1b46c2d38
--- /dev/null
+++ b/system-tests/projects/experimental-retries/cypress-legacy-retries.config.js
@@ -0,0 +1,18 @@
+module.exports = {
+  e2e: {
+    supportFile: false,
+    setupNodeEvents (on, config) {
+      // in the case the tests needed to be debugged:
+
+      // on('before:browser:launch', (browser, launchOptions) => {
+      //   launchOptions.args.push('--auto-open-devtools-for-tabs')
+
+      //   return launchOptions
+      // })
+    },
+  },
+  retries: {
+    runMode: 0,
+    openMode: 0,
+  },
+}
diff --git a/system-tests/projects/experimental-retries/cypress/e2e/override-with-experimental-retries.cy.js b/system-tests/projects/experimental-retries/cypress/e2e/override-with-experimental-retries.cy.js
new file mode 100644
index 000000000000..bd38d32d2a16
--- /dev/null
+++ b/system-tests/projects/experimental-retries/cypress/e2e/override-with-experimental-retries.cy.js
@@ -0,0 +1,29 @@
+describe('overriding legacy retries with experimental retries', () => {
+  const experimentalStrategy = 'detect-flake-and-pass-on-threshold'
+  const openMode = false
+  const runMode = true
+  const maxRetries = 3
+  const passesRequired = 1
+
+  describe('at the describe level', {
+    retries: {
+      experimentalStrategy,
+      openMode,
+      runMode,
+      experimentalOptions: {
+        maxRetries,
+        passesRequired,
+      },
+    },
+  }, () => {
+    it('sets the config', () => {
+      const retries = Cypress.config('retries')
+
+      expect(retries.experimentalStrategy).to.eq(experimentalStrategy)
+      expect(retries.experimentalOptions?.maxRetries).to.eq(maxRetries)
+      expect(retries.experimentalOptions?.passesRequired).to.eq(passesRequired)
+      expect(retries.runMode).to.eq(runMode)
+      expect(retries.openMode).to.eq(openMode)
+    })
+  })
+})
diff --git a/system-tests/projects/experimental-retries/cypress/e2e/override-with-legacy-retries.cy.js b/system-tests/projects/experimental-retries/cypress/e2e/override-with-legacy-retries.cy.js
new file mode 100644
index 000000000000..c9b8c26c1363
--- /dev/null
+++ b/system-tests/projects/experimental-retries/cypress/e2e/override-with-legacy-retries.cy.js
@@ -0,0 +1,16 @@
+describe('overriding experimental retries with legacy retries', () => {
+  const openMode = 1
+  const runMode = 3
+
+  it('sets the config', {
+    retries: {
+      openMode,
+      runMode,
+    },
+  }, () => {
+    const retries = Cypress.config('retries')
+
+    expect(retries.runMode).to.eq(runMode)
+    expect(retries.openMode).to.eq(openMode)
+  })
+})
diff --git a/system-tests/test/testConfigOverrides_spec.ts b/system-tests/test/testConfigOverrides_spec.ts
index a97b2ab48453..bfafcdc705f4 100644
--- a/system-tests/test/testConfigOverrides_spec.ts
+++ b/system-tests/test/testConfigOverrides_spec.ts
@@ -83,4 +83,22 @@ describe('testConfigOverrides', () => {
       expectedExitCode: 1,
     })
   })
+
+  describe('experimental retries specific behavior', () => {
+    systemTests.it('fails when attempting to set experimental retries as override', {
+      spec: 'override-with-experimental-retries.cy.js',
+      project: 'experimental-retries',
+      configFile: 'cypress-legacy-retries.config.js',
+      expectedExitCode: 1,
+      browser: '!webkit',
+    })
+
+    systemTests.it('succeeds when setting legacy retries as an override to experimental retries', {
+      spec: 'override-with-legacy-retries.cy.js',
+      project: 'experimental-retries',
+      configFile: 'cypress-experimental-retries.config.js',
+      expectedExitCode: 0,
+      browser: '!webkit',
+    })
+  })
 })