Skip to content

Commit

Permalink
Fixed some unit tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@MuazOthman
MuazOthman committed Oct 11, 2023
1 parent 0fecba0 commit 1b15b2a
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 25 deletions.
48 changes: 32 additions & 16 deletions packages/driver/cypress/e2e/util/mocha_custom_methods.cy.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ describe('mocha custom methods', () => {
expect(undefinedStrategyTest.final).to.be.true

const detectFlakeAndPassOnThresholdStrategyTest = createMockTest()
const detectFlakeAndPassOnThreshold = calculateTestStatus(detectFlakeAndPassOnThresholdStrategyTest, 'detect-flake-and-pass-on-threshold', {
const detectFlakeAndPassOnThreshold = calculateTestStatus(detectFlakeAndPassOnThresholdStrategyTest, {
strategy: 'detect-flake-and-pass-on-threshold',
maxRetries: 8,
passesRequired: 5,
})
Expand All @@ -41,7 +42,8 @@ describe('mocha custom methods', () => {
expect(detectFlakeAndPassOnThresholdStrategyTest.final).to.be.true

const detectFlakeButAlwaysFailStrategyTest = createMockTest()
const detectFlakeButAlwaysFail = calculateTestStatus(detectFlakeButAlwaysFailStrategyTest, 'detect-flake-but-always-fail', {
const detectFlakeButAlwaysFail = calculateTestStatus(detectFlakeButAlwaysFailStrategyTest, {
strategy: 'detect-flake-but-always-fail',
maxRetries: 8,
stopIfAnyPassed: false,
})
Expand Down Expand Up @@ -105,7 +107,8 @@ describe('mocha custom methods', () => {
it('passed: no longer signals to retry test after passesRequired threshold is reached', function () {
totalRetries = 5
const mockTest1 = createMockTest('failed')
const attempt1 = calculateTestStatus(mockTest1, 'detect-flake-and-pass-on-threshold', {
const attempt1 = calculateTestStatus(mockTest1, {
strategy: 'detect-flake-and-pass-on-threshold',
maxRetries: totalRetries,
passesRequired: 2,
})
Expand All @@ -117,7 +120,8 @@ describe('mocha custom methods', () => {
expect(mockTest1.final).to.be.false

const mockTest2 = createMockTest('failed', [mockTest1])
const attempt2 = calculateTestStatus(mockTest2, 'detect-flake-and-pass-on-threshold', {
const attempt2 = calculateTestStatus(mockTest2, {
strategy: 'detect-flake-and-pass-on-threshold',
maxRetries: totalRetries,
passesRequired: 2,
})
Expand All @@ -129,7 +133,8 @@ describe('mocha custom methods', () => {
expect(mockTest2.final).to.be.false

const mockTest3 = createMockTest('passed', [mockTest1, mockTest2])
const attempt3 = calculateTestStatus(mockTest3, 'detect-flake-and-pass-on-threshold', {
const attempt3 = calculateTestStatus(mockTest3, {
strategy: 'detect-flake-and-pass-on-threshold',
maxRetries: totalRetries,
passesRequired: 2,
})
Expand All @@ -141,7 +146,8 @@ describe('mocha custom methods', () => {
expect(mockTest3.final).to.be.false

const mockTest4 = createMockTest('passed', [mockTest1, mockTest2, mockTest3])
const attempt4 = calculateTestStatus(mockTest4, 'detect-flake-and-pass-on-threshold', {
const attempt4 = calculateTestStatus(mockTest4, {
strategy: 'detect-flake-and-pass-on-threshold',
maxRetries: totalRetries,
passesRequired: 2,
})
Expand All @@ -156,7 +162,8 @@ describe('mocha custom methods', () => {
it('failed: no longer signals to retry test if the passesRequired is impossible to meet', function () {
totalRetries = 4
const mockTest1 = createMockTest('failed')
const attempt1 = calculateTestStatus(mockTest1, 'detect-flake-and-pass-on-threshold', {
const attempt1 = calculateTestStatus(mockTest1, {
strategy: 'detect-flake-and-pass-on-threshold',
maxRetries: totalRetries,
passesRequired: 2,
})
Expand All @@ -168,7 +175,8 @@ describe('mocha custom methods', () => {
expect(mockTest1.final).to.be.false

const mockTest2 = createMockTest('failed', [mockTest1])
const attempt2 = calculateTestStatus(mockTest2, 'detect-flake-and-pass-on-threshold', {
const attempt2 = calculateTestStatus(mockTest2, {
strategy: 'detect-flake-and-pass-on-threshold',
maxRetries: totalRetries,
passesRequired: 2,
})
Expand All @@ -180,7 +188,8 @@ describe('mocha custom methods', () => {
expect(mockTest2.final).to.be.false

const mockTest3 = createMockTest('failed', [mockTest1, mockTest2])
const attempt3 = calculateTestStatus(mockTest3, 'detect-flake-and-pass-on-threshold', {
const attempt3 = calculateTestStatus(mockTest3, {
strategy: 'detect-flake-and-pass-on-threshold',
maxRetries: totalRetries,
passesRequired: 2,
})
Expand All @@ -192,7 +201,8 @@ describe('mocha custom methods', () => {
expect(mockTest3.final).to.be.false

const mockTest4 = createMockTest('failed', [mockTest1, mockTest2, mockTest3])
const attempt4 = calculateTestStatus(mockTest4, 'detect-flake-and-pass-on-threshold', {
const attempt4 = calculateTestStatus(mockTest4, {
strategy: 'detect-flake-and-pass-on-threshold',
maxRetries: totalRetries,
passesRequired: 2,
})
Expand All @@ -209,7 +219,8 @@ describe('mocha custom methods', () => {
it('failed: no longer signals to retry after retries are exhausted', function () {
totalRetries = 3
const mockTest1 = createMockTest('failed')
const attempt1 = calculateTestStatus(mockTest1, 'detect-flake-but-always-fail', {
const attempt1 = calculateTestStatus(mockTest1, {
strategy: 'detect-flake-but-always-fail',
maxRetries: totalRetries,
stopIfAnyPassed: false,
})
Expand All @@ -221,7 +232,8 @@ describe('mocha custom methods', () => {
expect(mockTest1.final).to.be.false

const mockTest2 = createMockTest('failed', [mockTest1])
const attempt2 = calculateTestStatus(mockTest2, 'detect-flake-but-always-fail', {
const attempt2 = calculateTestStatus(mockTest2, {
strategy: 'detect-flake-but-always-fail',
maxRetries: totalRetries,
stopIfAnyPassed: false,
})
Expand All @@ -233,7 +245,8 @@ describe('mocha custom methods', () => {
expect(mockTest2.final).to.be.false

const mockTest3 = createMockTest('passed', [mockTest1, mockTest2])
const attempt3 = calculateTestStatus(mockTest3, 'detect-flake-but-always-fail', {
const attempt3 = calculateTestStatus(mockTest3, {
strategy: 'detect-flake-but-always-fail',
maxRetries: totalRetries,
stopIfAnyPassed: false,
})
Expand All @@ -245,7 +258,8 @@ describe('mocha custom methods', () => {
expect(mockTest3.final).to.be.false

const mockTest4 = createMockTest('passed', [mockTest1, mockTest2, mockTest3])
const attempt4 = calculateTestStatus(mockTest4, 'detect-flake-but-always-fail', {
const attempt4 = calculateTestStatus(mockTest4, {
strategy: 'detect-flake-but-always-fail',
maxRetries: totalRetries,
stopIfAnyPassed: false,
})
Expand All @@ -262,7 +276,8 @@ describe('mocha custom methods', () => {
it('failed: short circuits after a retry has a passed test', function () {
totalRetries = 3
const mockTest1 = createMockTest('failed')
const attempt1 = calculateTestStatus(mockTest1, 'detect-flake-but-always-fail', {
const attempt1 = calculateTestStatus(mockTest1, {
strategy: 'detect-flake-but-always-fail',
maxRetries: totalRetries,
stopIfAnyPassed: true,
})
Expand All @@ -274,7 +289,8 @@ describe('mocha custom methods', () => {
expect(mockTest1.final).to.be.false

const mockTest2 = createMockTest('passed', [mockTest1])
const attempt2 = calculateTestStatus(mockTest2, 'detect-flake-but-always-fail', {
const attempt2 = calculateTestStatus(mockTest2, {
strategy: 'detect-flake-but-always-fail',
maxRetries: totalRetries,
stopIfAnyPassed: true,
})
Expand Down
18 changes: 9 additions & 9 deletions packages/driver/src/cypress/mocha.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ type NormalizedRetriesConfig = {
}

// NOTE: 'calculateTestStatus' is marked as an individual function to make functionality easier to test.
export function calculateTestStatus (test: CypressTest, config: NormalizedRetriesConfig) {
export function calculateTestStatus (test: CypressTest, config?: NormalizedRetriesConfig) {
// @ts-expect-error
const totalAttemptsAlreadyExecuted = test.currentRetry() + 1
let shouldAttemptsContinue: boolean = true
Expand All @@ -78,21 +78,21 @@ export function calculateTestStatus (test: CypressTest, config: NormalizedRetrie
const passingAttempts = passedTests.length

// Below variables are used for when strategy is "detect-flake-and-pass-on-threshold" or no strategy is defined
let passesRequired = config.strategy !== 'detect-flake-but-always-fail' ?
(config.passesRequired || 1) :
let passesRequired = config?.strategy !== 'detect-flake-but-always-fail' ?
(config?.passesRequired || 1) :
null

const neededPassingAttemptsLeft = config.strategy !== 'detect-flake-but-always-fail' ?
const neededPassingAttemptsLeft = config?.strategy !== 'detect-flake-but-always-fail' ?
(passesRequired as number) - passingAttempts :
null

// Below variables are used for when strategy is only "detect-flake-but-always-fail"
let stopIfAnyPassed = config.strategy === 'detect-flake-but-always-fail' ?
let stopIfAnyPassed = config?.strategy === 'detect-flake-but-always-fail' ?
(config.stopIfAnyPassed || false) :
null

// Do we have the required amount of passes? If yes, we no longer need to keep running the test.
if (config.strategy !== 'detect-flake-but-always-fail' && passingAttempts >= (passesRequired as number)) {
if (config?.strategy !== 'detect-flake-but-always-fail' && passingAttempts >= (passesRequired as number)) {
outerTestStatus = 'passed'
test.final = true
shouldAttemptsContinue = false
Expand All @@ -101,13 +101,13 @@ export function calculateTestStatus (test: CypressTest, config: NormalizedRetrie
// For strategy "detect-flake-and-pass-on-threshold" or no strategy (current GA retries):
// If we haven't met our max attempt limit AND we have enough remaining attempts that can satisfy the passing requirement.
// retry the test.
(config.strategy !== 'detect-flake-but-always-fail' && remainingAttempts >= (neededPassingAttemptsLeft as number)) ||
(config?.strategy !== 'detect-flake-but-always-fail' && remainingAttempts >= (neededPassingAttemptsLeft as number)) ||
// For strategy "detect-flake-but-always-fail":
// If we haven't met our max attempt limit AND
// stopIfAnyPassed is false OR
// stopIfAnyPassed is true and no tests have passed yet.
// retry the test.
(config.strategy === 'detect-flake-but-always-fail' && (!stopIfAnyPassed || stopIfAnyPassed && passingAttempts === 0))
(config?.strategy === 'detect-flake-but-always-fail' && (!stopIfAnyPassed || stopIfAnyPassed && passingAttempts === 0))
)) {
test.final = false
shouldAttemptsContinue = true
Expand All @@ -128,7 +128,7 @@ export function calculateTestStatus (test: CypressTest, config: NormalizedRetrie
}

return {
strategy: config.strategy,
strategy: config?.strategy,
shouldAttemptsContinue,
attempts: totalAttemptsAlreadyExecuted,
outerStatus: outerTestStatus,
Expand Down

4 comments on commit 1b15b2a

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 1b15b2a Oct 11, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the linux arm64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/13.4.0/linux-arm64/feature/experimental-retries-1b15b2a47ea10e51a5fc567732c58589a93e9c0c/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 1b15b2a Oct 11, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the linux x64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/13.4.0/linux-x64/feature/experimental-retries-1b15b2a47ea10e51a5fc567732c58589a93e9c0c/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 1b15b2a Oct 11, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the darwin x64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/13.4.0/darwin-x64/feature/experimental-retries-1b15b2a47ea10e51a5fc567732c58589a93e9c0c/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on 1b15b2a Oct 11, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the win32 x64 version of the Test Runner.

Learn more about this pre-release build at https://on.cypress.io/advanced-installation#Install-pre-release-version

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/13.4.0/win32-x64/feature/experimental-retries-1b15b2a47ea10e51a5fc567732c58589a93e9c0c/cypress.tgz

Please sign in to comment.