Skip to content

Commit f76fdb3

Browse files
AtofStrykerAtofStryker
committed
chore: add sanity system tests to verify console reporter output for experimental retries logic. Currently there is a bug in the reporter where the logged status doesnt wait for the aftereach to complete, which impacts the total exitCode and printed status.
1 parent 5d456e5 commit f76fdb3

File tree

6 files changed

+1986
-0
lines changed

6 files changed

+1986
-0
lines changed

‎system-tests/__snapshots__/experimental_retries.spec.ts.js

+1,685
Large diffs are not rendered by default.

‎system-tests/projects/experimental-retries/cypress.config.js

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
module.exports = {
2+
e2e: {
3+
supportFile: false,
4+
setupNodeEvents (on, config) {
5+
// in the case the tests needed to be debugged:
6+
7+
// on('before:browser:launch', (browser, launchOptions) => {
8+
// launchOptions.args.push('--auto-open-devtools-for-tabs')
9+
10+
// return launchOptions
11+
// })
12+
},
13+
},
14+
}

‎system-tests/projects/experimental-retries/cypress/e2e/always-fails.cy.js

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
describe('always fails', () => {
2+
it('always fails', function () {
3+
expect(true).to.be.false
4+
})
5+
})

‎system-tests/projects/experimental-retries/cypress/e2e/always-passes.cy.js

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
describe('always passes', () => {
2+
it('always passes', function () {
3+
expect(true).to.be.true
4+
})
5+
})

‎system-tests/projects/experimental-retries/cypress/e2e/deterministic-flaky.cy.js

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
describe('deterministic flaky test', () => {
2+
it('deterministically runs pass/fail on this test', function () {
3+
// this means the test WILL behave as
4+
// first attempt (FAIL)
5+
// second attempt (PASS)
6+
// third attempt (FAIL)
7+
// fourth attempt (PASS)
8+
// fifth attempt (FAIL)...
9+
if (this.test.currentRetry() % 2) {
10+
expect(true).to.be.true
11+
} else {
12+
expect(true).to.be.false
13+
}
14+
})
15+
})

‎system-tests/test/experimental_retries.spec.ts

+262
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,262 @@
1+
import systemTests from '../lib/system-tests'
2+
3+
describe('e2e retries.experimentalStrategy', () => {
4+
systemTests.setup()
5+
6+
describe('experimentalBurnIn=false', () => {
7+
describe('"detect-flake-and-pass-on-threshold"', () => {
8+
describe('passes', () => {
9+
systemTests.it('only runs the first attempt of the test if the test passes', {
10+
project: 'experimental-retries',
11+
browser: '!webkit',
12+
spec: 'always-passes.cy.js',
13+
snapshot: true,
14+
expectedExitCode: 0,
15+
config: {
16+
experimentalBurnIn: false,
17+
retries: {
18+
openMode: false,
19+
runMode: true,
20+
experimentalStrategy: 'detect-flake-and-pass-on-threshold',
21+
experimentalOptions: {
22+
maxRetries: 10,
23+
passesRequired: 3,
24+
},
25+
},
26+
screenshotOnRunFailure: false,
27+
},
28+
})
29+
30+
systemTests.it('retries up to the "passesRequired" limit if the config can be satisfied', {
31+
project: 'experimental-retries',
32+
browser: '!webkit',
33+
spec: 'deterministic-flaky.cy.js',
34+
snapshot: true,
35+
expectedExitCode: 0,
36+
config: {
37+
experimentalBurnIn: false,
38+
retries: {
39+
openMode: false,
40+
runMode: true,
41+
experimentalStrategy: 'detect-flake-and-pass-on-threshold',
42+
experimentalOptions: {
43+
maxRetries: 9,
44+
passesRequired: 3,
45+
},
46+
},
47+
screenshotOnRunFailure: false,
48+
},
49+
})
50+
51+
systemTests.it('retries up to the "passesRequired" limit if the config can be satisfied (max attempts)', {
52+
project: 'experimental-retries',
53+
browser: '!webkit',
54+
spec: 'deterministic-flaky.cy.js',
55+
snapshot: true,
56+
expectedExitCode: 0,
57+
config: {
58+
experimentalBurnIn: false,
59+
retries: {
60+
openMode: false,
61+
runMode: true,
62+
experimentalStrategy: 'detect-flake-and-pass-on-threshold',
63+
experimentalOptions: {
64+
maxRetries: 9,
65+
passesRequired: 5,
66+
},
67+
},
68+
screenshotOnRunFailure: false,
69+
},
70+
})
71+
})
72+
73+
describe('fails', () => {
74+
systemTests.it('short-circuits if the needed "passesRequired" cannot be satisfied by the remaining attempts available', {
75+
project: 'experimental-retries',
76+
browser: '!webkit',
77+
spec: 'deterministic-flaky.cy.js',
78+
snapshot: true,
79+
expectedExitCode: 1,
80+
config: {
81+
experimentalBurnIn: false,
82+
retries: {
83+
openMode: false,
84+
runMode: true,
85+
experimentalStrategy: 'detect-flake-and-pass-on-threshold',
86+
experimentalOptions: {
87+
maxRetries: 5,
88+
passesRequired: 5,
89+
},
90+
},
91+
screenshotOnRunFailure: false,
92+
},
93+
})
94+
95+
systemTests.it('retries up to the "passesRequired" limit if the config can be satisfied (max attempts possible)', {
96+
project: 'experimental-retries',
97+
browser: '!webkit',
98+
spec: 'deterministic-flaky.cy.js',
99+
snapshot: true,
100+
expectedExitCode: 1,
101+
config: {
102+
experimentalBurnIn: false,
103+
retries: {
104+
openMode: false,
105+
runMode: true,
106+
experimentalStrategy: 'detect-flake-and-pass-on-threshold',
107+
experimentalOptions: {
108+
maxRetries: 6,
109+
passesRequired: 4,
110+
},
111+
},
112+
screenshotOnRunFailure: false,
113+
},
114+
})
115+
})
116+
117+
/**
118+
* exercised additionally in cy-in-cy tests to verify correct mocha snapshots and cypress reporter output:
119+
* packages/app/cypress/e2e/runner/retries.experimentalRetries.mochaEvents.cy.ts
120+
* packages/app/cypress/e2e/runner/runner.experimentalRetries.mochaEvents.cy.ts
121+
*/
122+
systemTests.it('exercises experimental-retries suite to verify console reporter and final status code are correct.', {
123+
project: 'detect-flake-and-pass-on-threshold',
124+
browser: '!webkit',
125+
spec: 'runner/experimental-retries/*',
126+
snapshot: true,
127+
expectedExitCode: 2,
128+
config: {
129+
experimentalBurnIn: false,
130+
retries: {
131+
openMode: false,
132+
runMode: true,
133+
experimentalStrategy: 'detect-flake-and-pass-on-threshold',
134+
experimentalOptions: {
135+
maxRetries: 9,
136+
passesRequired: 5,
137+
},
138+
},
139+
screenshotOnRunFailure: false,
140+
},
141+
})
142+
})
143+
144+
describe('"detect-flake-but-always-fail"', () => {
145+
describe('passes', () => {
146+
systemTests.it('only runs the first attempt of the test if the test passes', {
147+
project: 'experimental-retries',
148+
browser: '!webkit',
149+
spec: 'always-passes.cy.js',
150+
snapshot: true,
151+
expectedExitCode: 0,
152+
config: {
153+
experimentalBurnIn: false,
154+
retries: {
155+
openMode: false,
156+
runMode: true,
157+
experimentalStrategy: 'detect-flake-but-always-fail',
158+
experimentalOptions: {
159+
maxRetries: 9,
160+
stopIfAnyPassed: false,
161+
},
162+
},
163+
screenshotOnRunFailure: false,
164+
},
165+
})
166+
})
167+
168+
describe('fails', () => {
169+
systemTests.it('runs all attempts of the test if the first attempt fails and "stopIfAnyPassed=false"', {
170+
project: 'experimental-retries',
171+
browser: '!webkit',
172+
spec: 'deterministic-flaky.cy.js',
173+
snapshot: true,
174+
expectedExitCode: 1,
175+
config: {
176+
experimentalBurnIn: false,
177+
retries: {
178+
openMode: false,
179+
runMode: true,
180+
experimentalStrategy: 'detect-flake-but-always-fail',
181+
experimentalOptions: {
182+
maxRetries: 9,
183+
stopIfAnyPassed: false,
184+
},
185+
},
186+
screenshotOnRunFailure: false,
187+
},
188+
})
189+
190+
systemTests.it('runs attempts of the test if the first attempt fails until the test passes if "stopIfAnyPassed=true"', {
191+
project: 'experimental-retries',
192+
browser: '!webkit',
193+
spec: 'deterministic-flaky.cy.js',
194+
snapshot: true,
195+
expectedExitCode: 1,
196+
config: {
197+
experimentalBurnIn: false,
198+
retries: {
199+
openMode: false,
200+
runMode: true,
201+
experimentalStrategy: 'detect-flake-but-always-fail',
202+
experimentalOptions: {
203+
maxRetries: 9,
204+
stopIfAnyPassed: true,
205+
},
206+
},
207+
screenshotOnRunFailure: false,
208+
},
209+
})
210+
})
211+
212+
/**
213+
* exercised additionally in cy-in-cy tests to verify correct mocha snapshots and cypress reporter output:
214+
* packages/app/cypress/e2e/runner/retries.experimentalRetries.mochaEvents.cy.ts
215+
* packages/app/cypress/e2e/runner/runner.experimentalRetries.mochaEvents.cy.ts
216+
*/
217+
systemTests.it('exercises experimental-retries suite to verify console reporter and final status code are correct.', {
218+
project: 'detect-flake-but-always-fail',
219+
browser: '!webkit',
220+
spec: 'runner/experimental-retries/*',
221+
snapshot: true,
222+
// FIXME: this should be 8
223+
expectedExitCode: 9,
224+
config: {
225+
experimentalBurnIn: false,
226+
retries: {
227+
openMode: false,
228+
runMode: true,
229+
experimentalStrategy: 'detect-flake-but-always-fail',
230+
experimentalOptions: {
231+
maxRetries: 9,
232+
stopIfAnyPassed: false,
233+
},
234+
},
235+
screenshotOnRunFailure: false,
236+
},
237+
})
238+
239+
systemTests.it('exercises experimental-retries suite to verify console reporter and final status code are correct (stopIfAnyPassed=true).', {
240+
project: 'detect-flake-but-always-fail-stop-any-passed',
241+
browser: '!webkit',
242+
spec: 'runner/experimental-retries/*',
243+
snapshot: true,
244+
// FIXME: this should be 8
245+
expectedExitCode: 9,
246+
config: {
247+
experimentalBurnIn: false,
248+
retries: {
249+
openMode: false,
250+
runMode: true,
251+
experimentalStrategy: 'detect-flake-but-always-fail',
252+
experimentalOptions: {
253+
maxRetries: 9,
254+
stopIfAnyPassed: true,
255+
},
256+
},
257+
screenshotOnRunFailure: false,
258+
},
259+
})
260+
})
261+
})
262+
})

0 commit comments

Comments
 (0)