chore: add attempt level burn in fields (#28414)

* fix(driver): add support for alias.all when using dynamic alias (#28352)

* add tests for failing instance

* fix: support for dynamic alias.all

* update changelog

* Update cli/CHANGELOG.md

Co-authored-by: Matt Schile <[email protected]>

* move test

* Update cli/CHANGELOG.md

Co-authored-by: Emily Rohrbough <[email protected]>

---------

Co-authored-by: Matt Schile <[email protected]>
Co-authored-by: Emily Rohrbough <[email protected]>

* chore: send initialStrategy and reasonToStop in postInstanceResults

* Revert "fix(driver): add support for alias.all when using dynamic alias (#28352)"

This reverts commit 02d0493.

* update record_spec snapshots

* add tests for burn-in

* add snapshots for burn in tests

* add tests for overrides and refactored assertion attempt function

* update test name

* refactor burn in types

* fix import

* add tests for failing hooks

* update e2e snapshots

* remove accidental changes

* update snapshot for record_spec

* remove reasonToStop and initialStrategy from results

* update snapshots for protocol_spec

* normalize absoluteFile path

* update reporter snapshots

* fix mocha_custom_methods

* format reporter with double quotes

* fix snapshots formatting for reporter_spec

* update formatting for burn_in_spec snapshot

* revert formatting for burn_in_spec to single quotes

---------

Co-authored-by: Jordan <[email protected]>
Co-authored-by: Matt Schile <[email protected]>
Co-authored-by: Emily Rohrbough <[email protected]>

packages/driver/src/burn-in/index.ts

@@ -1,3 +1,4 @@
+import type { AttemptStrategy, ReasonToStop } from '@packages/types'
 import { merge } from 'lodash'
 
 export type Strategy = 'detect-flake-and-pass-on-threshold' | 'detect-flake-but-always-fail' | undefined
@@ -17,19 +18,6 @@ export type CompleteBurnInConfig = {
 
 export type LatestScore = null | -2 | -1 | 0 | 1
 
-export type AttemptStrategy = 'RETRY' | 'BURN_IN' | 'NONE'
-
-export type ReasonToStop =
-| 'PASSED_FIRST_ATTEMPT' // no burn-in needed
-| 'PASSED_BURN_IN' // achieved burn-in
-| 'PASSED_MET_THRESHOLD' // passed after reaching threshold for strategy 'detect-flake-and-pass-on-threshold'
-| 'FAILED_NO_RETRIES' // failed and no retries
-| 'FAILED_REACHED_MAX_RETRIES' // failed after reaching max retries
-| 'FAILED_DID_NOT_MEET_THRESHOLD' // failed since it's impossible to meet threshold for strategy 'detect-flake-and-pass-on-threshold'
-| 'FAILED_STOPPED_ON_FLAKE' // failed with one attempt passing and using strategy 'detect-flake-but-always-fail' with `stopIfAnyPassed` set to true
-// NOTE: this is used in the mocha patch
-| 'FAILED_HOOK_FAILED' // failed because a hook failed
-
 export type EvaluateAttemptInput = {
   retriesConfig: NormalizedRetriesConfig
   burnInConfig: CompleteBurnInConfig

packages/driver/src/cypress/mocha.ts

@@ -7,8 +7,8 @@ import $stackUtils from './stack_utils'
 // in the browser mocha is coming back
 // as window
 import * as mocha from 'mocha'
-import type { AttemptStrategy, CompleteBurnInConfig, EvaluateAttemptInput, LatestScore, NormalizedRetriesConfig, ReasonToStop } from '../burn-in'
-import { evaluateAttempt, getBurnInConfig, mergeBurnInConfig } from '../burn-in'
+import { CompleteBurnInConfig, EvaluateAttemptInput, LatestScore, NormalizedRetriesConfig, evaluateAttempt, getBurnInConfig, mergeBurnInConfig } from '../burn-in'
+import type { AttemptStrategy, ReasonToStop } from '@packages/types'
 
 const { getTestFromRunnable } = $utils
 
@@ -91,7 +91,6 @@ export function calculateTestStatus (test: CypressTest, config?: NormalizedRetri
     shouldAttemptsContinue,
     attempts: totalAttemptsAlreadyExecuted,
     outerStatus: output.outerTestStatus,
-    reasonToStop: output.reasonToStop,
   }
 }
 

packages/driver/src/cypress/runner.ts

@@ -26,6 +26,7 @@ const RUNNABLE_AFTER_RUN_ASYNC_EVENT = 'runner:runnable:after:run:async'
 const RUNNABLE_LOGS = ['routes', 'agents', 'commands', 'hooks'] as const
 const RUNNABLE_PROPS = [
   '_cypressTestStatusInfo', '_testConfig', 'id', 'order', 'title', '_titlePath', 'root', 'hookName', 'hookId', 'err', 'state', 'pending', 'failedFromHookId', 'body', 'speed', 'type', 'duration', 'wallClockStartedAt', 'wallClockDuration', 'timings', 'file', 'originalTitle', 'invocationDetails', 'final', 'currentRetry', 'retries', '_slow',
+  'reasonToStop', 'thisAttemptInitialStrategy',
 ] as const
 
 const debug = debugFn('cypress:driver:runner')
@@ -940,11 +941,11 @@ const setHookFailureProps = (test, hook, err) => {
   test.duration = hook.duration // TODO: nope (?)
   test.hookName = hookName // TODO: why are we doing this?
   test.failedFromHookId = hook.hookId
+  test.reasonToStop = 'FAILED_HOOK_FAILED'
   // There should never be a case where the outerStatus of a test is set AND the last test attempt failed on a hook and the state is passed.
   // Therefore, if the last test attempt fails on a hook, the outerStatus should also indicate a failure.
   if (test?._cypressTestStatusInfo?.outerStatus) {
     test._cypressTestStatusInfo.outerStatus = test.state
-    test._cypressTestStatusInfo.reasonToStop = 'FAILED_HOOK_FAILED'
   }
 }
 
@@ -1171,13 +1172,14 @@ const _runnerListeners = (_runner, Cypress, _emissions, getTestById, getTest, se
       // as well as maybe incorrect (test passed on first attempt, but after hooks failed)
       const testStatus = test.calculateTestStatus()
 
+      runnable.reasonToStop = 'FAILED_HOOK_FAILED'
+
       runnable._cypressTestStatusInfo = {
         attempts: testStatus.attempts,
         strategy: testStatus.strategy,
         // regardless of the test state, we should ultimately fail the test here.
         outerStatus: runnable.state,
         shouldAttemptsContinue: false,
-        reasonToStop: 'FAILED_HOOK_FAILED',
       }
     }
 

packages/server/__snapshots__/reporter_spec.js

@@ -48,7 +48,9 @@ exports['lib/reporter #stats has reporterName stats, reporterStats, etc 1'] = {
           "failedFromHookId": null,
           "wallClockStartedAt": null,
           "wallClockDuration": null,
-          "videoTimestamp": null
+          "videoTimestamp": null,
+          "reasonToStop": null,
+          "initialStrategy": null
         }
       ]
     },
@@ -70,7 +72,9 @@ exports['lib/reporter #stats has reporterName stats, reporterStats, etc 1'] = {
           "failedFromHookId": null,
           "wallClockStartedAt": null,
           "wallClockDuration": null,
-          "videoTimestamp": null
+          "videoTimestamp": null,
+          "reasonToStop": null,
+          "initialStrategy": null
         }
       ]
     }

packages/server/lib/reporter.js

@@ -167,6 +167,8 @@ const toAttemptProps = (runnable) => {
     'failedFromHookId',
     'wallClockStartedAt',
     'wallClockDuration',
+    'reasonToStop',
+    'thisAttemptInitialStrategy',
   ])
 }
 
@@ -572,6 +574,8 @@ class Reporter {
           wallClockStartedAt: orNull(attempt.wallClockStartedAt && new Date(attempt.wallClockStartedAt)),
           wallClockDuration: orNull(attempt.wallClockDuration),
           videoTimestamp: null,
+          reasonToStop: orNull(attempt.reasonToStop),
+          initialStrategy: orNull(attempt.thisAttemptInitialStrategy),
         }
       }),
     }

packages/server/lib/types/reporter.ts

@@ -1,3 +1,5 @@
+import type { AttemptStrategy, ReasonToStop } from '@packages/types'
+
 interface ReporterTestAttempt {
   state: 'skipped' | 'failed' | 'passed'
   error: any
@@ -6,6 +8,8 @@ interface ReporterTestAttempt {
   wallClockStartedAt: Date
   wallClockDuration: number
   videoTimestamp: any
+  initialStrategy: AttemptStrategy
+  reasonToStop: ReasonToStop
 }
 interface ReporterTest {
   testId: string

packages/types/src/burnIn.ts

@@ -0,0 +1,11 @@
+export type AttemptStrategy = 'RETRY' | 'BURN_IN' | 'NONE'
+
+export type ReasonToStop =
+| 'PASSED_FIRST_ATTEMPT' // no burn-in needed
+| 'PASSED_BURN_IN' // achieved burn-in
+| 'PASSED_MET_THRESHOLD' // passed after reaching threshold for strategy 'detect-flake-and-pass-on-threshold'
+| 'FAILED_NO_RETRIES' // failed and no retries
+| 'FAILED_REACHED_MAX_RETRIES' // failed after reaching max retries
+| 'FAILED_DID_NOT_MEET_THRESHOLD' // failed since it's impossible to meet threshold for strategy 'detect-flake-and-pass-on-threshold'
+| 'FAILED_STOPPED_ON_FLAKE' // failed with one attempt passing and using strategy 'detect-flake-but-always-fail' with `stopIfAnyPassed` set to true
+| 'FAILED_HOOK_FAILED' // failed because a hook failed

packages/types/src/index.ts

@@ -45,3 +45,5 @@ export * from './video'
 export * from './protocol'
 
 export * from './proxy'
+
+export * from './burnIn'