test(e2e): restore report generation and fix location of received images (#3145)

When an test failure occurred, the report was not generated and the
development server did not stop at the end of the test execution.
This was due to our custom function `toMatchImageSnapshot` becoming
asynchronous in commit bffd25b.
The original function is synchronous, so the custom function must be
too.

In addition to this fix, the logic of the code managing image
attachments has been improved, mainly to better separate
responsibilities.
Storing the expected image copy and calculating the image location are
specific to j-i-s, so there's no need to do this. to j-i-s, and are
therefore performed first.
The function for attaching images to the report only uses the previously
calculated image location and is unaware that j-i-s is being used.

For tests that redefined the location of output images (usually in a
subfolder), the path to the received images was not updated accordingly.
It was stored in the same location for each test, so only one file was
kept and not stored with the diff and the expected image.

Also regenerate the image for the
“subprocess.03.collapsed.with.elements” test.
- Previously, the test failed when run with firefox and this was for
good reasons that were hidden with other browsers due to the threshold
value used.
- The sub-process on the top left is a "collapsed adhoc sub-process", so
2 markers must be displayed with a space between them. Currently, the
adhoc marker is not displayed, but the expand marker is correctly
positioned (not centered, but on the left) as if the adhoc marker were
displayed.
- Commit c0c50aa removed additional spacing between markers, so the
expand marker has moved to the right. But the reference snapshot was not
updated accordingly.
- The test probably accepted a minor visual change like this, but at
some point the new version of Firefox added other changes that are above
the threshold, and the error now appears.
- The problem didn't occur on macOS because the threshold was increased
in commit bfa02fc, but this hid the problem.

Author: tbouffard

test/config/jest.image.ts

@@ -25,7 +25,7 @@ import { toMatchImageSnapshot } from 'jest-image-snapshot';
 import MatcherContext = jest.MatcherContext;
 import CustomMatcherResult = jest.CustomMatcherResult;
 
-const jestLog = debugLogger('bv:test:jest:img');
+const log = debugLogger('bv:test:jest:img');
 const toMatchImageSnapshotWithRealSignature = toMatchImageSnapshot as (received: unknown, options?: MatchImageSnapshotOptions) => CustomMatcherResult;
 
 // The path is relative from the jest-html-reporters page to the folder storing the images
@@ -59,78 +59,99 @@ class RetriesCounter {
 
 const retriesCounter = new RetriesCounter();
 
-async function saveAndRegisterImages(matcherContext: MatcherContext, options: MatchImageSnapshotOptions): Promise<void> {
-  const snapshotIdentifier = options.customSnapshotIdentifier as string;
-  // Manage expected and received images
-  const baseImagePathWithName = `${options.customDiffDir}/${snapshotIdentifier}`;
-  const expectedImagePath = `${baseImagePathWithName}-expected.png`;
-  copyFileSync(`${options.customSnapshotsDir}/${snapshotIdentifier}.png`, expectedImagePath);
-  // this image is generated by jest-image-snapshot when activating `storeReceivedOnFailure`
-  const receivedImagePath = `${baseImagePathWithName}-received.png`;
-
+async function attachImagesForReport(images: LocationOfImagesForTestReport, matcherContext: MatcherContext): Promise<void> {
   // Attach the images to jest-html-reports
   // Chain the calls to preserve the attachment order
   // Create a custom context as the async call can be done whereas the global jest context has changed (another test is currently running).
   // So the test name and path changed, and the images would be attached to the wrong test.
   // For the context object structure, see https://github.com/Hazyzh/jest-html-reporters/blob/v3.0.5/helper.ts#L95
   const context: Record<symbol, unknown> = {};
+  const currentTestName = matcherContext.currentTestName;
+  log('Attaching images to report for test %s', currentTestName);
   context[Symbol('bpmn-visualization')] = {
     state: {
-      currentTestName: matcherContext.currentTestName,
+      currentTestName,
       testPath: matcherContext.testPath,
     },
     matchers: {}, // required by the jest-html-reporters getJestGlobalData function even if not used
   };
 
   try {
     await addAttach({
-      attach: computeRelativePathFromReportToSnapshots(`${baseImagePathWithName}-diff.png`),
+      attach: computeRelativePathFromReportToSnapshots(images.diff),
       description: 'diff',
       bufferFormat: 'png',
       context,
     });
 
     await addAttach({
-      attach: computeRelativePathFromReportToSnapshots(expectedImagePath),
+      attach: computeRelativePathFromReportToSnapshots(images.expected),
       description: 'expected',
       bufferFormat: 'png',
       context,
     });
 
     await addAttach({
-      attach: computeRelativePathFromReportToSnapshots(receivedImagePath),
+      attach: computeRelativePathFromReportToSnapshots(images.received),
       description: 'received',
       bufferFormat: 'png',
       context,
     });
+
+    log('Images attached to report for test %s', currentTestName);
   } catch (error) {
     console.error(
-      `Error while attaching images to test ${snapshotIdentifier}.` +
+      `Error while attaching images to test ${currentTestName}.` +
         `The 'jest-html-reporters' reporter is probably not in use. For instance, this occurs when running tests with the IntelliJ/Webstorm Jest runner.`,
       error,
     );
   }
 }
 
+type LocationOfImagesForTestReport = {
+  diff: string;
+  expected: string;
+  received: string;
+};
+
+function saveImages(options: MatchImageSnapshotOptions): LocationOfImagesForTestReport {
+  const snapshotIdentifier = options.customSnapshotIdentifier as string;
+  const baseImagePathWithName = `${options.customDiffDir}/${snapshotIdentifier}`;
+
+  const diffImagePath = `${baseImagePathWithName}-diff.png`;
+  const expectedImagePath = `${baseImagePathWithName}-expected.png`;
+  copyFileSync(`${options.customSnapshotsDir}/${snapshotIdentifier}.png`, expectedImagePath);
+  // this image is generated by jest-image-snapshot when activating `storeReceivedOnFailure`
+  const receivedImagePath = `${options.customReceivedDir}/${snapshotIdentifier}-received.png`;
+
+  return {
+    diff: diffImagePath,
+    expected: expectedImagePath,
+    received: receivedImagePath,
+  };
+}
+
 // Improve jest-image-snapshot outputs to facilitate debug
 // The 'options' parameter is mandatory for us, and some properties must be set as well
 // All options properties used here are always set in bpmn-visualization tests
 // If the following implementation would be done directly in jest-image-snapshot, this won't be required as it set default values we cannot access here
-async function toMatchImageSnapshotCustom(this: MatcherContext, received: Buffer, options: MatchImageSnapshotOptions): Promise<CustomMatcherResult> {
+function toMatchImageSnapshotCustom(this: MatcherContext, received: Buffer, options: MatchImageSnapshotOptions): CustomMatcherResult {
   const testId = this.currentTestName;
   retriesCounter.incrementExecutionCount(testId);
-  jestLog("Test: '%s' (test file path: '%s')", this.currentTestName, this.testPath);
+  log("Test: '%s' (test file path: '%s')", this.currentTestName, this.testPath);
   const executionCount = retriesCounter.getExecutionCount(testId);
-  jestLog('Ready to execute toMatchImageSnapshot, execution count: %s', executionCount);
+  log('Ready to execute toMatchImageSnapshot, execution count: %s', executionCount);
   const result = toMatchImageSnapshotWithRealSignature.call(this, received, options);
-  jestLog('toMatchImageSnapshot executed');
+  log('toMatchImageSnapshot executed');
 
   if (result.pass) {
-    jestLog('Result: success');
+    log('Result: success');
   } else {
-    jestLog('Result: failure');
+    log('Result: failure');
     if (retriesCounter.hasReachMaxRetries(testId)) {
-      await saveAndRegisterImages(this, options);
+      const imageLocationInformation = saveImages(options);
+      log('Images saved and ready to be registered to the report', imageLocationInformation);
+      attachImagesForReport(imageLocationInformation, this).catch(error => log('Fail to attach images to the report %s', error));
     }
 
     // Add configured failure threshold in the error message

test/e2e/__image_snapshots__/bpmn-rendering/subprocess.03.collapsed.with.elements.png

@@ -165,12 +165,6 @@ class ImageSnapshotThresholds extends MultiBrowserImageSnapshotThresholds {
           windows: 0.66 / 100, // 0.6566433292574891%
         },
       ],
-      [
-        'subprocess.03.collapsed.with.elements',
-        {
-          macos: 0.11 / 100, // 0.10203186226079852%
-        },
-      ],
     ]);
   }
 

test/e2e/bpmn.rendering.test.ts

@@ -19,9 +19,9 @@ import type { MatchImageSnapshotOptions } from 'jest-image-snapshot';
 
 import path from 'node:path';
 
-import 'jest-playwright-preset';
+import { ImageSnapshotConfigurator, MultiBrowserImageSnapshotThresholds, withCustomOutputDirectory } from './helpers/visu/image-snapshot-config';
 
-import { ImageSnapshotConfigurator, MultiBrowserImageSnapshotThresholds } from './helpers/visu/image-snapshot-config';
+import 'jest-playwright-preset';
 
 import { FitType } from '@lib/component/options';
 import { AvailableTestPages, PageTester } from '@test/shared/visu/bpmn-page-utils';
@@ -34,10 +34,13 @@ class FitImageSnapshotConfigurator extends ImageSnapshotConfigurator {
     fitType: FitType;
     margin?: number;
   }): MatchImageSnapshotOptions {
-    const config = super.getConfig(parameter);
-    config.customSnapshotsDir = FitImageSnapshotConfigurator.buildSnapshotFitDirectory(config.customSnapshotsDir, parameter.fitType, true, parameter.margin ?? 0);
-    config.customDiffDir = parameter.buildCustomDiffDir(config, parameter.fitType, parameter.margin);
-    return config;
+    return withCustomOutputDirectory(
+      {
+        ...super.getConfig(parameter),
+        customSnapshotsDir: FitImageSnapshotConfigurator.buildSnapshotFitDirectory(super.getConfig(parameter).customSnapshotsDir, parameter.fitType, true, parameter.margin ?? 0),
+      },
+      parameter.buildCustomDiffDir(super.getConfig(parameter), parameter.fitType, parameter.margin),
+    );
   }
 
   private static buildSnapshotFitDirectory(parentDirectory: string, fitType: FitType, withMargin = false, margin?: number): string {

test/e2e/diagram.navigation.fit.test.ts

@@ -20,9 +20,9 @@ import type { Point } from '@test/shared/visu/bpmn-page-utils';
 import path from 'node:path';
 
 import debugLogger from 'debug';
-import 'jest-playwright-preset';
 
-import { ImageSnapshotConfigurator, MultiBrowserImageSnapshotThresholds } from './helpers/visu/image-snapshot-config';
+import { withCustomOutputDirectory, ImageSnapshotConfigurator, MultiBrowserImageSnapshotThresholds } from './helpers/visu/image-snapshot-config';
+import 'jest-playwright-preset';
 
 import { ZoomType } from '@lib/component/options';
 import { AvailableTestPages, PageTester } from '@test/shared/visu/bpmn-page-utils';
@@ -119,11 +119,15 @@ describe('diagram navigation - zoom and pan with mouse', () => {
 
     const image = await page.screenshot({ fullPage: true });
     const config = imageSnapshotConfigurator.getConfig(bpmnDiagramName);
-    expect(image).toMatchImageSnapshot({
-      ...config,
-      customSnapshotIdentifier: 'initial.zoom',
-      customDiffDir: path.join(config.customDiffDir, `mouse-zoom-in-out-${xTimes}-times`),
-    });
+    expect(image).toMatchImageSnapshot(
+      withCustomOutputDirectory(
+        {
+          ...config,
+          customSnapshotIdentifier: 'initial.zoom',
+        },
+        path.join(config.customDiffDir, `mouse-zoom-in-out-${xTimes}-times`),
+      ),
+    );
   });
 });
 
@@ -163,11 +167,15 @@ describe('diagram navigation - zoom with buttons', () => {
 
     const image = await page.screenshot({ fullPage: true });
     const config = imageSnapshotConfigurator.getConfig(bpmnDiagramName);
-    expect(image).toMatchImageSnapshot({
-      ...config,
-      customSnapshotIdentifier: 'initial.zoom',
-      customDiffDir: path.join(config.customDiffDir, `button-zoom-in-out-${xTimes}-times`),
-    });
+    expect(image).toMatchImageSnapshot(
+      withCustomOutputDirectory(
+        {
+          ...config,
+          customSnapshotIdentifier: 'initial.zoom',
+        },
+        path.join(config.customDiffDir, `button-zoom-in-out-${xTimes}-times`),
+      ),
+    );
   });
 });
 
@@ -205,11 +213,15 @@ describe('diagram navigation - zoom with buttons and mouse', () => {
 
     const image = await page.screenshot({ fullPage: true });
     const config = imageSnapshotConfigurator.getConfig(bpmnDiagramName);
-    expect(image).toMatchImageSnapshot({
-      ...config,
-      customSnapshotIdentifier: 'initial.zoom',
-      customDiffDir: path.join(config.customDiffDir, `zoom-button-then-mouse-${firstZoom}-then-${secondZoom}`),
-    });
+    expect(image).toMatchImageSnapshot(
+      withCustomOutputDirectory(
+        {
+          ...config,
+          customSnapshotIdentifier: 'initial.zoom',
+        },
+        path.join(config.customDiffDir, `zoom-button-then-mouse-${firstZoom}-then-${secondZoom}`),
+      ),
+    );
   });
 
   it.each`
@@ -222,10 +234,14 @@ describe('diagram navigation - zoom with buttons and mouse', () => {
 
     const image = await page.screenshot({ fullPage: true });
     const config = imageSnapshotConfigurator.getConfig(bpmnDiagramName);
-    expect(image).toMatchImageSnapshot({
-      ...config,
-      customSnapshotIdentifier: 'initial.zoom',
-      customDiffDir: path.join(config.customDiffDir, `zoom-mouse-then-button-${firstZoom}-then-${secondZoom}`),
-    });
+    expect(image).toMatchImageSnapshot(
+      withCustomOutputDirectory(
+        {
+          ...config,
+          customSnapshotIdentifier: 'initial.zoom',
+        },
+        path.join(config.customDiffDir, `zoom-mouse-then-button-${firstZoom}-then-${secondZoom}`),
+      ),
+    );
   });
 });

test/e2e/diagram.navigation.zoom.pan.test.ts

@@ -55,14 +55,15 @@ export class ImageSnapshotConfigurator {
     const fileName = typeof parameter === 'string' ? parameter : parameter.fileName;
     const failureThreshold = this.getFailureThreshold(fileName);
 
-    return {
-      ...defaultImageSnapshotConfig,
-      failureThreshold: failureThreshold,
-      customSnapshotIdentifier: fileName,
-      customSnapshotsDir: this.defaultCustomSnapshotsDir,
-      customDiffDir: this.defaultCustomDiffDir,
-      customReceivedDir: this.defaultCustomDiffDir,
-    };
+    return withCustomOutputDirectory(
+      {
+        ...defaultImageSnapshotConfig,
+        failureThreshold: failureThreshold,
+        customSnapshotIdentifier: fileName,
+        customSnapshotsDir: this.defaultCustomSnapshotsDir,
+      },
+      this.defaultCustomDiffDir,
+    );
   }
 
   private getFailureThreshold(fileName: string): number {
@@ -166,3 +167,12 @@ export class MultiBrowserImageSnapshotThresholds {
     }
   }
 }
+
+/**
+ * Set both the customDiffDir and customReceivedDir to the same value.
+ */
+export const withCustomOutputDirectory = (options: MatchImageSnapshotOptions, customOutputDirectory: string): MatchImageSnapshotOptions => ({
+  ...options,
+  customDiffDir: customOutputDirectory,
+  customReceivedDir: customOutputDirectory,
+});