Setup and introduce visual regression tests

.github/workflows/test.yaml

@@ -1,6 +1,6 @@
 name: Tests
 on:
-  push:
+  push:    
     branches:
       - master
       - 'release-*'
@@ -259,3 +259,21 @@ jobs:
       - name: Check e2e tags
         run: |
           ./scripts/check-e2e-tests-for-tags
+
+  visual-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Setup env
+        uses: ./.github/actions/setup
+
+      - name: Run tests
+        shell: bash
+        run: |
+          yarn build-storybook
+          yarn add -D -W concurrently
+          yarn visual:ci
+

cypress/.gitignore

@@ -1 +1 @@
-downloads
+downloads

cypress/tsconfig.json

@@ -25,4 +25,4 @@
     "./**/*.ts",
     "../types/*.ts"
   ]
-}
+}

package.json

@@ -23,6 +23,9 @@
     "lint-l10n": "./node_modules/.bin/yamllint ./shell/assets/translations",
     "test": "NODE_OPTIONS=--max_old_space_size=8192 jest --watch",
     "test:ci": "NODE_OPTIONS=--max_old_space_size=8192 jest --collectCoverage --silent",
+    "visual": "cypress run --config-file visual/cypress.visual.config.ts",
+    "visual:open": "cypress open --env requireSnapshots=true --config-file visual/cypress.visual.config.ts",
+    "visual:ci": "npx concurrently -k -s first \"yarn storybook --quiet --ci \" \"yarn visual \"",
     "install:ci": "yarn install --frozen-lockfile",
     "dev": "bash -c 'source ./scripts/version && NODE_ENV=dev ./node_modules/.bin/vue-cli-service serve'",
     "mem-dev": "bash -c 'source ./scripts/version && NODE_ENV=dev node --max-old-space-size=8192 ./node_modules/.bin/vue-cli-service serve'",
@@ -48,7 +51,6 @@
     "coverage": "npx nyc merge coverage coverage/coverage.json",
     "storybook": "cd storybook && yarn storybook",
     "build-storybook": "cd storybook && yarn install --no-lockfile && NODE_OPTIONS=--max_old_space_size=4096 yarn build-storybook --quiet",
-    "storybook-test": "cd storybook && yarn test-storybook --stories-json ",
     "docs:install": "cd docusaurus/ && yarn install",
     "docs:build": "cd docusaurus/ && yarn build",
     "docs:start": "cd docusaurus/ && yarn start",
@@ -144,6 +146,7 @@
     "@cypress/grep": "3.1.5",
     "@cypress/vue": "5.0.5",
     "@cypress/webpack-dev-server": "3.4.1",
+    "@emerson-eps/cypress-image-snapshot": "^1.5.3",
     "@types/copy-webpack-plugin": "5.0.3",
     "@types/dompurify": "3.0.0",
     "@types/is-url": "1.2.30",
@@ -172,6 +175,7 @@
     "cypress-delete-downloads-folder": "0.0.4",
     "cypress-mochawesome-reporter": "^3.8.2",
     "cypress": "11.1.0",
+    "cypress-plugin-snapshots": "^1.4.4",
     "eslint-config-standard": "16.0.3",
     "eslint-import-resolver-node": "0.3.4",
     "eslint-module-utils": "2.6.1",

visual/.gitignore

@@ -0,0 +1,2 @@
+downloads
+snapshots/**/__diff_output__/

visual/command.ts

@@ -0,0 +1,182 @@
+import path from 'path';
+import extend from 'just-extend';
+import { CHECKSNAP, MATCH, RECORD } from './constants';
+import type {
+  CypressImageSnapshotOptions,
+  DiffSnapshotResult,
+  SnapshotOptions,
+  Subject,
+} from './types';
+
+const COMMAND_NAME = 'cypress-image-snapshot';
+const screenshotsFolder = Cypress.config('screenshotsFolder') || 'cypress/screenshots';
+const isUpdateSnapshots: boolean = Cypress.env('updateSnapshots') || false;
+const isSnapshotDebug: boolean = Cypress.env('debugSnapshots') || false;
+const newImageMessage = 'A new reference Image was created for';
+
+let currentTest: string;
+let errorMessages: { [key: string]: string } = {};
+
+export const defaultOptions: SnapshotOptions = {
+  screenshotsFolder,
+  isUpdateSnapshots,
+  isSnapshotDebug,
+  specFileName:         Cypress.spec.name,
+  specRelativeFolder:   Cypress.spec.relative,
+  extraFolders:         '',
+  currentTestTitle:     '',
+  failureThreshold:     0,
+  failureThresholdType: 'pixel',
+  timeout:              5000,
+  delayBetweenTries:    1000,
+};
+
+const matchImageSnapshot = (defaultOptionsOverrides: CypressImageSnapshotOptions) => (
+  subject: Subject,
+  nameOrCommandOptions: CypressImageSnapshotOptions | string,
+  commandOptions?: CypressImageSnapshotOptions,
+) => {
+  const { filename, options } = getNameAndOptions(
+    nameOrCommandOptions,
+    defaultOptionsOverrides,
+    commandOptions,
+  );
+
+  if (!currentTest) {
+    currentTest = Cypress.currentTest.title;
+  } else if (
+    currentTest !== Cypress.currentTest.title
+  ) {
+    // we need to ensure the errors messages about new references being created are kept
+    if (currentTest === Cypress.currentTest.title) {
+      errorMessages = Object.fromEntries(
+        Object.entries(errorMessages).filter(([, value]) => value.includes(newImageMessage)),
+      );
+    } else {
+      errorMessages = {};
+    }
+    currentTest = Cypress.currentTest.title;
+  }
+
+  recursiveSnapshot(subject, options, filename);
+  cy.wrap(errorMessages).as('matchImageSnapshot');
+};
+
+/**
+ * Add this function to your `supportFile` for e2e/component
+ * Accepts options that are used for all instances of `toMatchSnapshot`
+ */
+export const addMatchImageSnapshotCommand = (defaultOptionsOverrides: CypressImageSnapshotOptions = {}) => {
+  Cypress.Commands.add(
+    'matchImageSnapshot',
+    { prevSubject: ['optional', 'element', 'document', 'window'] },
+    matchImageSnapshot(defaultOptionsOverrides),
+  );
+};
+
+const recursiveSnapshot = (subject: Subject, options: SnapshotOptions, filename: string): void => {
+  const screenshotName = getScreenshotFilename(filename);
+
+  cy.task(MATCH, {
+    ...options,
+    currentTestTitle: Cypress.currentTest.title,
+  });
+
+  cy.task<boolean>(CHECKSNAP, { screenshotName, options }).then((exist) => {
+    if (!exist) {
+      // base image does not exist yes
+      // so make sure we have a valid image by waiting the maximum timeout
+      cy.wait(options.timeout);
+    }
+    cy.wrap(subject).screenshot(screenshotName, options);
+    cy.task<DiffSnapshotResult>(RECORD).then(({
+      added,
+      pass,
+      updated,
+      imageDimensions,
+      diffPixelCount,
+      diffRatio,
+      diffSize,
+      diffOutputPath,
+    }) => {
+      if (pass) return;
+
+      if (added) {
+        const message = `New snapshot: '${ screenshotName }' was added`;
+
+        Cypress.log({ name: COMMAND_NAME, message });
+        // An after each hook should check if @matchImageSnapshot is defined, if yes it should fail the tests
+        errorMessages[screenshotName] = `${ newImageMessage } ${ screenshotName }`;
+
+        return;
+      }
+
+      if (!pass && !added && !updated) {
+        const message = diffSize ? `Image size (${ imageDimensions.baselineWidth }x${ imageDimensions.baselineHeight }) different than saved snapshot size (${ imageDimensions.receivedWidth }x${ imageDimensions.receivedHeight }).\nSee diff for details: ${ diffOutputPath }` : `Image was ${ diffRatio * 100 }% different from saved snapshot with ${ diffPixelCount } different pixels.\nSee diff for details: ${ diffOutputPath }`;
+
+        // An after each hook should check if @matchImageSnapshot is defined, if yes it should fail the tests
+        errorMessages[screenshotName] = message;
+
+        Cypress.log({ name: COMMAND_NAME, message });
+      }
+    });
+  });
+};
+
+const getNameAndOptions = (
+  nameOrCommandOptions: CypressImageSnapshotOptions | string,
+  defaultOptionsOverrides: CypressImageSnapshotOptions,
+  commandOptions?: CypressImageSnapshotOptions,
+) => {
+  let filename: string | undefined;
+  let options = extend(
+    true,
+    {},
+    defaultOptions,
+    defaultOptionsOverrides,
+  ) as SnapshotOptions;
+
+  if (typeof nameOrCommandOptions === 'string' && commandOptions) {
+    filename = nameOrCommandOptions;
+    options = extend(
+      true,
+      {},
+      defaultOptions,
+      defaultOptionsOverrides,
+      commandOptions,
+    ) as SnapshotOptions;
+  }
+  if (typeof nameOrCommandOptions === 'string') {
+    filename = nameOrCommandOptions;
+  }
+  if (typeof nameOrCommandOptions === 'object') {
+    options = extend(
+      true,
+      {},
+      defaultOptions,
+      defaultOptionsOverrides,
+      nameOrCommandOptions,
+    ) as SnapshotOptions;
+  }
+
+  return {
+    filename,
+    options,
+  };
+};
+
+const getScreenshotFilename = (filename: string | undefined) => {
+  if (filename) {
+    return filename;
+  }
+
+  return Cypress.currentTest.titlePath.join(' -- ');
+};
+
+/**
+ * replaces forward slashes (/) and backslashes (\) in a given input string with the appropriate path separator based on the operating system
+ * @param input string to replace
+ */
+export const replaceSlashes = (input: string): string => {
+  return input.replace(/\\/g, path.sep).replace(/\//g, path.sep);
+};

visual/constants.ts

@@ -0,0 +1,3 @@
+export const MATCH = 'Matching image snapshot';
+export const RECORD = 'Recording snapshot result';
+export const CHECKSNAP = 'Check if base Image already exits';

visual/cypress.visual.config.ts

@@ -0,0 +1,50 @@
+/* eslint-disable no-console */
+import { defineConfig } from 'cypress';
+import { addMatchImageSnapshotPlugin } from './plugin';
+
+// Required for env vars to be available in cypress
+require('dotenv').config();
+
+// const baseUrl = (process.env.TEST_BASE_URL || 'https://localhost:8005').replace(/\/$/, '');
+const baseUrl = 'http://localhost:6006/'; // Storybook URL
+
+/**
+ * CONFIGURATION
+ */
+export default defineConfig({
+  trashAssetsBeforeRuns:  true,
+  chromeWebSecurity:      false,
+  video:                  false,
+  screenshotOnRunFailure: false,
+  // Define viewport to avoid issues with CI
+  viewportWidth:          1000,
+  viewportHeight:         660,
+  retries:                0,
+  env:                    {
+    baseUrl,
+    coverage: false
+  },
+
+  // https://docs.cypress.io/app/references/configuration#component
+  component: {
+    devServer: {
+      bundler:       'webpack',
+      framework:     'vue-cli',
+      webpackConfig: require('@vue/cli-service/webpack.config.js')
+    },
+    specPattern: './visual/*.visual.{js,jsx,ts,tsx}',
+    supportFile: false
+  },
+
+  // https://docs.cypress.io/app/references/configuration#Testing-Type-Specific-Options
+  e2e: {
+    experimentalSessionAndOrigin: true,
+    supportFile:                  './visual/support.ts',
+    specPattern:                  './visual/tests/*.visual.{js,jsx,ts,tsx}',
+    baseUrl,
+    screenshotOnRunFailure:       false,
+    setupNodeEvents(on, config) {
+      addMatchImageSnapshotPlugin(on);
+    },
+  },
+});