From 18d19f872059b422a20a8a928d703535257e02cc Mon Sep 17 00:00:00 2001 From: Joe Fusco Date: Sun, 22 Feb 2026 13:35:03 -0500 Subject: [PATCH] Build/Test Tools: Add visual regression tests for admin pages. --- tests/visual-regression/config/screenshot.css | 75 +++ tests/visual-regression/playwright.config.js | 61 ++- .../specs/visual-snapshots.test.js | 450 ++++++++++++------ 3 files changed, 446 insertions(+), 140 deletions(-) create mode 100644 tests/visual-regression/config/screenshot.css diff --git a/tests/visual-regression/config/screenshot.css b/tests/visual-regression/config/screenshot.css new file mode 100644 index 0000000000000..5ad106d0514b7 --- /dev/null +++ b/tests/visual-regression/config/screenshot.css @@ -0,0 +1,75 @@ +/* + * Global stylesheet applied during screenshot capture. + * + * Hides volatile elements that change between environments or runs, + * preventing false positives in visual regression comparisons. + * Applied via Playwright's stylePath config option. + * + * See: https://playwright.dev/docs/test-snapshots#stylepath + */ + +/* + * Uses `visibility: hidden` instead of `display: none` to preserve + * each element's layout space. Collapsing elements with `display: none` + * would shift surrounding content and cause false positives elsewhere. + */ + +/* WordPress version/update nag in the admin footer. */ +#footer-upgrade { + visibility: hidden !important; +} + +/* Admin bar user-specific content (Howdy, gravatar). */ +#wp-admin-bar-root-default { + visibility: hidden !important; +} + +/* Gutenberg plugin menu item — not present in all environments. */ +#toplevel_page_gutenberg { + visibility: hidden !important; +} + +/* Gravatar images — external service, different per environment. */ +.avatar { + visibility: hidden !important; +} + +/* Date columns in list tables — relative timestamps shift between runs. */ +.column-date { + visibility: hidden !important; +} + +/* Dashboard widgets with dynamic counts and activity. */ +#dashboard_right_now .inside, +#dashboard_activity .inside { + visibility: hidden !important; +} + +/* Update-related timestamps. */ +.update-last-checked { + visibility: hidden !important; +} + +/* + * Admin notices — various nags (PHP deprecation, updates, etc.). + * `.error:not(#error)` excludes the `
` database error + * container from wpdb (wp-includes/class-wpdb.php) as a defensive measure. + */ +.notice, +.update-nag, +.updated, +.error:not(#error), +#message { + visibility: hidden !important; +} + +/* General Settings — live date/time preview changes on every run. */ +#local-time, +.example { + visibility: hidden !important; +} + +/* Users list table — post counts vary based on test data. */ +.column-posts { + visibility: hidden !important; +} diff --git a/tests/visual-regression/playwright.config.js b/tests/visual-regression/playwright.config.js index 759d887bf71c2..2701450b9f7b6 100644 --- a/tests/visual-regression/playwright.config.js +++ b/tests/visual-regression/playwright.config.js @@ -1,3 +1,18 @@ +/** + * Playwright config for visual regression tests. + * + * Captures full-page screenshots of WordPress admin screens and compares + * them against baseline snapshots. Intended for local use to catch + * unintended visual changes during development. + * + * Usage: + * npm run test:visual -- --update-snapshots # generate baselines + * npm run test:visual # compare against baselines + * + * @see tests/visual-regression/config/screenshot.css for globally hidden elements. + * @see tests/visual-regression/specs/visual-snapshots.test.js for the test spec. + */ + /** * External dependencies */ @@ -15,9 +30,53 @@ process.env.STORAGE_STATE_PATH ??= path.join( 'storage-states/admin.json' ); +// Reporters: +// - 'list' — prints pass/fail per test in the terminal. +// - 'github' — adds inline PR annotations when running in CI. +// - 'html' — generates a visual report with side-by-side diff images; +// opens automatically after local runs. +const reporter = [ + [ 'list' ], + ...( process.env.CI ? [ [ 'github' ] ] : [] ), + [ + 'html', + { + open: process.env.CI ? 'never' : 'always', + outputFolder: path.join( + process.env.WP_ARTIFACTS_PATH, + 'visual-report' + ), + }, + ], +]; + const config = defineConfig( { ...baseConfig, - globalSetup: undefined, + fullyParallel: true, + // No retries — visual diffs are expected when regressions exist; + // retrying would just re-confirm the same diff. + retries: 0, + // Serialize tests in CI to reduce flakiness from resource contention. + workers: process.env.CI ? 1 : undefined, + reporter, + use: { + ...baseConfig.use, + viewport: { width: 1280, height: 720 }, + }, + expect: { + toHaveScreenshot: { + // Only disables CSS animations/transitions. JavaScript-driven + // animations (e.g. jQuery .animate()) can still cause flakes. + animations: 'disabled', + // Captures the entire scrollable page, not just the viewport. + // The viewport width (1280) still matters — it controls layout. + fullPage: true, + // 1% tolerance — catches real regressions while ignoring + // sub-pixel anti-aliasing differences across environments. + maxDiffPixelRatio: 0.01, + stylePath: path.join( __dirname, 'config', 'screenshot.css' ), + }, + }, webServer: { ...baseConfig.webServer, command: 'npm run env:start', diff --git a/tests/visual-regression/specs/visual-snapshots.test.js b/tests/visual-regression/specs/visual-snapshots.test.js index 048a6bc0b47cb..1fbb7e1abaa82 100644 --- a/tests/visual-regression/specs/visual-snapshots.test.js +++ b/tests/visual-regression/specs/visual-snapshots.test.js @@ -1,166 +1,338 @@ +/** + * Visual regression tests for WordPress admin screens. + * + * Each entry in the `pages` array generates a test that navigates to the page, + * waits for stability, and takes a full-page screenshot compared against a + * baseline snapshot. + * + * To add a new page, append an entry to the `pages` array. If the page + * contains dynamic content not already covered by screenshot.css, add a + * `masks` array of CSS selectors for those elements. + * + * @see tests/visual-regression/config/screenshot.css for globally hidden elements. + * @see tests/visual-regression/playwright.config.js for snapshot settings. + */ + +/** + * WordPress dependencies + */ import { test, expect } from '@wordpress/e2e-test-utils-playwright'; -const elementsToHide = [ - '#footer-upgrade', - '#wp-admin-bar-root-default', - '#toplevel_page_gutenberg' -]; +/** + * Waits for network activity, fonts, and jQuery animations to settle. + * + * @param {import('@playwright/test').Page} page + */ +async function waitForPageReady( page ) { + await page.waitForLoadState( 'load' ); -test.describe( 'Admin Visual Snapshots', () => { - test( 'All Posts', async ({ admin, page }) => { - await admin.visitAdminPage( '/edit.php' ); - await expect( page ).toHaveScreenshot( 'All Posts.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); + // Wait for in-flight requests (AJAX heartbeat, dashboard widgets) to + // finish. The 5 s timeout keeps the suite moving when a long-poll + // endpoint (e.g. heartbeat-tick) holds the connection open. + await page + .waitForLoadState( 'networkidle', { timeout: 5000 } ) + .catch( () => {} ); - test( 'Categories', async ({ admin, page }) => { - await admin.visitAdminPage( '/edit-tags.php', 'taxonomy=category' ); - await expect( page ).toHaveScreenshot( 'Categories.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); + // If a webfont fails to load (network issue, Docker DNS), this resolves + // with fallback fonts and the diff will surface the discrepancy. + await page.evaluate( () => document.fonts.ready ); - test( 'Tags', async ({ admin, page }) => { - await admin.visitAdminPage( '/edit-tags.php', 'taxonomy=post_tag' ); - await expect( page ).toHaveScreenshot( 'Tags.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); + // Wait for jQuery animations (e.g. dashboard widget slide-in) to + // complete. CSS animations are already disabled in the Playwright config, + // but jQuery .animate() bypasses that setting. + await page.evaluate( () => { + if ( typeof jQuery === 'undefined' ) { + return; + } + return new Promise( ( resolve ) => { + if ( jQuery.active === 0 && jQuery( ':animated' ).length === 0 ) { + resolve(); + return; + } + const interval = setInterval( () => { + if ( jQuery.active === 0 && jQuery( ':animated' ).length === 0 ) { + clearInterval( interval ); + resolve(); + } + }, 100 ); + // Safety valve: resolve after 10 s so a stuck animation or + // unresolved AJAX request doesn't hang the entire suite. + setTimeout( () => { + clearInterval( interval ); + resolve(); + }, 10000 ); + } ); } ); +} - test( 'Media Library', async ({ admin, page }) => { - await admin.visitAdminPage( '/upload.php' ); - await expect( page ).toHaveScreenshot( 'Media Library.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); +/** + * @typedef {Object} PageEntry + * @property {string} name Display name used as the test title and snapshot filename. + * @property {string} path Admin-relative URL path (e.g. '/edit.php'). + * @property {string | ( data: * ) => string} [query] Query string appended to path. + * When a function, it receives the return value of `setup`. + * @property {string[]} [masks] CSS selectors for elements to mask in the screenshot. + * @property {( requestUtils: Object ) => Promise<*>} [setup] + * Called before navigation. Return value is forwarded to `query` (if a function) and `teardown`. + * @property {( requestUtils: Object, data: * ) => Promise} [teardown] + * Called after the screenshot assertion (in a `finally` block) to clean up resources created by `setup`. + */ - test( 'Add Media', async ({ admin, page }) => { - await admin.visitAdminPage( '/media-new.php' ); - await expect( page ).toHaveScreenshot( 'Add Media.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); +/** + * Admin pages to capture, ordered by admin menu section. + * + * Convention: use screenshot.css for elements that appear on many pages + * (admin bar, footer, notices); use masks here for page-specific volatility. + */ +const pages = [ + // -- Dashboard -- + { + name: 'Dashboard', + path: '/index.php', + masks: [ + // Health status varies by environment and installed plugins. + '#dashboard_site_health', + ], + }, + { + name: 'Updates', + path: '/update-core.php', + masks: [ + // Available updates and version numbers change per environment. + 'form.upgrade', + '.last-checked', + ], + }, - test( 'All Pages', async ({ admin, page }) => { - await admin.visitAdminPage( '/edit.php', 'post_type=page' ); - await expect( page ).toHaveScreenshot( 'All Pages.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); + // -- Posts -- + { name: 'All Posts', path: '/edit.php' }, + { + name: 'Add New Post', + path: '/post-new.php', + masks: [ + // Editor content area — empty state markup varies. + '#wp-content-editor-container', + ], + }, + { + name: 'Edit Post', + path: '/post.php', + query: ( data ) => `post=${ data.id }&action=edit`, + masks: [ + // Editor content area — markup varies. + '#wp-content-editor-container', + ], + setup: async ( requestUtils ) => + await requestUtils.rest( { + method: 'POST', + path: '/wp/v2/posts', + data: { + title: 'Visual Regression Test Post', + content: 'Test content for visual regression.', + status: 'publish', + }, + } ), + teardown: async ( requestUtils, data ) => + // force: true bypasses the trash — permanently deletes the post. + await requestUtils.rest( { + method: 'DELETE', + path: `/wp/v2/posts/${ data.id }`, + params: { force: true }, + } ), + }, + { name: 'Categories', path: '/edit-tags.php', query: 'taxonomy=category' }, + { name: 'Tags', path: '/edit-tags.php', query: 'taxonomy=post_tag' }, - test( 'Comments', async ({ admin, page }) => { - await admin.visitAdminPage( '/edit-comments.php' ); - await expect( page ).toHaveScreenshot( 'Comments.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); + // -- Media -- + { name: 'Media Library', path: '/upload.php' }, + { name: 'Add Media', path: '/media-new.php' }, - test( 'Widgets', async ({ admin, page }) => { - await admin.visitAdminPage( '/widgets.php' ); - await expect( page ).toHaveScreenshot( 'Widgets.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); + // -- Pages -- + { name: 'All Pages', path: '/edit.php', query: 'post_type=page' }, + { + name: 'Add New Page', + path: '/post-new.php', + query: 'post_type=page', + masks: [ + // Editor content area — empty state markup varies. + '#wp-content-editor-container', + ], + }, + { + name: 'Edit Page', + path: '/post.php', + query: ( data ) => `post=${ data.id }&action=edit`, + masks: [ + // Editor content area — markup varies. + '#wp-content-editor-container', + ], + setup: async ( requestUtils ) => + await requestUtils.rest( { + method: 'POST', + path: '/wp/v2/pages', + data: { + title: 'Visual Regression Test Page', + content: 'Test content for visual regression.', + status: 'publish', + }, + } ), + teardown: async ( requestUtils, data ) => + await requestUtils.rest( { + method: 'DELETE', + path: `/wp/v2/pages/${ data.id }`, + params: { force: true }, + } ), + }, - test( 'Menus', async ({ admin, page }) => { - await admin.visitAdminPage( '/nav-menus.php' ); - await expect( page ).toHaveScreenshot( 'Menus.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); + // -- Comments -- + { name: 'Comments', path: '/edit-comments.php' }, - test( 'Plugins', async ({ admin, page }) => { - await admin.visitAdminPage( '/plugins.php' ); - await expect( page ).toHaveScreenshot( 'Plugins.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); + // -- Appearance -- + { + name: 'Themes', + path: '/themes.php', + masks: [ + // Theme screenshot images differ across environments. + '.theme-screenshot img', + ], + }, + { name: 'Widgets', path: '/widgets.php' }, + { name: 'Menus', path: '/nav-menus.php' }, + { name: 'Theme File Editor', path: '/theme-editor.php', masks: [ '#newcontent' ] }, - test( 'All Users', async ({ admin, page }) => { - await admin.visitAdminPage( '/users.php' ); - await expect( page ).toHaveScreenshot( 'All Users.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); + // -- Plugins -- + { + name: 'Plugins', + path: '/plugins.php', + masks: [ + // Version numbers and author URIs change with plugin updates. + '.plugin-version-author-uri', + ], + }, + { + name: 'Add New Plugin', + path: '/plugin-install.php', + masks: [ + // Plugin cards show external content (descriptions, ratings, + // download counts) that changes frequently. Masking all cards + // means this test only verifies the page shell — search bar, + // header tabs, and pagination layout. + '.plugin-card', + ], + }, + { name: 'Plugin File Editor', path: '/plugin-editor.php', masks: [ '#newcontent' ] }, - test( 'Add User', async ({ admin, page }) => { - await admin.visitAdminPage( '/user-new.php' ); - await expect( page ).toHaveScreenshot( 'Add User.png', { - mask: [ - ...elementsToHide, - '.password-input-wrapper' - ].map( ( selector ) => page.locator( selector ) ), - }); - } ); + // -- Users -- + { name: 'All Users', path: '/users.php' }, + { + name: 'Add User', + path: '/user-new.php', + masks: [ + // Auto-generated password is random on every page load. + '.password-input-wrapper', + ], + }, + { name: 'Your Profile', path: '/profile.php' }, - test( 'Your Profile', async ({ admin, page }) => { - await admin.visitAdminPage( '/profile.php' ); - await expect( page ).toHaveScreenshot( 'Your Profile.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); + // -- Tools -- + { name: 'Available Tools', path: '/tools.php' }, + { name: 'Import', path: '/import.php' }, + { name: 'Export', path: '/export.php' }, + { name: 'Export Personal Data', path: '/export-personal-data.php' }, + { name: 'Erase Personal Data', path: '/erase-personal-data.php' }, + { + name: 'Site Health', + path: '/site-health.php', + masks: [ + // Health check results depend on server config and plugins. + '.site-health-issues .health-check-accordion', + '.site-status-all-clear', + '.site-health-progress', + ], + }, - test( 'Available Tools', async ({ admin, page }) => { - await admin.visitAdminPage( '/tools.php' ); - await expect( page ).toHaveScreenshot( 'Available Tools.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); + // -- Settings -- + { + name: 'General Settings', + path: '/options-general.php', + masks: [ + // Timezone dropdown value depends on server config. + 'td:has(> #timezone_string)', + '.timezone-info', + ], + }, + { name: 'Writing Settings', path: '/options-writing.php' }, + { name: 'Reading Settings', path: '/options-reading.php' }, + { name: 'Discussion Settings', path: '/options-discussion.php' }, + { name: 'Media Settings', path: '/options-media.php' }, + { name: 'Permalink Settings', path: '/options-permalink.php' }, + { name: 'Privacy Settings', path: '/options-privacy.php' }, +]; - test( 'Import', async ({ admin, page }) => { - await admin.visitAdminPage( '/import.php' ); - await expect( page ).toHaveScreenshot( 'Import.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); +test.describe( 'Admin Visual Snapshots', () => { + for ( const { name, path, query, masks, setup, teardown } of pages ) { + test( name, async ( { admin, page, requestUtils } ) => { + const data = setup + ? await setup( requestUtils ) + : undefined; - test( 'Export', async ({ admin, page }) => { - await admin.visitAdminPage( '/export.php' ); - await expect( page ).toHaveScreenshot( 'Export.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); + try { + const resolvedQuery = + typeof query === 'function' ? query( data ) : query; - test( 'Export Personal Data', async ({ admin, page }) => { - await admin.visitAdminPage( '/export-personal-data.php' ); - await expect( page ).toHaveScreenshot( 'Export Personal Data.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); + await admin.visitAdminPage( path, resolvedQuery ); + await waitForPageReady( page ); - test( 'Erase Personal Data', async ({ admin, page }) => { - await admin.visitAdminPage( '/erase-personal-data.php' ); - await expect( page ).toHaveScreenshot( 'Erase Personal Data.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); + let screenshotOptions = {}; + if ( Array.isArray( masks ) ) { + const locators = masks.map( ( s ) => page.locator( s ) ); - test( 'Reading Settings', async ({ admin, page }) => { - await admin.visitAdminPage( '/options-reading.php' ); - await expect( page ).toHaveScreenshot( 'Reading Settings.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); + // Warn when a mask selector matches nothing — the volatile + // element may have been removed or renamed, causing false diffs. + for ( let i = 0; i < locators.length; i++ ) { + const count = await locators[ i ].count(); + if ( count === 0 ) { + // eslint-disable-next-line no-console + console.warn( + `[${ name }] mask selector "${ masks[ i ] }" matched 0 elements` + ); + } + } - test( 'Discussion Settings', async ({ admin, page }) => { - await admin.visitAdminPage( '/options-discussion.php' ); - await expect( page ).toHaveScreenshot( 'Discussion Settings.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); + screenshotOptions = { mask: locators }; + } - test( 'Media Settings', async ({ admin, page }) => { - await admin.visitAdminPage( '/options-media.php' ); - await expect( page ).toHaveScreenshot( 'Media Settings.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); - } ); + await expect( page ).toHaveScreenshot( + `${ name }.png`, + screenshotOptions + ); + } finally { + if ( teardown ) { + try { + await teardown( requestUtils, data ); + } catch ( err ) { + // Log but don't mask the original assertion failure. + // eslint-disable-next-line no-console + console.error( + `[${ name }] teardown failed:`, + err.message + ); + } + } + } + } ); + } +} ); + +test.describe( 'Unauthenticated Visual Snapshots', () => { + // Clear authentication so the login page is captured as a logged-out user. + // Must be an empty object — omitting storageState entirely inherits the + // authenticated state from the parent config. + test.use( { storageState: {} } ); - test( 'Privacy Settings', async ({ admin, page }) => { - await admin.visitAdminPage( '/options-privacy.php' ); - await expect( page ).toHaveScreenshot( 'Privacy Settings.png', { - mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), - }); + test( 'Login', async ( { page } ) => { + await page.goto( '/wp-login.php' ); + await waitForPageReady( page ); + await expect( page ).toHaveScreenshot( 'Login.png' ); } ); } );