e2e: add support for perf metrics & start logging a few scenarios (#8969)

### Summary
Our shared fixtures file (`_test.setup.ts`) was getting unwieldy, so I
reorganized it to improve maintainability. Most fixtures (though not
all) are now grouped under `test/e2e/fixtures/test-setup/<area>/`,
making it easier to locate and manage them by scope.

I also introduced a new `metrics` fixture to support sending metric data
to the e2e-test-insights API. While the dashboard doesn’t surface this
data yet, we’re starting to collect it now in preparation. A few tests
that were previously logging event timing have been updated to send this
data to the API. I also added a metrics README to make it clear how to
record/send metrics for new feature areas.

### QA Notes

@:performance @:web @:win

Author: midleman

build/secrets/.secrets.baseline

@@ -90,10 +90,6 @@
     {
       "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
     },
-    {
-      "path": "detect_secrets.filters.common.is_baseline_file",
-      "filename": "build/secrets/.secrets.baseline"
-    },
     {
       "path": "detect_secrets.filters.heuristic.is_indirect_reference"
     },
@@ -1738,5 +1734,5 @@
       }
     ]
   },
-  "generated_at": "2025-08-25T14:05:12Z"
+  "generated_at": "2025-08-25T17:39:02Z"
 }

test/e2e/fixtures/test-setup/app.fixtures.ts

@@ -0,0 +1,158 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (C) 2025 Posit Software, PBC. All rights reserved.
+ *  Licensed under the Elastic License 2.0. See LICENSE.txt for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import path = require('path');
+import { join } from 'path';
+import * as fs from 'fs';
+import { constants, access, rm, mkdir, rename } from 'fs/promises';
+import * as os from 'os';
+import * as playwright from '@playwright/test';
+import { Application, ApplicationOptions, MultiLogger, createApp, copyFixtureFile, Quality, getRandomUserDataDir } from '../../infra';
+import { SPEC_NAME, setFixtureScreenshot, ROOT_PATH, TEMP_DIR } from './constants';
+
+export interface CustomTestOptions {
+	web: boolean;
+	artifactDir: string;
+	headless?: boolean;
+}
+
+export interface AppFixtureOptions {
+	options: ApplicationOptions;
+	logsPath: string;
+	logger: MultiLogger;
+	workerInfo: playwright.WorkerInfo;
+}
+
+export function OptionsFixture() {
+	return async (logsPath: string, logger: MultiLogger, snapshots: boolean, project: CustomTestOptions) => {
+		const TEST_DATA_PATH = join(os.tmpdir(), 'vscsmoke');
+		const EXTENSIONS_PATH = join(TEST_DATA_PATH, 'extensions-dir');
+		const WORKSPACE_PATH = join(TEST_DATA_PATH, 'qa-example-content');
+		const SPEC_CRASHES_PATH = join(ROOT_PATH, '.build', 'crashes', project.artifactDir, TEMP_DIR);
+
+		// get the version from package.json
+		const packageJsonPath = join(ROOT_PATH, 'package.json');
+		const packageJson = JSON.parse(await fs.promises.readFile(packageJsonPath, 'utf-8'));
+		const packageVersion = packageJson.version || '0.0.0';
+		const version = {
+			major: parseInt(packageVersion.split('.')[0], 10),
+			minor: parseInt(packageVersion.split('.')[1], 10),
+			patch: parseInt(packageVersion.split('.')[2], 10),
+		};
+
+		const options: ApplicationOptions = {
+			codePath: process.env.BUILD,
+			workspacePath: WORKSPACE_PATH,
+			userDataDir: join(TEST_DATA_PATH, 'd'),
+			extensionsPath: EXTENSIONS_PATH,
+			logger,
+			logsPath,
+			crashesPath: SPEC_CRASHES_PATH,
+			verbose: !!process.env.VERBOSE,
+			remote: !!process.env.REMOTE,
+			web: project.web,
+			headless: project.headless,
+			tracing: true,
+			snapshots,
+			quality: Quality.Dev,
+			version
+		};
+
+		options.userDataDir = getRandomUserDataDir(options);
+
+		return options;
+	};
+}
+
+export function UserDataDirFixture() {
+	return async (options: ApplicationOptions) => {
+		const userDir = options.web ? join(options.userDataDir, 'data', 'User') : join(options.userDataDir, 'User');
+		process.env.PLAYWRIGHT_USER_DATA_DIR = userDir;
+
+		// Copy keybindings and settings fixtures to the user data directory
+		await copyFixtureFile('keybindings.json', userDir, true);
+
+		const settingsFileName = 'settings.json';
+		if (fs.existsSync('/.dockerenv')) {
+
+			const fixturesDir = path.join(ROOT_PATH, 'test/e2e/fixtures');
+			const settingsFile = path.join(fixturesDir, 'settings.json');
+
+			const mergedSettings = {
+				...JSON.parse(fs.readFileSync(settingsFile, 'utf8')),
+				...JSON.parse(fs.readFileSync(path.join(fixturesDir, 'settingsDocker.json'), 'utf8')),
+			};
+
+			// Overwrite file
+			fs.writeFileSync(settingsFile, JSON.stringify(mergedSettings, null, 2));
+		}
+
+		await copyFixtureFile(settingsFileName, userDir);
+
+		return userDir;
+	};
+}
+
+export function AppFixture() {
+	return async (fixtureOptions: AppFixtureOptions, use: (arg0: Application) => Promise<void>) => {
+		const { options, logsPath, logger, workerInfo } = fixtureOptions;
+		const app = createApp(options);
+
+		try {
+			await app.start();
+			await app.workbench.sessions.expectNoStartUpMessaging();
+
+			await use(app);
+		} catch (error) {
+			// capture a screenshot on failure
+			const screenshotPath = path.join(logsPath, 'app-start-failure.png');
+			try {
+				const page = app.code?.driver?.page;
+				if (page) {
+					const screenshot = await page.screenshot({ path: screenshotPath });
+					setFixtureScreenshot(screenshot);
+				}
+			} catch {
+				// ignore
+			}
+
+			throw error; // re-throw the error to ensure test failure
+		} finally {
+			await app.stop();
+
+			// rename the temp logs dir to the spec name (if available)
+			const specLogsPath = path.join(path.dirname(logsPath), SPEC_NAME || `worker-${workerInfo.workerIndex}`);
+			await moveAndOverwrite(logger, logsPath, specLogsPath);
+		}
+	};
+}
+
+async function moveAndOverwrite(logger: MultiLogger, sourcePath: string, destinationPath: string) {
+	try {
+		await access(sourcePath, constants.F_OK);
+	} catch {
+		console.error(`moveAndOverwrite: source path does not exist: ${sourcePath}`);
+		return;
+	}
+
+	// check if the destination exists and delete it if so
+	try {
+		await access(destinationPath, constants.F_OK);
+		await rm(destinationPath, { recursive: true, force: true });
+	} catch (err) { }
+
+	// ensure parent directory of destination path exists
+	const destinationDir = path.dirname(destinationPath);
+	await mkdir(destinationDir, { recursive: true });
+
+	// rename source to destination
+	try {
+		await rename(sourcePath, destinationPath);
+		logger.setPath(destinationPath);
+		logger.log('Logger path updated to:', destinationPath);
+	} catch (err) {
+		logger.log(`moveAndOverwrite: failed to move ${sourcePath} to ${destinationPath}:`, err);
+	}
+}

test/e2e/fixtures/test-setup/constants.ts

@@ -0,0 +1,24 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (C) 2025 Posit Software, PBC. All rights reserved.
+ *  Licensed under the Elastic License 2.0. See LICENSE.txt for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { join } from 'path';
+import { randomUUID } from 'crypto';
+
+// Constants used across test fixtures
+export const TEMP_DIR = `temp-${randomUUID()}`;
+export const ROOT_PATH = process.cwd();
+export const LOGS_ROOT_PATH = join(ROOT_PATH, 'test-logs');
+
+// Global state variables that need to be mutable
+export let SPEC_NAME = '';
+export let fixtureScreenshot: Buffer;
+
+export function setSpecName(name: string) {
+	SPEC_NAME = name;
+}
+
+export function setFixtureScreenshot(screenshot: Buffer) {
+	fixtureScreenshot = screenshot;
+}

test/e2e/fixtures/test-setup/file-ops.fixtures.ts

@@ -0,0 +1,69 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (C) 2025 Posit Software, PBC. All rights reserved.
+ *  Licensed under the Elastic License 2.0. See LICENSE.txt for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as playwright from '@playwright/test';
+import * as path from 'path';
+import { test } from '@playwright/test';
+import { Application } from '../../infra';
+
+/**
+ * Create file operation helpers for opening files and folders
+ */
+export function FileOperationsFixture(app: Application) {
+	return {
+		openFile: async (filePath: string, waitForFocus = true) => {
+			await test.step(`Open file: ${path.basename(filePath)}`, async () => {
+				await app.workbench.quickaccess.openFile(path.join(app.workspacePathOrFolder, filePath), waitForFocus);
+			});
+		},
+
+		openDataFile: async (filePath: string) => {
+			await test.step(`Open data file: ${path.basename(filePath)}`, async () => {
+				await app.workbench.quickaccess.openDataFile(path.join(app.workspacePathOrFolder, filePath));
+			});
+		},
+
+		openFolder: async (folderPath: string) => {
+			await test.step(`Open folder: ${folderPath}`, async () => {
+				await app.workbench.hotKeys.openFolder();
+				await playwright.expect(app.workbench.quickInput.quickInputList.locator('a').filter({ hasText: '..' })).toBeVisible();
+
+				const folderNames = folderPath.split('/');
+
+				for (const folderName of folderNames) {
+					const quickInputOption = app.workbench.quickInput.quickInputResult.getByText(folderName);
+
+					// Ensure we are ready to select the next folder
+					const timeoutMs = 30000;
+					const retryInterval = 2000;
+					const maxRetries = Math.ceil(timeoutMs / retryInterval);
+
+					for (let i = 0; i < maxRetries; i++) {
+						try {
+							await playwright.expect(quickInputOption).toBeVisible({ timeout: retryInterval });
+							// Success — exit loop
+							break;
+						} catch (error) {
+							// Press PageDown if not found
+							await app.code.driver.page.keyboard.press('PageDown');
+
+							// If last attempt, rethrow
+							if (i === maxRetries - 1) {
+								throw error;
+							}
+						}
+					}
+
+					await app.workbench.quickInput.quickInput.pressSequentially(folderName + '/');
+
+					// Ensure next folder is no longer visible
+					await playwright.expect(quickInputOption).not.toBeVisible();
+				}
+
+				await app.workbench.quickInput.clickOkButton();
+			});
+		}
+	};
+}

test/e2e/fixtures/test-setup/index.ts

@@ -0,0 +1,7 @@
+// Re-export constants
+export { TEMP_DIR, LOGS_ROOT_PATH, fixtureScreenshot, setSpecName } from './constants';
+export * from './metrics.fixtures';
+export * from './reporting.fixtures';
+export * from './settings.fixtures';
+export * from './app.fixtures';
+export * from './file-ops.fixtures';

test/e2e/fixtures/test-setup/metrics.fixtures.ts

@@ -0,0 +1,71 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (C) 2025 Posit Software, PBC. All rights reserved.
+ *  Licensed under the Elastic License 2.0. See LICENSE.txt for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { recordDataFileLoad, recordDataFilter, recordDataSort, recordToCode, type DataExplorerAutoContext, type DataExplorerShortcutOptions } from '../../utils/metrics/metric-data-explorer.js';
+import { recordRunCell, type NotebookShortcutOptions } from '../../utils/metrics/metric-notebooks.js';
+import { type RecordMetric, type MetricResult, type MetricContext, type MetricTargetType } from '../../utils/metrics/metric-base.js';
+import { Application, MultiLogger } from '../../infra/index.js';
+
+/**
+ * Creates a metrics recorder for performance testing
+ */
+export function MetricsFixture(app: Application, logger: MultiLogger): RecordMetric {
+	const dataExplorerAutoContext: DataExplorerAutoContext = {
+		getRowCount: async () => {
+			return app.workbench.dataExplorer.grid.getRowCount();
+		},
+		getColumnCount: async () => {
+			return app.workbench.dataExplorer.grid.getColumnCount();
+		}
+	};
+
+	return {
+		dataExplorer: {
+			loadData: async <T>(
+				operation: () => Promise<T>,
+				targetType: MetricTargetType,
+				options?: DataExplorerShortcutOptions
+			): Promise<MetricResult<T>> => {
+				return recordDataFileLoad(operation, targetType, !!app.code.electronApp, logger, dataExplorerAutoContext, options);
+			},
+			filter: async <T>(
+				operation: () => Promise<T>,
+				targetType: MetricTargetType,
+				options?: DataExplorerShortcutOptions
+			): Promise<MetricResult<T>> => {
+				return recordDataFilter(operation, targetType, !!app.code.electronApp, logger, dataExplorerAutoContext, options);
+			},
+			sort: async <T>(
+				operation: () => Promise<T>,
+				targetType: MetricTargetType,
+				options?: DataExplorerShortcutOptions
+			): Promise<MetricResult<T>> => {
+				return recordDataSort(operation, targetType, !!app.code.electronApp, logger, dataExplorerAutoContext, options);
+			},
+			toCode: async <T>(
+				operation: () => Promise<T>,
+				targetType: MetricTargetType,
+				options?: DataExplorerShortcutOptions
+			): Promise<MetricResult<T>> => {
+				return recordToCode(operation, targetType, !!app.code.electronApp, logger, dataExplorerAutoContext, options);
+			}
+		},
+		notebooks: {
+			runCell: async <T>(
+				operation: () => Promise<T>,
+				targetType: MetricTargetType,
+				language?: string,
+				description?: string,
+				context?: MetricContext | (() => Promise<MetricContext>)
+			): Promise<MetricResult<T>> => {
+				const options: NotebookShortcutOptions = {
+					description,
+					additionalContext: context
+				};
+				return recordRunCell(operation, targetType, !!app.code.electronApp, logger, language, options);
+			}
+		}
+	};
+}