e2e-test: browser in parallel & enable Currents

.github/workflows/e2e-test-release-run-ubuntu.yml

@@ -88,10 +88,15 @@ jobs:
         env:
           POSITRON_PY_VER_SEL: 3.10.12
           POSITRON_R_VER_SEL: 4.4.0
-        id: electron-smoke-tests
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
+          CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+          COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }}
+          PWTEST_BLOB_DO_NOT_REMOVE: 1
+          CURRENTS_TAG: "electron,release,${{ inputs.e2e_grep }}"
+        id: electron-e2e-tests
         run: |
           export DISPLAY=:10
-          BUILD=/usr/share/positron npx playwright test --project e2e-electron --workers 2 --grep="${{ env.E2E_GREP }}"
+          BUILD=/usr/share/positron npx playwright test --project e2e-electron --workers 3 --grep=${{ env.E2E_GREP }}
 
       - name: Upload Playwright Report to S3
         if: ${{ !cancelled() }}

.github/workflows/positron-full-test.yml

@@ -96,11 +96,6 @@ jobs:
   e2e-electron-tests:
     runs-on: ubuntu-latest-8x
     timeout-minutes: 60
-    strategy:
-      fail-fast: false
-      matrix:
-        shardIndex: [1, 2]
-        shardTotal: [2]
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       POSITRON_BUILD_NUMBER: 0 # CI skips building releases
@@ -131,26 +126,32 @@ jobs:
         env:
           POSITRON_PY_VER_SEL: 3.10.12
           POSITRON_R_VER_SEL: 4.4.0
+          CURRENTS_PROJECT_ID: ZOs5z2
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
+          CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+          COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }} # only works on push events
+          PWTEST_BLOB_DO_NOT_REMOVE: 1
+          CURRENTS_TAG: "electron"
         id: electron-tests
-        run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 1 --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 3
 
       - name: Upload blob report
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: blob-report-electron-${{ matrix.shardIndex }}
+          name: blob-report-electron
           path: blob-report
           retention-days: 14
 
       - name: Upload junit report
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: junit-report-electron-${{ matrix.shardIndex }}
+          name: junit-report-electron
           path: test-results/junit.xml
 
   e2e-browser-tests:
-    runs-on: ubuntu-latest-4x
+    runs-on: ubuntu-latest-8x
     timeout-minutes: 50
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -188,8 +189,14 @@ jobs:
         env:
           POSITRON_PY_VER_SEL: 3.10.12
           POSITRON_R_VER_SEL: 4.4.0
+          CURRENTS_PROJECT_ID: ZOs5z2
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
+          CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+          COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }}
+          PWTEST_BLOB_DO_NOT_REMOVE: 1
+          CURRENTS_TAG: "chromium"
         id: browser-tests
-        run: DISPLAY=:10 npx playwright test --project e2e-browser --workers 1
+        run: DISPLAY=:10 npx playwright test --project e2e-browser --workers 2
 
       - name: Upload blob report
         if: ${{ !cancelled() }}

.github/workflows/positron-merge-to-branch.yml

@@ -70,8 +70,13 @@ jobs:
         env:
           POSITRON_PY_VER_SEL: 3.10.12
           POSITRON_R_VER_SEL: 4.4.0
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
+          CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+          COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }}
+          PWTEST_BLOB_DO_NOT_REMOVE: 1
+          CURRENTS_TAG: "electron,${{ inputs.e2e_grep }}"
         id: e2e-playwright-tests
-        run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 2 --grep="${{ env.E2E_GREP }}"
+        run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 2 --grep=${{ env.E2E_GREP }}
 
       - name: Upload Playwright Report to S3
         if: ${{ success() || failure() }}

.github/workflows/positron-windows-nightly.yml

@@ -102,23 +102,29 @@ jobs:
         env:
           POSITRON_PY_VER_SEL: 3.10.10
           POSITRON_R_VER_SEL: 4.4.0
+          CURRENTS_PROJECT_ID: ZOs5z2
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
+          CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+          COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }} # only works on push events
+          PWTEST_BLOB_DO_NOT_REMOVE: 1
+          CURRENTS_TAG: "electron,@win"
         if: ${{ !cancelled() }}
         id: e2e-win-electron-tests
-        run: npx playwright test --project e2e-electron --grep "@win" --workers 1
+        run: npx playwright test --project e2e-electron --grep "@win" --workers 2
 
       - name: Upload blob report
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: blob-report-electron-${{ matrix.shardIndex }}
+          name: blob-report-electron
           path: blob-report
           retention-days: 14
 
       - name: Upload junit report
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: junit-report-electron-${{ matrix.shardIndex }}
+          name: junit-report-electron
           path: test-results/junit.xml
 
   e2e-report:

package.json

@@ -46,7 +46,7 @@
     "update-localization-extension": "node build/npm/update-localization-extension.js",
     "e2e": "yarn e2e-electron",
     "e2e-electron": "npx playwright test --project e2e-electron",
-    "e2e-browser": "npx playwright test --project e2e-browser --workers 1",
+    "e2e-browser": "npx playwright test --project e2e-browser",
     "e2e-pr": "npx playwright test --project e2e-electron --grep @pr",
     "e2e-win": "npx playwright test --project e2e-electron --grep @win",
     "e2e-failed": "npx playwright test --last-failed",
@@ -129,6 +129,7 @@
     "yazl": "^2.4.3"
   },
   "devDependencies": {
+    "@currents/playwright": "^1.8.0",
     "@midleman/github-actions-reporter": "^1.9.5",
     "@playwright/test": "^1.49.0",
     "@swc/core": "1.3.62",

playwright.config.ts

@@ -6,6 +6,7 @@
 import { defineConfig } from '@playwright/test';
 import { CustomTestOptions } from './test/smoke/src/areas/positron/_test.setup';
 import type { GitHubActionOptions } from '@midleman/github-actions-reporter';
+import { currentsReporter } from '@currents/playwright';
 
 /**
  * See https://playwright.dev/docs/test-configuration.
@@ -37,7 +38,13 @@ export default defineConfig<CustomTestOptions>({
 				includeResults: ['fail', 'flaky']
 			}],
 			['junit', { outputFile: 'test-results/junit.xml' }],
-			['list'], ['html'], ['blob']
+			['list'], ['html'], ['blob'],
+			currentsReporter({
+				ciBuildId: process.env.CURRENTS_CI_BUILD_ID || Date.now().toString(),
+				recordKey: process.env.CURRENTS_RECORD_KEY || '',
+				projectId: 'ZOs5z2',
+				disableTitleTags: true,
+			}),
 		]
 		: [
 			['list'],

test/automation/src/playwrightBrowser.ts

@@ -42,55 +42,94 @@ async function launchServer(options: LaunchOptions) {
 		...process.env
 	};
 
-	const args = [
-		'--disable-telemetry',
-		'--disable-workspace-trust',
-		`--port=${port++}`,
-		'--enable-smoke-test-driver',
-		`--extensions-dir=${extensionsPath}`,
-		`--server-data-dir=${agentFolder}`,
-		'--accept-server-license-terms',
-		`--logsPath=${serverLogsPath}`,
-		// --- Start Positron ---
-		`--connection-token`,
-		`dev-token`
+	// --- Start Positron ---
+	// Adding support for multiple ports to enable parallel test execution
+
+	let serverProcess: ChildProcess | null = null;
+	let endpoint: string | undefined;
+
+	const maxRetries = 10; // Number of ports to try before giving up
+	for (let attempts = 0; attempts < maxRetries; attempts++) {
+		const currentPort = port++; // Increment the port on each retry
 		// --- End Positron ---
-	];
 
-	if (options.verbose) {
-		args.push('--log=trace');
-	}
+		const args = [
+			'--disable-telemetry',
+			'--disable-workspace-trust',
+			// --- Start Positron ---
+			`--port=${currentPort}`,
+			// --- End Positron ---
+			'--enable-smoke-test-driver',
+			`--extensions-dir=${extensionsPath}`,
+			`--server-data-dir=${agentFolder}`,
+			'--accept-server-license-terms',
+			`--logsPath=${serverLogsPath}`,
+			// --- Start Positron ---
+			`--connection-token`,
+			`dev-token`
+			// --- End Positron ---
+		];
 
-	let serverLocation: string | undefined;
-	if (codeServerPath) {
-		const { serverApplicationName } = require(join(codeServerPath, 'product.json'));
-		serverLocation = join(codeServerPath, 'bin', `${serverApplicationName}${process.platform === 'win32' ? '.cmd' : ''}`);
+		if (options.verbose) {
+			args.push('--log=trace');
+		}
 
-		logger.log(`Starting built server from '${serverLocation}'`);
-	} else {
-		serverLocation = join(root, `scripts/code-server.${process.platform === 'win32' ? 'bat' : 'sh'}`);
+		let serverLocation: string | undefined;
+		if (codeServerPath) {
+			const { serverApplicationName } = require(join(codeServerPath, 'product.json'));
+			serverLocation = join(codeServerPath, 'bin', `${serverApplicationName}${process.platform === 'win32' ? '.cmd' : ''}`);
 
-		logger.log(`Starting server out of sources from '${serverLocation}'`);
-	}
+			logger.log(`Starting built server from '${serverLocation}'`);
+		} else {
+			serverLocation = join(root, `scripts/code-server.${process.platform === 'win32' ? 'bat' : 'sh'}`);
 
-	logger.log(`Storing log files into '${serverLogsPath}'`);
+			logger.log(`Starting server out of sources from '${serverLocation}'`);
+		}
 
-	logger.log(`Command line: '${serverLocation}' ${args.join(' ')}`);
-	const shell: boolean = (process.platform === 'win32');
-	const serverProcess = spawn(
-		serverLocation,
-		args,
-		{ env, shell }
-	);
+		logger.log(`Storing log files into '${serverLogsPath}'`);
 
-	logger.log(`Started server for browser smoke tests (pid: ${serverProcess.pid})`);
+		logger.log(`Command line: '${serverLocation}' ${args.join(' ')}`);
+		const shell: boolean = (process.platform === 'win32');
 
-	return {
-		serverProcess,
-		endpoint: await measureAndLog(() => waitForEndpoint(serverProcess, logger), 'waitForEndpoint(serverProcess)', logger)
-	};
+		// --- Start Positron ---
+		try {
+			serverProcess = spawn(
+				serverLocation,
+				args,
+				{ env, shell }
+			);
+
+			logger.log(`Started server on port ${currentPort} (pid: ${serverProcess.pid})`);
+
+			endpoint = await measureAndLog(
+				() => waitForEndpoint(serverProcess!, logger),
+				'waitForEndpoint(serverProcess)',
+				logger
+			);
+
+			// If we reach here, the server started successfully
+			break;
+		} catch (error) {
+			if ((error as Error).message.includes('EADDRINUSE')) {
+				logger.log(`Port ${currentPort} is already in use. Retrying with the next port...`);
+				if (serverProcess) {
+					serverProcess.kill();
+				}
+			} else {
+				throw error; // Rethrow non-EADDRINUSE errors
+			}
+		}
+	}
+
+	if (!serverProcess || !endpoint) {
+		throw new Error('Failed to launch the server after multiple attempts.');
+	}
+
+	return { serverProcess, endpoint };
+	// --- End Positron ---
 }
 
+
 async function launchBrowser(options: LaunchOptions, endpoint: string) {
 	const { logger, workspacePath, tracing, headless } = options;
 

test/automation/src/positron/positronNotebooks.ts

@@ -82,7 +82,7 @@ export class PositronNotebooks {
 	}
 
 	async assertCellOutput(text: string): Promise<void> {
-		await expect(this.frameLocator.getByText(text)).toBeVisible();
+		await expect(this.frameLocator.getByText(text)).toBeVisible({ timeout: 15000 });
 	}
 
 	async closeNotebookWithoutSaving() {

test/automation/src/positron/positronVariables.ts

@@ -57,7 +57,6 @@ export class PositronVariables {
 	}
 
 	async doubleClickVariableRow(variableName: string) {
-
 		const desiredRow = await this.waitForVariableRow(variableName);
 		await desiredRow.dblclick();
 	}

test/smoke/src/areas/positron/_test.setup.ts

@@ -30,6 +30,7 @@ const TEMP_DIR = `temp-${randomUUID()}`;
 const ROOT_PATH = process.cwd();
 const LOGS_ROOT_PATH = join(ROOT_PATH, 'test-logs');
 let SPEC_NAME = '';
+let fixtureScreenshot: Buffer;
 
 export const test = base.extend<TestFixtures, WorkerFixtures>({
 	suiteId: ['', { scope: 'worker', option: true }],
@@ -78,17 +79,33 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
 		await use(app);
 	}, { scope: 'test', timeout: 60000 }],
 
-	app: [async ({ options, logsPath, logger }, use) => {
+	app: [async ({ options, logsPath }, use, workerInfo) => {
 		const app = createApp(options);
-		await app.start();
 
-		await use(app);
+		try {
+			await app.start();
+
+			await use(app);
+		} catch (error) {
+			// capture a screenshot on failure
+			const screenshotPath = path.join(logsPath, 'app-start-failure.png');
+			try {
+				const page = app.code?.driver?.page;
+				if (page) {
+					fixtureScreenshot = await page.screenshot({ path: screenshotPath });
+				}
+			} catch {
+				// ignore
+			}
 
-		await app.stop();
+			throw error; // re-throw the error to ensure test failure
+		} finally {
+			await app.stop();
 
-		// rename the temp logs dir to the spec name
-		const specLogsPath = path.join(path.dirname(logsPath), SPEC_NAME);
-		await moveAndOverwrite(logsPath, specLogsPath);
+			// rename the temp logs dir to the spec name (if available)
+			const specLogsPath = path.join(path.dirname(logsPath), SPEC_NAME || `worker-${workerInfo.workerIndex}`);
+			await moveAndOverwrite(logsPath, specLogsPath);
+		}
 	}, { scope: 'worker', auto: true, timeout: 60000 }],
 
 	interpreter: [async ({ app, page }, use) => {
@@ -268,9 +285,17 @@ test.beforeAll(async ({ logger }, testInfo) => {
 });
 
 test.afterAll(async function ({ logger }, testInfo) {
-	logger.log('');
-	logger.log(`>>> Suite end: '${testInfo.titlePath[0] ?? 'unknown'}' <<<`);
-	logger.log('');
+	try {
+		logger.log('');
+		logger.log(`>>> Suite end: '${testInfo.titlePath[0] ?? 'unknown'}' <<<`);
+		logger.log('');
+	} catch (error) {
+		// ignore
+	}
+
+	if (fixtureScreenshot) {
+		await testInfo.attach('on-fixture-fail', { body: fixtureScreenshot, contentType: 'image/png' });
+	}
 });
 
 export { playwrightExpect as expect };