e2e-test: browser in parallel & enable Currents (#5442)

### Summary This PR enables the Currents dashboard integration to evaluate the tool’s usefulness. Additionally, the browser is now configured to run on multiple ports, allowing browser tests to execute in parallel. I also consolidated matrix runs into a single machine and enabled multiple threads/workers. This adjustment seems to align better with the current setup and is likely more cost-efficient. ### QA Notes I performed numerous test runs to monitor the potential for increased flakiness with parallel workers. The results are promising overall. While I observed some minor flakiness, these issues appear manageable and can be addressed with further adjustments.
posit-dev · Nov 27, 2024 · c5ce275 · c5ce275
1 parent 7a118ad
commit c5ce275
Show file tree

Hide file tree

Showing 14 changed files with 775 additions and 74 deletions.
diff --git a/.github/workflows/e2e-test-release-run-ubuntu.yml b/.github/workflows/e2e-test-release-run-ubuntu.yml
@@ -88,10 +88,15 @@ jobs:
         env:
           POSITRON_PY_VER_SEL: 3.10.12
           POSITRON_R_VER_SEL: 4.4.0
-        id: electron-smoke-tests
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
+          CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+          COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }}
+          PWTEST_BLOB_DO_NOT_REMOVE: 1
+          CURRENTS_TAG: "electron,release,${{ inputs.e2e_grep }}"
+        id: electron-e2e-tests
         run: |
           export DISPLAY=:10
-          BUILD=/usr/share/positron npx playwright test --project e2e-electron --workers 2 --grep="${{ env.E2E_GREP }}"
+          BUILD=/usr/share/positron npx playwright test --project e2e-electron --workers 3 --grep=${{ env.E2E_GREP }}
 
       - name: Upload Playwright Report to S3
         if: ${{ !cancelled() }}

diff --git a/.github/workflows/positron-full-test.yml b/.github/workflows/positron-full-test.yml
@@ -96,11 +96,6 @@ jobs:
   e2e-electron-tests:
     runs-on: ubuntu-latest-8x
     timeout-minutes: 60
-    strategy:
-      fail-fast: false
-      matrix:
-        shardIndex: [1, 2]
-        shardTotal: [2]
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       POSITRON_BUILD_NUMBER: 0 # CI skips building releases
@@ -131,26 +126,32 @@ jobs:
         env:
           POSITRON_PY_VER_SEL: 3.10.12
           POSITRON_R_VER_SEL: 4.4.0
+          CURRENTS_PROJECT_ID: ZOs5z2
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
+          CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+          COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }} # only works on push events
+          PWTEST_BLOB_DO_NOT_REMOVE: 1
+          CURRENTS_TAG: "electron"
         id: electron-tests
-        run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 1 --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 3
 
       - name: Upload blob report
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: blob-report-electron-${{ matrix.shardIndex }}
+          name: blob-report-electron
           path: blob-report
           retention-days: 14
 
       - name: Upload junit report
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: junit-report-electron-${{ matrix.shardIndex }}
+          name: junit-report-electron
           path: test-results/junit.xml
 
   e2e-browser-tests:
-    runs-on: ubuntu-latest-4x
+    runs-on: ubuntu-latest-8x
     timeout-minutes: 50
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -188,8 +189,14 @@ jobs:
         env:
           POSITRON_PY_VER_SEL: 3.10.12
           POSITRON_R_VER_SEL: 4.4.0
+          CURRENTS_PROJECT_ID: ZOs5z2
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
+          CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+          COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }}
+          PWTEST_BLOB_DO_NOT_REMOVE: 1
+          CURRENTS_TAG: "chromium"
         id: browser-tests
-        run: DISPLAY=:10 npx playwright test --project e2e-browser --workers 1
+        run: DISPLAY=:10 npx playwright test --project e2e-browser --workers 2
 
       - name: Upload blob report
         if: ${{ !cancelled() }}

diff --git a/.github/workflows/positron-merge-to-branch.yml b/.github/workflows/positron-merge-to-branch.yml
@@ -70,8 +70,13 @@ jobs:
         env:
           POSITRON_PY_VER_SEL: 3.10.12
           POSITRON_R_VER_SEL: 4.4.0
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
+          CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+          COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }}
+          PWTEST_BLOB_DO_NOT_REMOVE: 1
+          CURRENTS_TAG: "electron,${{ inputs.e2e_grep }}"
         id: e2e-playwright-tests
-        run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 2 --grep="${{ env.E2E_GREP }}"
+        run: DISPLAY=:10 npx playwright test --project e2e-electron --workers 2 --grep=${{ env.E2E_GREP }}
 
       - name: Upload Playwright Report to S3
         if: ${{ success() || failure() }}

diff --git a/.github/workflows/positron-windows-nightly.yml b/.github/workflows/positron-windows-nightly.yml
@@ -102,23 +102,29 @@ jobs:
         env:
           POSITRON_PY_VER_SEL: 3.10.10
           POSITRON_R_VER_SEL: 4.4.0
+          CURRENTS_PROJECT_ID: ZOs5z2
+          CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
+          CURRENTS_CI_BUILD_ID: ${{ github.run_id }}-${{ github.run_attempt }}
+          COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }} # only works on push events
+          PWTEST_BLOB_DO_NOT_REMOVE: 1
+          CURRENTS_TAG: "electron,@win"
         if: ${{ !cancelled() }}
         id: e2e-win-electron-tests
-        run: npx playwright test --project e2e-electron --grep "@win" --workers 1
+        run: npx playwright test --project e2e-electron --grep "@win" --workers 2
 
       - name: Upload blob report
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: blob-report-electron-${{ matrix.shardIndex }}
+          name: blob-report-electron
           path: blob-report
           retention-days: 14
 
       - name: Upload junit report
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: junit-report-electron-${{ matrix.shardIndex }}
+          name: junit-report-electron
           path: test-results/junit.xml
 
   e2e-report:

diff --git a/package.json b/package.json
@@ -46,7 +46,7 @@
     "update-localization-extension": "node build/npm/update-localization-extension.js",
     "e2e": "yarn e2e-electron",
     "e2e-electron": "npx playwright test --project e2e-electron",
-    "e2e-browser": "npx playwright test --project e2e-browser --workers 1",
+    "e2e-browser": "npx playwright test --project e2e-browser",
     "e2e-pr": "npx playwright test --project e2e-electron --grep @pr",
     "e2e-win": "npx playwright test --project e2e-electron --grep @win",
     "e2e-failed": "npx playwright test --last-failed",
@@ -129,6 +129,7 @@
     "yazl": "^2.4.3"
   },
   "devDependencies": {
+    "@currents/playwright": "^1.8.0",
     "@midleman/github-actions-reporter": "^1.9.5",
     "@playwright/test": "^1.49.0",
     "@swc/core": "1.3.62",

diff --git a/playwright.config.ts b/playwright.config.ts
@@ -6,6 +6,7 @@
 import { defineConfig } from '@playwright/test';
 import { CustomTestOptions } from './test/smoke/src/areas/positron/_test.setup';
 import type { GitHubActionOptions } from '@midleman/github-actions-reporter';
+import { currentsReporter } from '@currents/playwright';
 
 /**
  * See https://playwright.dev/docs/test-configuration.
@@ -37,7 +38,13 @@ export default defineConfig<CustomTestOptions>({
 				includeResults: ['fail', 'flaky']
 			}],
 			['junit', { outputFile: 'test-results/junit.xml' }],
-			['list'], ['html'], ['blob']
+			['list'], ['html'], ['blob'],
+			currentsReporter({
+				ciBuildId: process.env.CURRENTS_CI_BUILD_ID || Date.now().toString(),
+				recordKey: process.env.CURRENTS_RECORD_KEY || '',
+				projectId: 'ZOs5z2',
+				disableTitleTags: true,
+			}),
 		]
 		: [
 			['list'],

diff --git a/test/automation/src/playwrightBrowser.ts b/test/automation/src/playwrightBrowser.ts
@@ -30,66 +30,115 @@ export async function launch(options: LaunchOptions): Promise<{ serverProcess: C
 	};
 }
 
+// --- Start Positron ---
+// Modified `launchServer` function to add support for multiple ports to enable parallel test
+// execution of browser tests. Also added helper functions: `getServerArgs`, `resolveServerLocation`,
+// and `startServer` to make this code easier to read.
 async function launchServer(options: LaunchOptions) {
 	const { userDataDir, codePath, extensionsPath, logger, logsPath } = options;
 	const serverLogsPath = join(logsPath, 'server');
 	const codeServerPath = codePath ?? process.env.VSCODE_REMOTE_SERVER_PATH;
 	const agentFolder = userDataDir;
+
 	await measureAndLog(() => mkdirp(agentFolder), `mkdirp(${agentFolder})`, logger);
 
 	const env = {
 		VSCODE_REMOTE_SERVER_PATH: codeServerPath,
-		...process.env
+		...process.env,
 	};
 
+	const maxRetries = 10;
+	let serverProcess: ChildProcess | null = null;
+	let endpoint: string | undefined;
+
+	for (let attempts = 0; attempts < maxRetries; attempts++) {
+		const currentPort = port++;
+		const args = getServerArgs(currentPort, extensionsPath, agentFolder, serverLogsPath, options.verbose);
+		const serverLocation = resolveServerLocation(codeServerPath, logger);
+
+		logger.log(`Attempting to start server on port ${currentPort}`);
+		logger.log(`Command: '${serverLocation}' ${args.join(' ')}`);
+
+		try {
+			serverProcess = await startServer(serverLocation, args, env, logger);
+			endpoint = await measureAndLog(
+				() => waitForEndpoint(serverProcess!, logger),
+				'waitForEndpoint(serverProcess)',
+				logger
+			);
+
+			logger.log(`Server started successfully on port ${currentPort}`);
+			break; // Exit loop on success
+		} catch (error) {
+			if ((error as Error).message.includes('EADDRINUSE')) {
+				logger.log(`Port ${currentPort} is already in use. Retrying...`);
+				serverProcess?.kill();
+			} else {
+				throw error; // Rethrow non-port-related errors
+			}
+		}
+	}
+
+	if (!serverProcess || !endpoint) {
+		throw new Error('Failed to launch the server after multiple attempts.');
+	}
+
+	return { serverProcess, endpoint };
+}
+
+function getServerArgs(
+	port: number,
+	extensionsPath: string,
+	agentFolder: string,
+	logsPath: string,
+	verbose?: boolean
+): string[] {
 	const args = [
 		'--disable-telemetry',
 		'--disable-workspace-trust',
-		`--port=${port++}`,
+		`--port=${port}`,
 		'--enable-smoke-test-driver',
 		`--extensions-dir=${extensionsPath}`,
 		`--server-data-dir=${agentFolder}`,
 		'--accept-server-license-terms',
-		`--logsPath=${serverLogsPath}`,
-		// --- Start Positron ---
-		`--connection-token`,
-		`dev-token`
-		// --- End Positron ---
+		`--logsPath=${logsPath}`,
+		'--connection-token',
+		'dev-token',
 	];
 
-	if (options.verbose) {
+	if (verbose) {
 		args.push('--log=trace');
 	}
 
-	let serverLocation: string | undefined;
+	return args;
+}
+
+function resolveServerLocation(codeServerPath: string | undefined, logger: Logger): string {
 	if (codeServerPath) {
 		const { serverApplicationName } = require(join(codeServerPath, 'product.json'));
-		serverLocation = join(codeServerPath, 'bin', `${serverApplicationName}${process.platform === 'win32' ? '.cmd' : ''}`);
-
-		logger.log(`Starting built server from '${serverLocation}'`);
-	} else {
-		serverLocation = join(root, `scripts/code-server.${process.platform === 'win32' ? 'bat' : 'sh'}`);
-
-		logger.log(`Starting server out of sources from '${serverLocation}'`);
+		const serverLocation = join(codeServerPath, 'bin', `${serverApplicationName}${process.platform === 'win32' ? '.cmd' : ''}`);
+		logger.log(`Using built server from '${serverLocation}'`);
+		return serverLocation;
 	}
 
-	logger.log(`Storing log files into '${serverLogsPath}'`);
-
-	logger.log(`Command line: '${serverLocation}' ${args.join(' ')}`);
-	const shell: boolean = (process.platform === 'win32');
-	const serverProcess = spawn(
-		serverLocation,
-		args,
-		{ env, shell }
-	);
-
-	logger.log(`Started server for browser smoke tests (pid: ${serverProcess.pid})`);
+	const scriptPath = join(root, `scripts/code-server.${process.platform === 'win32' ? 'bat' : 'sh'}`);
+	logger.log(`Using source server from '${scriptPath}'`);
+	return scriptPath;
+}
 
-	return {
-		serverProcess,
-		endpoint: await measureAndLog(() => waitForEndpoint(serverProcess, logger), 'waitForEndpoint(serverProcess)', logger)
-	};
+async function startServer(
+	serverLocation: string,
+	args: string[],
+	env: NodeJS.ProcessEnv,
+	logger: Logger
+): Promise<ChildProcess> {
+	logger.log(`Starting server: ${serverLocation}`);
+	const serverProcess = spawn(serverLocation, args, { env, shell: process.platform === 'win32' });
+	logger.log(`Server started (pid: ${serverProcess.pid})`);
+	return serverProcess;
 }
+// --- End Positron ---
+
 
 async function launchBrowser(options: LaunchOptions, endpoint: string) {
 	const { logger, workspacePath, tracing, headless } = options;

diff --git a/test/automation/src/positron/positronNotebooks.ts b/test/automation/src/positron/positronNotebooks.ts
@@ -82,7 +82,7 @@ export class PositronNotebooks {
 	}
 
 	async assertCellOutput(text: string): Promise<void> {
-		await expect(this.frameLocator.getByText(text)).toBeVisible();
+		await expect(this.frameLocator.getByText(text)).toBeVisible({ timeout: 15000 });
 	}
 
 	async closeNotebookWithoutSaving() {

diff --git a/test/automation/src/positron/positronVariables.ts b/test/automation/src/positron/positronVariables.ts
@@ -57,7 +57,6 @@ export class PositronVariables {
 	}
 
 	async doubleClickVariableRow(variableName: string) {
-
 		const desiredRow = await this.waitForVariableRow(variableName);
 		await desiredRow.dblclick();
 	}
-Original file line number
+Diff line change
@@ Expand Up / @@ -57,7 +57,6 @@ export class PositronVariables { @@
     	}
     	async doubleClickVariableRow(variableName: string) {
     		const desiredRow = await this.waitForVariableRow(variableName);
     		await desiredRow.dblclick();
     	}
@@ Expand Down @@