From 8cc2be8de2689686da2b0a23390631f2bd1f1863 Mon Sep 17 00:00:00 2001 From: Marie Idleman Date: Fri, 25 Oct 2024 13:41:41 -0500 Subject: [PATCH] e2e test: `pythonApplications.test.ts` improvements (#5161) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I’ve made improvements to the Python application tests to reduce flaky results. Currently, the tests are tagged with #pr and #web as I’m running them multiple times to confirm stability. I’ll remove these tags before merging. Additionally, I discovered a potential bug that only affects web (python) tests. I’ll bring this up with the team but will remove this one test from the web runs for now. ### QA Notes ✅ this test passed 5 consecutive runs on PR ✅ [Full test suite](https://github.com/posit-dev/positron/actions/runs/11508968684) passed ✅ [Windows suite](https://github.com/posit-dev/positron/actions/runs/11521232269/job/32074401285) passed --- .../fixtures/positronPythonFixtures.ts | 7 ++- .../src/positron/positronNotebooks.ts | 3 +- .../src/positron/positronVariables.ts | 7 +-- .../automation/src/positron/positronViewer.ts | 21 ++------ ...pythonApps.test.ts => python-apps.test.ts} | 41 +++++---------- .../src/areas/positron/quarto/quarto.test.ts | 6 +-- .../positron/rmarkdown/rmarkdown.test.ts | 13 ++--- .../variables/variables-notebook.test.ts | 52 +++++-------------- 8 files changed, 46 insertions(+), 104 deletions(-) rename test/smoke/src/areas/positron/apps/{pythonApps.test.ts => python-apps.test.ts} (75%) diff --git a/test/automation/src/positron/fixtures/positronPythonFixtures.ts b/test/automation/src/positron/fixtures/positronPythonFixtures.ts index 646cdf25109..2c51ec7708a 100644 --- a/test/automation/src/positron/fixtures/positronPythonFixtures.ts +++ b/test/automation/src/positron/fixtures/positronPythonFixtures.ts @@ -6,6 +6,7 @@ import { fail } from 'assert'; import { Application } from '../../application'; import { InterpreterInfo, InterpreterType } from '../utils/positronInterpreterInfo'; +import { expect } from '@playwright/test'; /* * Reuseable Positron Python fixture tests can leverage to get a Python interpreter selected. @@ -16,7 +17,11 @@ export class PositronPythonFixtures { static async SetupFixtures(app: Application, skipReadinessCheck: boolean = false) { const fixtures = new PositronPythonFixtures(app); - await fixtures.startPythonInterpreter(skipReadinessCheck); + await expect(async () => { + await fixtures.startPythonInterpreter(skipReadinessCheck); + }).toPass({ + timeout: 60000 + }); } async startPythonInterpreter(skipReadinessCheck: boolean = false) { diff --git a/test/automation/src/positron/positronNotebooks.ts b/test/automation/src/positron/positronNotebooks.ts index f7406fa1ec1..819f8f72977 100644 --- a/test/automation/src/positron/positronNotebooks.ts +++ b/test/automation/src/positron/positronNotebooks.ts @@ -18,6 +18,7 @@ const NEW_NOTEBOOK_COMMAND = 'ipynb.newUntitledIpynb'; const CELL_LINE = '.cell div.view-lines'; const EXECUTE_CELL_COMMAND = 'notebook.cell.execute'; const EXECUTE_CELL_SPINNER = '.cell-status-item .codicon-modifier-spin'; +const OUTER_FRAME = '.webview'; const INNER_FRAME = '#active-frame'; const REVERT_AND_CLOSE = 'workbench.action.revertAndCloseActiveEditor'; const MARKDOWN_TEXT = '#preview'; @@ -29,7 +30,7 @@ const ACTIVE_ROW_SELECTOR = `.notebook-editor .monaco-list-row.focused`; */ export class PositronNotebooks { kernelLabel = this.code.driver.getLocator(KERNEL_LABEL); - frameLocator = this.code.driver.page.frameLocator('iframe').frameLocator(INNER_FRAME); + frameLocator = this.code.driver.page.frameLocator(OUTER_FRAME).frameLocator(INNER_FRAME); constructor(private code: Code, private quickinput: QuickInput, private quickaccess: QuickAccess, private notebook: Notebook) { } diff --git a/test/automation/src/positron/positronVariables.ts b/test/automation/src/positron/positronVariables.ts index acfd3eec560..a4bfddbb0b7 100644 --- a/test/automation/src/positron/positronVariables.ts +++ b/test/automation/src/positron/positronVariables.ts @@ -6,7 +6,6 @@ import { Code } from '../code'; import * as os from 'os'; -import { IElement } from '../driver'; import { expect, Locator } from '@playwright/test'; interface FlatVariables { @@ -27,6 +26,7 @@ const VARIABLE_INDENTED = '.name-column-indenter[style*="margin-left: 40px"]'; * Reuseable Positron variables functionality for tests to leverage. */ export class PositronVariables { + interpreterLocator = this.code.driver.page.locator(VARIABLES_INTERPRETER); constructor(private code: Code) { } @@ -99,11 +99,6 @@ export class PositronVariables { await this.toggleVariable({ variableName, action: 'collapse' }); } - async getVariablesInterpreter(): Promise { - const interpreter = await this.code.waitForElement(VARIABLES_INTERPRETER); - return interpreter; - } - /** * Gets the data (value and type) for the children of a parent variable. * NOTE: it assumes that either ALL variables are collapsed or ONLY the parent variable is expanded. diff --git a/test/automation/src/positron/positronViewer.ts b/test/automation/src/positron/positronViewer.ts index 7c6eafac04d..c00142ff640 100644 --- a/test/automation/src/positron/positronViewer.ts +++ b/test/automation/src/positron/positronViewer.ts @@ -15,27 +15,16 @@ const FULL_APP = 'body'; export class PositronViewer { fullApp = this.code.driver.getLocator(FULL_APP); + viewerFrame = this.code.driver.page.frameLocator(OUTER_FRAME).frameLocator(INNER_FRAME); constructor(private code: Code) { } - getViewerLocator(sublocator: string, additionalNesting = false): Locator { - const outerFrame = this.code.driver.getFrame(OUTER_FRAME); - const innerFrame = outerFrame.frameLocator(INNER_FRAME); - if (!additionalNesting) { - const element = innerFrame.locator(sublocator); - return element; - } else { - const innerInnerFrame = innerFrame.frameLocator('//iframe'); - const element = innerInnerFrame.locator(sublocator); - return element; - } + getViewerLocator(locator: string,): Locator { + return this.viewerFrame.locator(locator); } - getViewerFrame(frameLocator: string): FrameLocator { - const outerFrame = this.code.driver.getFrame(OUTER_FRAME); - const innerFrame = outerFrame.frameLocator(INNER_FRAME); - const frame = innerFrame.frameLocator(frameLocator); - return frame; + getViewerFrame(): FrameLocator { + return this.viewerFrame; } async refreshViewer() { diff --git a/test/smoke/src/areas/positron/apps/pythonApps.test.ts b/test/smoke/src/areas/positron/apps/python-apps.test.ts similarity index 75% rename from test/smoke/src/areas/positron/apps/pythonApps.test.ts rename to test/smoke/src/areas/positron/apps/python-apps.test.ts index 7760e8dadc0..136411fb36f 100644 --- a/test/smoke/src/areas/positron/apps/pythonApps.test.ts +++ b/test/smoke/src/areas/positron/apps/python-apps.test.ts @@ -13,12 +13,11 @@ describe('Python Applications #pr #win', () => { describe('Python Applications', () => { before(async function () { - await this.app.workbench.positronConsole.waitForReadyOrNoInterpreter(); - await PositronPythonFixtures.SetupFixtures(this.app as Application); }); afterEach(async function () { + await this.app.workbench.quickaccess.runCommand('workbench.action.terminal.focus'); await this.app.workbench.positronTerminal.sendKeysToTerminal('Control+C'); // unreliable on ubuntu: @@ -28,63 +27,49 @@ describe('Python Applications #pr #win', () => { }); it('Python - Verify Basic Dash App [C903305]', async function () { - this.retries(1); - const app = this.app as Application; + const viewer = app.workbench.positronViewer; await app.workbench.quickaccess.openFile(join(app.workspacePathOrFolder, 'workspaces', 'python_apps', 'dash_example', 'dash_example.py')); - await app.workbench.positronEditor.pressPlay(); - - const headerLocator = app.workbench.positronViewer.getViewerLocator('#_dash-app-content'); - - await expect(headerLocator).toHaveText('Hello World', { timeout: 45000 }); - + await expect(viewer.getViewerFrame().getByText('Hello World')).toBeVisible({ timeout: 30000 }); }); // https://github.com/posit-dev/positron/issues/4949 // FastAPI is not working as expected on Ubuntu it.skip('Python - Verify Basic FastAPI App [C903306]', async function () { const app = this.app as Application; + const viewer = app.workbench.positronViewer; await app.workbench.quickaccess.openFile(join(app.workspacePathOrFolder, 'workspaces', 'python_apps', 'fastapi_example', 'fastapi_example.py')); - await app.workbench.positronEditor.pressPlay(); - - const headerLocator = app.workbench.positronViewer.getViewerLocator('h2.title'); - - await expect(headerLocator).toContainText('FastAPI', { timeout: 45000 }); - + await expect(viewer.getViewerFrame().getByText('FastAPI')).toBeVisible({ timeout: 30000 }); }); it('Python - Verify Basic Gradio App [C903307]', async function () { const app = this.app as Application; + const viewer = app.workbench.positronViewer; await app.workbench.quickaccess.openFile(join(app.workspacePathOrFolder, 'workspaces', 'python_apps', 'gradio_example', 'gradio_example.py')); - await app.workbench.positronEditor.pressPlay(); - - const headerLocator = app.workbench.positronViewer.getViewerLocator('button.primary'); - - await expect(headerLocator).toHaveText('Submit', { timeout: 45000 }); - + await expect(viewer.getViewerFrame().getByRole('button', { name: 'Submit' })).toBeVisible({ timeout: 30000 }); }); it('Python - Verify Basic Streamlit App [C903308] #web', async function () { const app = this.app as Application; + const viewer = app.workbench.positronViewer; await app.workbench.quickaccess.openFile(join(app.workspacePathOrFolder, 'workspaces', 'python_apps', 'streamlit_example', 'streamlit_example.py')); - await app.workbench.positronEditor.pressPlay(); - const headerLocator = app.workbench.positronViewer.getViewerLocator('div.stAppDeployButton', this.app.web); - await app.workbench.positronLayouts.enterLayout('fullSizedAuxBar'); - await expect(headerLocator).toHaveText('Deploy', { timeout: 45000 }); + const viewerFrame = viewer.getViewerFrame(); + const headerLocator = this.app.web + ? viewerFrame.frameLocator('iframe').getByRole('button', { name: 'Deploy' }) + : viewerFrame.getByRole('button', { name: 'Deploy' }); + await expect(headerLocator).toBeVisible({ timeout: 30000 }); await app.workbench.positronLayouts.enterLayout('stacked'); - await app.workbench.quickaccess.runCommand('workbench.action.terminal.focus'); - }); }); }); diff --git a/test/smoke/src/areas/positron/quarto/quarto.test.ts b/test/smoke/src/areas/positron/quarto/quarto.test.ts index 7548da6e03e..7db51dfa29a 100644 --- a/test/smoke/src/areas/positron/quarto/quarto.test.ts +++ b/test/smoke/src/areas/positron/quarto/quarto.test.ts @@ -45,10 +45,10 @@ describe('Quarto #web', () => { it('should be able to generate preview [C842891]', async function () { await app.workbench.quickaccess.runCommand('quarto.preview', { keepOpen: true }); - const viewerFrame = app.workbench.positronViewer.getViewerFrame('//iframe'); - // verify preview displays - expect(await viewerFrame.locator('h1').innerText()).toBe('Diamond sizes'); + const previewHeader = app.workbench.positronViewer.getViewerFrame().frameLocator('iframe').locator('h1'); + await expect(previewHeader).toBeVisible({ timeout: 20000 }); + await expect(previewHeader).toHaveText('Diamond sizes'); }); }); diff --git a/test/smoke/src/areas/positron/rmarkdown/rmarkdown.test.ts b/test/smoke/src/areas/positron/rmarkdown/rmarkdown.test.ts index 1bcf83a28db..109e8e2d907 100644 --- a/test/smoke/src/areas/positron/rmarkdown/rmarkdown.test.ts +++ b/test/smoke/src/areas/positron/rmarkdown/rmarkdown.test.ts @@ -45,16 +45,9 @@ describe('RMarkdown #web', () => { // inner most frame has no useful identifying features // not factoring this locator because its not part of positron - const viewerFrame = app.workbench.positronViewer.getViewerFrame('//iframe'); - - // not factoring this locator because its not part of positron - const gettingStarted = viewerFrame.locator('h2[data-anchor-id="getting-started"]'); - - const gettingStartedText = await gettingStarted.innerText(); - - expect(gettingStartedText).toBe('Getting started'); - - await app.workbench.positronTerminal.sendKeysToTerminal('Control+C'); + const gettingStarted = app.workbench.positronViewer.getViewerFrame().frameLocator('iframe').locator('h2[data-anchor-id="getting-started"]'); + await expect(gettingStarted).toBeVisible({ timeout: 30000 }); + await expect(gettingStarted).toHaveText('Getting started'); }); }); diff --git a/test/smoke/src/areas/positron/variables/variables-notebook.test.ts b/test/smoke/src/areas/positron/variables/variables-notebook.test.ts index 1a01e251bda..d3329922fc0 100644 --- a/test/smoke/src/areas/positron/variables/variables-notebook.test.ts +++ b/test/smoke/src/areas/positron/variables/variables-notebook.test.ts @@ -7,86 +7,60 @@ import { expect } from '@playwright/test'; import { Application, PositronPythonFixtures, PositronRFixtures } from '../../../../../automation'; import { setupAndStartApp } from '../../../test-runner/test-hooks'; -describe('Variables Pane - Notebook #pr #web', () => { +describe('Variables Pane - Notebook', () => { - describe('Python Notebook Variables Pane', () => { + // This test fails on WEB: https://github.com/posit-dev/positron/issues/2452 + describe('Python Notebook Variables Pane #pr', () => { setupAndStartApp(); before(async function () { await PositronPythonFixtures.SetupFixtures(this.app as Application); }); - after(async function () { - - const app = this.app as Application; - await app.workbench.positronNotebooks.closeNotebookWithoutSaving(); - - await app.workbench.positronLayouts.enterLayout('stacked'); - }); - it('Verifies Variables pane basic function for notebook with python interpreter [C669188]', async function () { const app = this.app as Application; + await PositronPythonFixtures.SetupFixtures(this.app as Application); await app.workbench.positronNotebooks.createNewNotebook(); - await app.workbench.positronNotebooks.selectInterpreter('Python Environments', process.env.POSITRON_PY_VER_SEL!); - await app.workbench.positronNotebooks.addCodeToFirstCell('y = [2, 3, 4, 5]'); - await app.workbench.positronNotebooks.executeCodeInCell(); - const interpreter = await app.workbench.positronVariables.getVariablesInterpreter(); - - expect(interpreter.textContent).toBe('Untitled-1.ipynb'); + const interpreter = app.workbench.positronVariables.interpreterLocator; + await expect(interpreter).toBeVisible(); + await expect(interpreter).toHaveText('Untitled-1.ipynb'); await app.workbench.positronLayouts.enterLayout('fullSizedAuxBar'); - const variablesMap = await app.workbench.positronVariables.getFlatVariables(); - expect(variablesMap.get('y')).toStrictEqual({ value: '[2, 3, 4, 5]', type: 'list [4]' }); - }); - }); - - describe('R Notebook Variables Pane', () => { + describe('R Notebook Variables Pane #pr #web', () => { setupAndStartApp(); before(async function () { await PositronRFixtures.SetupFixtures(this.app as Application); }); - after(async function () { - - const app = this.app as Application; - await app.workbench.positronNotebooks.closeNotebookWithoutSaving(); - - await app.workbench.positronLayouts.enterLayout('stacked'); - }); - it('Verifies Variables pane basic function for notebook with R interpreter [C669189]', async function () { const app = this.app as Application; + await PositronRFixtures.SetupFixtures(this.app as Application); await app.workbench.positronNotebooks.createNewNotebook(); - await app.workbench.positronNotebooks.selectInterpreter('R Environments', process.env.POSITRON_R_VER_SEL!); - await app.workbench.positronNotebooks.addCodeToFirstCell('y <- c(2, 3, 4, 5)'); - await app.workbench.positronNotebooks.executeCodeInCell(); - const interpreter = await app.workbench.positronVariables.getVariablesInterpreter(); - - expect(interpreter.textContent).toBe('Untitled-1.ipynb'); + const interpreter = app.workbench.positronVariables.interpreterLocator; + await expect(interpreter).toBeVisible(); + await expect(interpreter).toHaveText('Untitled-1.ipynb'); await app.workbench.positronLayouts.enterLayout('fullSizedAuxBar'); - const variablesMap = await app.workbench.positronVariables.getFlatVariables(); - expect(variablesMap.get('y')).toStrictEqual({ value: '2 3 4 5', type: 'dbl [4]' }); - }); }); }); +