Skip to content

Commit

Permalink
e2e test: pythonApplications.test.ts improvements (#5161)
Browse files Browse the repository at this point in the history 
I’ve made improvements to the Python application tests to reduce flaky
results. Currently, the tests are tagged with #pr and #web as I’m
running them multiple times to confirm stability. I’ll remove these tags
before merging. Additionally, I discovered a potential bug that only
affects web (python) tests. I’ll bring this up with the team but will
remove this one test from the web runs for now.

### QA Notes
✅ this test passed 5 consecutive runs on PR
✅ [Full test
suite](https://github.com/posit-dev/positron/actions/runs/11508968684)
passed
✅ [Windows
suite](https://github.com/posit-dev/positron/actions/runs/11521232269/job/32074401285)
passed
  • Loading branch information
@midleman
midleman authored Oct 25, 2024
1 parent b453786 commit 8cc2be8
Show file tree
Hide file tree
Showing 8 changed files with 46 additions and 104 deletions.
7 changes: 6 additions & 1 deletion test/automation/src/positron/fixtures/positronPythonFixtures.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import { fail } from 'assert';
import { Application } from '../../application';
import { InterpreterInfo, InterpreterType } from '../utils/positronInterpreterInfo';
import { expect } from '@playwright/test';

/*
* Reuseable Positron Python fixture tests can leverage to get a Python interpreter selected.
Expand All @@ -16,7 +17,11 @@ export class PositronPythonFixtures {

static async SetupFixtures(app: Application, skipReadinessCheck: boolean = false) {
const fixtures = new PositronPythonFixtures(app);
await fixtures.startPythonInterpreter(skipReadinessCheck);
await expect(async () => {
await fixtures.startPythonInterpreter(skipReadinessCheck);
}).toPass({
timeout: 60000
});
}

async startPythonInterpreter(skipReadinessCheck: boolean = false) {
Expand Down
3 changes: 2 additions & 1 deletion test/automation/src/positron/positronNotebooks.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ const NEW_NOTEBOOK_COMMAND = 'ipynb.newUntitledIpynb';
const CELL_LINE = '.cell div.view-lines';
const EXECUTE_CELL_COMMAND = 'notebook.cell.execute';
const EXECUTE_CELL_SPINNER = '.cell-status-item .codicon-modifier-spin';
const OUTER_FRAME = '.webview';
const INNER_FRAME = '#active-frame';
const REVERT_AND_CLOSE = 'workbench.action.revertAndCloseActiveEditor';
const MARKDOWN_TEXT = '#preview';
Expand All @@ -29,7 +30,7 @@ const ACTIVE_ROW_SELECTOR = `.notebook-editor .monaco-list-row.focused`;
*/
export class PositronNotebooks {
kernelLabel = this.code.driver.getLocator(KERNEL_LABEL);
frameLocator = this.code.driver.page.frameLocator('iframe').frameLocator(INNER_FRAME);
frameLocator = this.code.driver.page.frameLocator(OUTER_FRAME).frameLocator(INNER_FRAME);

constructor(private code: Code, private quickinput: QuickInput, private quickaccess: QuickAccess, private notebook: Notebook) { }

Expand Down
7 changes: 1 addition & 6 deletions test/automation/src/positron/positronVariables.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

import { Code } from '../code';
import * as os from 'os';
import { IElement } from '../driver';
import { expect, Locator } from '@playwright/test';

interface FlatVariables {
Expand All @@ -27,6 +26,7 @@ const VARIABLE_INDENTED = '.name-column-indenter[style*="margin-left: 40px"]';
* Reuseable Positron variables functionality for tests to leverage.
*/
export class PositronVariables {
interpreterLocator = this.code.driver.page.locator(VARIABLES_INTERPRETER);

constructor(private code: Code) { }

Expand Down Expand Up @@ -99,11 +99,6 @@ export class PositronVariables {
await this.toggleVariable({ variableName, action: 'collapse' });
}

async getVariablesInterpreter(): Promise<IElement> {
const interpreter = await this.code.waitForElement(VARIABLES_INTERPRETER);
return interpreter;
}

/**
* Gets the data (value and type) for the children of a parent variable.
* NOTE: it assumes that either ALL variables are collapsed or ONLY the parent variable is expanded.
Expand Down
21 changes: 5 additions & 16 deletions test/automation/src/positron/positronViewer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,16 @@ const FULL_APP = 'body';
export class PositronViewer {

fullApp = this.code.driver.getLocator(FULL_APP);
viewerFrame = this.code.driver.page.frameLocator(OUTER_FRAME).frameLocator(INNER_FRAME);

constructor(private code: Code) { }

getViewerLocator(sublocator: string, additionalNesting = false): Locator {
const outerFrame = this.code.driver.getFrame(OUTER_FRAME);
const innerFrame = outerFrame.frameLocator(INNER_FRAME);
if (!additionalNesting) {
const element = innerFrame.locator(sublocator);
return element;
} else {
const innerInnerFrame = innerFrame.frameLocator('//iframe');
const element = innerInnerFrame.locator(sublocator);
return element;
}
getViewerLocator(locator: string,): Locator {
return this.viewerFrame.locator(locator);
}

getViewerFrame(frameLocator: string): FrameLocator {
const outerFrame = this.code.driver.getFrame(OUTER_FRAME);
const innerFrame = outerFrame.frameLocator(INNER_FRAME);
const frame = innerFrame.frameLocator(frameLocator);
return frame;
getViewerFrame(): FrameLocator {
return this.viewerFrame;
}

async refreshViewer() {
Expand Down
41 changes: 13 additions & 28 deletions ...rc/areas/positron/apps/pythonApps.test.ts → ...c/areas/positron/apps/python-apps.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,11 @@ describe('Python Applications #pr #win', () => {

describe('Python Applications', () => {
before(async function () {
await this.app.workbench.positronConsole.waitForReadyOrNoInterpreter();

await PositronPythonFixtures.SetupFixtures(this.app as Application);
});

afterEach(async function () {
await this.app.workbench.quickaccess.runCommand('workbench.action.terminal.focus');
await this.app.workbench.positronTerminal.sendKeysToTerminal('Control+C');

// unreliable on ubuntu:
Expand All @@ -28,63 +27,49 @@ describe('Python Applications #pr #win', () => {
});

it('Python - Verify Basic Dash App [C903305]', async function () {

this.retries(1);

const app = this.app as Application;
const viewer = app.workbench.positronViewer;

await app.workbench.quickaccess.openFile(join(app.workspacePathOrFolder, 'workspaces', 'python_apps', 'dash_example', 'dash_example.py'));

await app.workbench.positronEditor.pressPlay();

const headerLocator = app.workbench.positronViewer.getViewerLocator('#_dash-app-content');

await expect(headerLocator).toHaveText('Hello World', { timeout: 45000 });

await expect(viewer.getViewerFrame().getByText('Hello World')).toBeVisible({ timeout: 30000 });
});

// https://github.com/posit-dev/positron/issues/4949
// FastAPI is not working as expected on Ubuntu
it.skip('Python - Verify Basic FastAPI App [C903306]', async function () {
const app = this.app as Application;
const viewer = app.workbench.positronViewer;

await app.workbench.quickaccess.openFile(join(app.workspacePathOrFolder, 'workspaces', 'python_apps', 'fastapi_example', 'fastapi_example.py'));

await app.workbench.positronEditor.pressPlay();

const headerLocator = app.workbench.positronViewer.getViewerLocator('h2.title');

await expect(headerLocator).toContainText('FastAPI', { timeout: 45000 });

await expect(viewer.getViewerFrame().getByText('FastAPI')).toBeVisible({ timeout: 30000 });
});

it('Python - Verify Basic Gradio App [C903307]', async function () {
const app = this.app as Application;
const viewer = app.workbench.positronViewer;

await app.workbench.quickaccess.openFile(join(app.workspacePathOrFolder, 'workspaces', 'python_apps', 'gradio_example', 'gradio_example.py'));

await app.workbench.positronEditor.pressPlay();

const headerLocator = app.workbench.positronViewer.getViewerLocator('button.primary');

await expect(headerLocator).toHaveText('Submit', { timeout: 45000 });

await expect(viewer.getViewerFrame().getByRole('button', { name: 'Submit' })).toBeVisible({ timeout: 30000 });
});

it('Python - Verify Basic Streamlit App [C903308] #web', async function () {
const app = this.app as Application;
const viewer = app.workbench.positronViewer;

await app.workbench.quickaccess.openFile(join(app.workspacePathOrFolder, 'workspaces', 'python_apps', 'streamlit_example', 'streamlit_example.py'));

await app.workbench.positronEditor.pressPlay();

const headerLocator = app.workbench.positronViewer.getViewerLocator('div.stAppDeployButton', this.app.web);

await app.workbench.positronLayouts.enterLayout('fullSizedAuxBar');
await expect(headerLocator).toHaveText('Deploy', { timeout: 45000 });
const viewerFrame = viewer.getViewerFrame();
const headerLocator = this.app.web
? viewerFrame.frameLocator('iframe').getByRole('button', { name: 'Deploy' })
: viewerFrame.getByRole('button', { name: 'Deploy' });
await expect(headerLocator).toBeVisible({ timeout: 30000 });
await app.workbench.positronLayouts.enterLayout('stacked');
await app.workbench.quickaccess.runCommand('workbench.action.terminal.focus');

});
});
});
Expand Down
6 changes: 3 additions & 3 deletions test/smoke/src/areas/positron/quarto/quarto.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,10 @@ describe('Quarto #web', () => {

it('should be able to generate preview [C842891]', async function () {
await app.workbench.quickaccess.runCommand('quarto.preview', { keepOpen: true });
const viewerFrame = app.workbench.positronViewer.getViewerFrame('//iframe');

// verify preview displays
expect(await viewerFrame.locator('h1').innerText()).toBe('Diamond sizes');
const previewHeader = app.workbench.positronViewer.getViewerFrame().frameLocator('iframe').locator('h1');
await expect(previewHeader).toBeVisible({ timeout: 20000 });
await expect(previewHeader).toHaveText('Diamond sizes');
});
});

Expand Down
13 changes: 3 additions & 10 deletions test/smoke/src/areas/positron/rmarkdown/rmarkdown.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,16 +45,9 @@ describe('RMarkdown #web', () => {

// inner most frame has no useful identifying features
// not factoring this locator because its not part of positron
const viewerFrame = app.workbench.positronViewer.getViewerFrame('//iframe');

// not factoring this locator because its not part of positron
const gettingStarted = viewerFrame.locator('h2[data-anchor-id="getting-started"]');

const gettingStartedText = await gettingStarted.innerText();

expect(gettingStartedText).toBe('Getting started');

await app.workbench.positronTerminal.sendKeysToTerminal('Control+C');
const gettingStarted = app.workbench.positronViewer.getViewerFrame().frameLocator('iframe').locator('h2[data-anchor-id="getting-started"]');

await expect(gettingStarted).toBeVisible({ timeout: 30000 });
await expect(gettingStarted).toHaveText('Getting started');
});
});
52 changes: 13 additions & 39 deletions test/smoke/src/areas/positron/variables/variables-notebook.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,86 +7,60 @@ import { expect } from '@playwright/test';
import { Application, PositronPythonFixtures, PositronRFixtures } from '../../../../../automation';
import { setupAndStartApp } from '../../../test-runner/test-hooks';

describe('Variables Pane - Notebook #pr #web', () => {
describe('Variables Pane - Notebook', () => {

describe('Python Notebook Variables Pane', () => {
// This test fails on WEB: https://github.com/posit-dev/positron/issues/2452
describe('Python Notebook Variables Pane #pr', () => {
setupAndStartApp();

before(async function () {
await PositronPythonFixtures.SetupFixtures(this.app as Application);
});

after(async function () {

const app = this.app as Application;
await app.workbench.positronNotebooks.closeNotebookWithoutSaving();

await app.workbench.positronLayouts.enterLayout('stacked');
});

it('Verifies Variables pane basic function for notebook with python interpreter [C669188]', async function () {
const app = this.app as Application;
await PositronPythonFixtures.SetupFixtures(this.app as Application);

await app.workbench.positronNotebooks.createNewNotebook();

await app.workbench.positronNotebooks.selectInterpreter('Python Environments', process.env.POSITRON_PY_VER_SEL!);

await app.workbench.positronNotebooks.addCodeToFirstCell('y = [2, 3, 4, 5]');

await app.workbench.positronNotebooks.executeCodeInCell();

const interpreter = await app.workbench.positronVariables.getVariablesInterpreter();

expect(interpreter.textContent).toBe('Untitled-1.ipynb');
const interpreter = app.workbench.positronVariables.interpreterLocator;
await expect(interpreter).toBeVisible();
await expect(interpreter).toHaveText('Untitled-1.ipynb');

await app.workbench.positronLayouts.enterLayout('fullSizedAuxBar');

const variablesMap = await app.workbench.positronVariables.getFlatVariables();

expect(variablesMap.get('y')).toStrictEqual({ value: '[2, 3, 4, 5]', type: 'list [4]' });

});

});


describe('R Notebook Variables Pane', () => {
describe('R Notebook Variables Pane #pr #web', () => {
setupAndStartApp();

before(async function () {
await PositronRFixtures.SetupFixtures(this.app as Application);
});

after(async function () {

const app = this.app as Application;
await app.workbench.positronNotebooks.closeNotebookWithoutSaving();

await app.workbench.positronLayouts.enterLayout('stacked');
});

it('Verifies Variables pane basic function for notebook with R interpreter [C669189]', async function () {
const app = this.app as Application;
await PositronRFixtures.SetupFixtures(this.app as Application);

await app.workbench.positronNotebooks.createNewNotebook();

await app.workbench.positronNotebooks.selectInterpreter('R Environments', process.env.POSITRON_R_VER_SEL!);

await app.workbench.positronNotebooks.addCodeToFirstCell('y <- c(2, 3, 4, 5)');

await app.workbench.positronNotebooks.executeCodeInCell();

const interpreter = await app.workbench.positronVariables.getVariablesInterpreter();

expect(interpreter.textContent).toBe('Untitled-1.ipynb');
const interpreter = app.workbench.positronVariables.interpreterLocator;
await expect(interpreter).toBeVisible();
await expect(interpreter).toHaveText('Untitled-1.ipynb');

await app.workbench.positronLayouts.enterLayout('fullSizedAuxBar');

const variablesMap = await app.workbench.positronVariables.getFlatVariables();

expect(variablesMap.get('y')).toStrictEqual({ value: '2 3 4 5', type: 'dbl [4]' });

});

});
});

0 comments on commit 8cc2be8

Please sign in to comment.