Start ui-tests (#167)

* Start `ui-tests`

* Add workflows

* Update gitignore

* Update workflow

* Add basic UI tests config

* Add basic execute test

* Install lab in workflow

* setup conda env instead

* Add defaults

* Lint step

* ignore ui-tests folder for now

* Fix linting later

* Remove duplicated lint step

* Fix artifact name

* Lint

* Fix ruff

* Create notebook

* Add workaround for firefox

* Add launcher snapshot

* Add filesystem test

* Update snapshots

* Increase test timeout

* Only screenshot the launcher

* update refs

.eslintignore

@@ -5,3 +5,6 @@ coverage
 tests
 src/worker.ts
 src/web_worker_kernel.ts
+
+# TODO: remove
+ui-tests

.github/workflows/build.yml

@@ -49,7 +49,7 @@ jobs:
 
       - uses: actions/upload-artifact@v2
         with:
-          name: dist ${{ github.run_number }}
+          name: jupyterlite-xeus-python-dist-${{ github.run_number }}
           path: ./dist
 
   test_isolated:
@@ -66,7 +66,7 @@ jobs:
           architecture: 'x64'
       - uses: actions/download-artifact@v2
         with:
-          name: dist ${{ github.run_number }}
+          name: jupyterlite-xeus-python-dist-${{ github.run_number }}
           path: ./dist
       - name: Install and Test
         run: |
@@ -92,7 +92,7 @@ jobs:
 
       - uses: actions/download-artifact@v2
         with:
-          name: dist ${{ github.run_number }}
+          name: jupyterlite-xeus-python-dist-${{ github.run_number }}
           path: ./dist
 
       - name: Install Conda environment with Micromamba
@@ -125,7 +125,7 @@ jobs:
 
       - uses: actions/download-artifact@v2
         with:
-          name: dist ${{ github.run_number }}
+          name: jupyterlite-xeus-python-dist-${{ github.run_number }}
           path: ./dist
 
       - name: Install Conda environment with Micromamba
@@ -153,7 +153,7 @@ jobs:
 
       - uses: actions/download-artifact@v2
         with:
-          name: dist ${{ github.run_number }}
+          name: jupyterlite-xeus-python-dist-${{ github.run_number }}
           path: ./dist
 
       - name: Install Conda environment with Micromamba
@@ -181,7 +181,7 @@ jobs:
 
       - uses: actions/download-artifact@v2
         with:
-          name: dist ${{ github.run_number }}
+          name: jupyterlite-xeus-python-dist-${{ github.run_number }}
           path: ./dist
 
       - name: Install Conda environment with Micromamba

.github/workflows/playwright-update.yml

@@ -0,0 +1,67 @@
+name: Update Playwright Snapshots
+
+on:
+  issue_comment:
+    types: [created, edited]
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  update-snapshots:
+    if:
+      ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'update
+      playwright snapshots') }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        browser: [firefox, chromium]
+
+    steps:
+      - name: React to the triggering comment
+        run: |
+          gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions --raw-field 'content=+1'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Checkout the branch from the PR that triggered the job
+        run: |
+          # PR branch remote must be checked out using https URL
+          git config --global hub.protocol https
+
+          gh pr checkout ${{ github.event.issue.number }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Base Setup
+        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+
+      - name: Build
+        run: |
+          # TODO: install addon
+
+          # disable git hooks
+          git config core.hooksPath no-hooks
+
+      - name: Install the test dependencies
+        run: |
+          cd ui-tests
+          jlpm
+          jlpm build
+          jlpm playwright install
+
+      - name: Update snapshots
+        uses: jupyterlab/maintainer-tools/.github/actions/update-snapshots@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          npm_client: jlpm
+          test_folder: ui-tests
+          start_server_script: 'null'
+          update_script: test:update --browser ${{ matrix.browser }}
+        env:
+          DEBUG: pw:webserver

.github/workflows/ui-tests.yml

@@ -0,0 +1,107 @@
+name: UI Tests
+
+on: [push, pull_request]
+
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Conda environment with Micromamba
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          micromamba-version: '1.5.1-0'
+          environment-file: environment.yml
+          cache-environment: true
+
+      - name: Install jupyterlite-xeus-python
+        run: |
+          jlpm
+          jlpm run build
+          python -m pip install -v .
+
+      - name: Build UI tests
+        run: |
+          cd ui-tests
+          jlpm
+          # Build the JupyterLite website
+          jlpm build
+
+      - name: Upload the JupyterLite website
+        uses: actions/upload-artifact@v3
+        with:
+          name: jupyterlite-xeus-python-ui-tests-app-${{ github.run_number }}
+          path: ./ui-tests/ui-tests-app
+
+  ui-tests:
+    needs: [build]
+    name: Visual Regression
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        browser: [firefox, chromium]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: jupyterlite-xeus-python-ui-tests-app-${{ github.run_number }}
+          path: ./ui-tests/ui-tests-app
+
+      - name: Base Setup
+        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+
+      - name: Install dependencies and browser
+        run: |
+          # Install JupyterLab to get jlpm
+          python -m pip install jupyterlab~=4.0
+          cd ui-tests
+          jlpm
+          jlpm playwright install ${{ matrix.browser }} --with-deps
+
+      - name: Test
+        run: |
+          cd ui-tests
+          jlpm run test --browser ${{ matrix.browser }}
+
+      - name: Upload Playwright Test assets
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: jupyterlite-xeux-python-${{ matrix.browser }}-test-assets
+          path: |
+            ui-tests/test-results
+
+      - name: Upload Playwright Test report
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: jupyterlite-xeus-python-${{ matrix.browser }}-test-report
+          path: |
+            ui-tests/playwright-report
+
+      - name: Update snapshots
+        if: failure()
+        run: |
+          cd ui-tests
+          # remove previous snapshots from other browser
+          jlpm run clean:snapshots
+          # generate new snapshots
+          jlpm run test:update --browser ${{ matrix.browser }}
+
+      - name: Upload updated snapshots
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: jupyterlite-xeus-python-${{ matrix.browser }}-updated-snapshots
+          path: ui-tests/test

.gitignore

@@ -135,4 +135,13 @@ jupyterlite_xeus_python/_version.py
 
 # Yarn 3
 .pnp*
-.yarn
+.yarn
+
+# UI tests
+ui-tests/playwright-report
+ui-tests/test-results
+ui-tests/ui-tests-app
+
+# Lint
+.eslintcache
+.stylelintcache

README.md

@@ -76,7 +76,7 @@ The `jlpm` command is JupyterLab's pinned version of
 # Clone the repo to your local environment
 # Change directory to the jupyterlite-xeus-python directory
 # Install package in development mode
-python -m pip install -e .
+python -m pip install -e ".[dev]"
 
 # Link your development version of the extension with JupyterLab
 jupyter labextension develop . --overwrite

pyproject.toml

@@ -136,6 +136,10 @@ ignore = [
 "tests/*" = ["S101", "F841", "PLR2004"]
 
 # B008 Do not perform function call `typer.Option` in argument defaults
-# E501 `subprocess` call: check for execution of untrusted input
+# E501 Line too long
 # S603 `subprocess` call: check for execution of untrusted input
 "jupyterlite_xeus_python/build.py" = ["B008", "E501", "S603"]
+
+# E501 Line too long
+# S602 `subprocess` call with `shell=True` identified, security issue
+"ui-tests/build.py" = ["E501", "S602"]

ui-tests/.yarnrc.yml

@@ -0,0 +1,6 @@
+enableImmutableInstalls: false
+enableInlineBuilds: false
+enableTelemetry: false
+httpTimeout: 60000
+nodeLinker: node-modules
+npmRegistryServer: 'https://registry.yarnpkg.com'

ui-tests/build.py

@@ -0,0 +1,17 @@
+"""
+Custom script to build the UI tests app and load the Galata extension
+"""
+
+from pathlib import Path
+from subprocess import run
+
+import jupyterlab
+
+extra_labextensions_path = str(Path(jupyterlab.__file__).parent / "galata")
+cmd = f"jupyter lite build --FederatedExtensionAddon.extra_labextensions_path={extra_labextensions_path}"
+
+run(
+    cmd,
+    check=True,
+    shell=True,
+)

ui-tests/jupyter-lite.json

@@ -0,0 +1,7 @@
+{
+  "jupyter-lite-schema-version": 0,
+  "jupyter-config-data": {
+    "appName": "JupyterLite UI Tests",
+    "exposeAppInBrowser": true
+  }
+}

ui-tests/jupyter_lite_config.json

@@ -0,0 +1,5 @@
+{
+  "LiteBuildConfig": {
+    "output_dir": "ui-tests-app"
+  }
+}

ui-tests/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@jupyterlite/jupyterlite-xeus-python-ui-tests",
+  "private": true,
+  "version": "0.1.0",
+  "author": "JupyterLite Contributors",
+  "license": "BSD-3-Clause",
+  "description": "JupyterLite Xeus Python UI Tests",
+  "scripts": {
+    "build": "python build.py",
+    "clean": "rimraf .jupyterlite.doit.db ui-tests-app",
+    "clean:snapshots": "rimraf -g \"test/**/*-snapshots/*.png\"",
+    "start": "cd ui-tests-app && python -m http.server -b 127.0.0.1 8000",
+    "test": "playwright test",
+    "test:report": "http-server ./playwright-report -a localhost -o",
+    "test:update": "playwright test --update-snapshots"
+  },
+  "dependencies": {
+    "@jupyterlab/galata": "~5.0.5",
+    "@playwright/test": "^1.36.2"
+  },
+  "devDependencies": {
+    "rimraf": "^5"
+  }
+}

ui-tests/playwright.config.js

@@ -0,0 +1,29 @@
+const baseConfig = require('@jupyterlab/galata/lib/playwright-config');
+
+module.exports = {
+  ...baseConfig,
+  retries: 1,
+  use: {
+    acceptDownloads: true,
+    appPath: '',
+    autoGoto: false,
+    baseURL: 'http://localhost:8000',
+    trace: 'retain-on-failure',
+    video: 'retain-on-failure',
+
+    waitForApplication: async ({ baseURL }, use, testInfo) => {
+      const waitIsReady = async page => {
+        await page.waitForSelector('.jp-LauncherCard');
+      };
+      await use(waitIsReady);
+    }
+  },
+  webServer: [
+    {
+      command: 'jlpm run start',
+      port: 8000,
+      timeout: 120 * 1000,
+      reuseExistingServer: true
+    }
+  ]
+};

ui-tests/test/execute.test.ts

@@ -0,0 +1,26 @@
+// Copyright (c) JupyterLite Contributors
+// Distributed under the terms of the Modified BSD License.
+
+import { test } from '@jupyterlab/galata';
+
+import { expect } from '@playwright/test';
+
+import { firefoxWaitForApplication } from './utils';
+
+test.use({ waitForApplication: firefoxWaitForApplication });
+
+test.describe('Code execution', () => {
+  test.beforeEach(async ({ page }) => {
+    await page.goto('lab/index.html');
+  });
+
+  test('Basic code execution', async ({ page }) => {
+    await page.notebook.createNew();
+    await page.notebook.setCell(0, 'code', '2 + 2');
+    await page.notebook.run();
+    const output = await page.notebook.getCellTextOutput(0);
+
+    expect(output).toBeTruthy();
+    expect(output![0]).toBe('4');
+  });
+});