Run all smoke tests on merge to main (#4141)

This pull request changes the testing flow of smoke tests run with PRs
and merges to main. Previously, the same github action was run on pull
request and on merge to main. That action would run some unit, some
integration, and some smoke tests (tagged with #pr). Now, instead of
running the same action on PR and on merge, the on-merge action has been
updated to run all smoke tests on merge to main (not just the ones
tagged with #pr).

The smoke tests are broken up in this fashion because they are time
consuming. The idea is to keep the PR set of tests under 7 minutes. If
the merge-to-main tests fail, the PR author will be notified via email
by github as long as they have their notification settings properly
configured (https://github.com/settings/notifications):

<img width="1089" alt="image"
src="https://github.com/user-attachments/assets/7e889ce4-6813-4499-a946-40dc3cfac34d">

### QA Notes

All smoke tests should pass.

.github/workflows/positron-full-test.yml

@@ -119,8 +119,8 @@ jobs:
 
       - name: Run Smoke Tests (Electron)
         env:
-          POSITRON_PY_VER_SEL: Python 3.10.12
-          POSITRON_R_VER_SEL: R 4.4.0
+          POSITRON_PY_VER_SEL: 3.10.12
+          POSITRON_R_VER_SEL: 4.4.0
         id: electron-smoke-tests
         run: DISPLAY=:10 yarn smoketest-no-compile --tracing
 

.github/workflows/positron-merge-to-main.yml

@@ -0,0 +1,88 @@
+name: "Positron: CI - merge to main"
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+
+  linux:
+    name: Tests on Linux
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      POSITRON_BUILD_NUMBER: 0 # CI skips building releases
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Build Environment
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y vim curl build-essential clang make cmake git python3-pip python-is-python3 libsodium-dev libxkbfile-dev pkg-config libsecret-1-dev libxss1 dbus xvfb libgtk-3-0 libgbm1 libnss3 libnspr4 libasound2 libkrb5-dev libcairo-dev libsdl-pango-dev libjpeg-dev libgif-dev
+          sudo cp build/azure-pipelines/linux/xvfb.init /etc/init.d/xvfb
+          sudo chmod +x /etc/init.d/xvfb
+          sudo update-rc.d xvfb defaults
+          sudo service xvfb start
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+
+      - name: Execute yarn
+        env:
+          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+          ELECTRON_SKIP_BINARY_DOWNLOAD: 1
+          POSITRON_GITHUB_PAT: ${{ github.token }}
+        run: |
+          # Install Yarn
+          npm install -g yarn
+
+          # Install node-gyp; this is required by some packages, and yarn
+          # sometimes fails to automatically install it.
+          yarn global add node-gyp
+
+          # Perform the main yarn command; this installs all Node packages and
+          # dependencies
+          yarn --immutable --network-timeout 120000
+          yarn --cwd test/automation install
+          yarn --cwd test/smoke install
+
+      - name: Compile and Download
+        run: yarn npm-run-all --max_old_space_size=4095 -lp compile "electron x64" playwright-install download-builtin-extensions
+
+      - name: Compile Smoke Tests
+        run: yarn --cwd test/smoke compile
+
+      - name: Install rig, R, and R packages
+        run: |
+          curl -Ls https://github.com/r-lib/rig/releases/download/latest/rig-linux-"$(arch)"-latest.tar.gz | $(which sudo) tar xz -C /usr/local
+          rig add 4.4.0
+          curl https://raw.githubusercontent.com/posit-dev/qa-example-content/main/DESCRIPTION --output DESCRIPTION
+          Rscript -e "pak::local_install_dev_deps(ask = FALSE)"
+
+      - name: Setup Graphviz
+        uses: ts-graphviz/[email protected]
+
+      - name: Install python dependencies
+        run: |
+          curl https://raw.githubusercontent.com/posit-dev/qa-example-content/main/requirements.txt --output requirements.txt
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements.txt
+          python -m pip install matplotlib ipykernel graphviz
+
+      - name: Run Smoke Tests (Electron)
+        env:
+          POSITRON_PY_VER_SEL: 3.10.12
+          POSITRON_R_VER_SEL: 4.4.0
+        id: electron-smoke-tests
+        run: DISPLAY=:10 yarn smoketest-merge-to-main --tracing
+
+      - name: Upload run artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: run-artifacts
+          path: .build/logs/smoke-tests-electron/

.github/workflows/positron-ci.yml → .github/workflows/positron-pull-requests.yml

@@ -1,9 +1,6 @@
-name: "Positron: CI"
+name: "Positron: CI - pull request"
 
 on:
-  push:
-    branches:
-      - main
   pull_request:
     branches:
       - main
@@ -88,8 +85,8 @@ jobs:
 
       - name: Run Smoke Tests (Electron)
         env:
-          POSITRON_PY_VER_SEL: Python 3.10.12
-          POSITRON_R_VER_SEL: R 4.4.0
+          POSITRON_PY_VER_SEL: 3.10.12
+          POSITRON_R_VER_SEL: 4.4.0
         id: electron-smoke-tests
         run: DISPLAY=:10 yarn smoketest-pr --tracing
 

package.json

@@ -42,6 +42,7 @@
     "smoketest": "node build/lib/preLaunch.js && cd test/smoke && BUILD_ARTIFACTSTAGINGDIRECTORY=../../.build/logs/smoke-tests-electron yarn compile && node test/index.js",
     "smoketest-no-compile": "cd test/smoke && BUILD_ARTIFACTSTAGINGDIRECTORY=../../.build/logs/smoke-tests-electron node test/index.js",
     "smoketest-pr": "cd test/smoke && TEST_FILTER=/#pr/ node test/index.js",
+    "smoketest-merge-to-main": "cd test/smoke && node test/index.js",
     "download-builtin-extensions": "node build/lib/builtInExtensions.js",
     "download-builtin-extensions-cg": "node build/lib/builtInExtensionsCG.js",
     "download-pandoc": "node build/lib/pandoc.js",

test/smoke/test/index.js

@@ -39,8 +39,13 @@ if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 const mocha = new Mocha(options);
 mocha.addFile('out/main.js');
 // --- Start Positron ---
-mocha.grep(process.env.TEST_FILTER || '');
-// mocha.dryRun(); // debugging
+if (process.env.TEST_FILTER) {
+	mocha.grep(process.env.TEST_FILTER);
+} else if (process.env.INVERSE_FILTER) {
+	mocha.grep(process.env.INVERSE_FILTER);
+	mocha.invert();
+}
+// mocha.dryRun(); // debugging filters
 // --- End Positron ---
 mocha.run(failures => {