From 8c256f4ab39521c09b80c394b198ece59eded4f2 Mon Sep 17 00:00:00 2001 From: Christopher Mead Date: Wed, 24 Jul 2024 14:52:23 -0600 Subject: [PATCH] Run all smoke tests on merge to main (#4141) This pull request changes the testing flow of smoke tests run with PRs and merges to main. Previously, the same github action was run on pull request and on merge to main. That action would run some unit, some integration, and some smoke tests (tagged with #pr). Now, instead of running the same action on PR and on merge, the on-merge action has been updated to run all smoke tests on merge to main (not just the ones tagged with #pr). The smoke tests are broken up in this fashion because they are time consuming. The idea is to keep the PR set of tests under 7 minutes. If the merge-to-main tests fail, the PR author will be notified via email by github as long as they have their notification settings properly configured (https://github.com/settings/notifications): image ### QA Notes All smoke tests should pass. --- .github/workflows/positron-full-test.yml | 4 +- .github/workflows/positron-merge-to-main.yml | 88 +++++++++++++++++++ ...tron-ci.yml => positron-pull-requests.yml} | 9 +- package.json | 1 + test/smoke/test/index.js | 9 +- 5 files changed, 101 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/positron-merge-to-main.yml rename .github/workflows/{positron-ci.yml => positron-pull-requests.yml} (95%) diff --git a/.github/workflows/positron-full-test.yml b/.github/workflows/positron-full-test.yml index 350240054bc..1837f392e13 100644 --- a/.github/workflows/positron-full-test.yml +++ b/.github/workflows/positron-full-test.yml @@ -119,8 +119,8 @@ jobs: - name: Run Smoke Tests (Electron) env: - POSITRON_PY_VER_SEL: Python 3.10.12 - POSITRON_R_VER_SEL: R 4.4.0 + POSITRON_PY_VER_SEL: 3.10.12 + POSITRON_R_VER_SEL: 4.4.0 id: electron-smoke-tests run: DISPLAY=:10 yarn smoketest-no-compile --tracing diff --git a/.github/workflows/positron-merge-to-main.yml b/.github/workflows/positron-merge-to-main.yml new file mode 100644 index 00000000000..31438c0658b --- /dev/null +++ b/.github/workflows/positron-merge-to-main.yml @@ -0,0 +1,88 @@ +name: "Positron: CI - merge to main" + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + + linux: + name: Tests on Linux + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + POSITRON_BUILD_NUMBER: 0 # CI skips building releases + steps: + - uses: actions/checkout@v4 + + - name: Setup Build Environment + run: | + sudo apt-get update + sudo apt-get install -y vim curl build-essential clang make cmake git python3-pip python-is-python3 libsodium-dev libxkbfile-dev pkg-config libsecret-1-dev libxss1 dbus xvfb libgtk-3-0 libgbm1 libnss3 libnspr4 libasound2 libkrb5-dev libcairo-dev libsdl-pango-dev libjpeg-dev libgif-dev + sudo cp build/azure-pipelines/linux/xvfb.init /etc/init.d/xvfb + sudo chmod +x /etc/init.d/xvfb + sudo update-rc.d xvfb defaults + sudo service xvfb start + + - uses: actions/setup-node@v3 + with: + node-version: 20 + + - name: Execute yarn + env: + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 + ELECTRON_SKIP_BINARY_DOWNLOAD: 1 + POSITRON_GITHUB_PAT: ${{ github.token }} + run: | + # Install Yarn + npm install -g yarn + + # Install node-gyp; this is required by some packages, and yarn + # sometimes fails to automatically install it. + yarn global add node-gyp + + # Perform the main yarn command; this installs all Node packages and + # dependencies + yarn --immutable --network-timeout 120000 + yarn --cwd test/automation install + yarn --cwd test/smoke install + + - name: Compile and Download + run: yarn npm-run-all --max_old_space_size=4095 -lp compile "electron x64" playwright-install download-builtin-extensions + + - name: Compile Smoke Tests + run: yarn --cwd test/smoke compile + + - name: Install rig, R, and R packages + run: | + curl -Ls https://github.com/r-lib/rig/releases/download/latest/rig-linux-"$(arch)"-latest.tar.gz | $(which sudo) tar xz -C /usr/local + rig add 4.4.0 + curl https://raw.githubusercontent.com/posit-dev/qa-example-content/main/DESCRIPTION --output DESCRIPTION + Rscript -e "pak::local_install_dev_deps(ask = FALSE)" + + - name: Setup Graphviz + uses: ts-graphviz/setup-graphviz@v2.0.2 + + - name: Install python dependencies + run: | + curl https://raw.githubusercontent.com/posit-dev/qa-example-content/main/requirements.txt --output requirements.txt + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + python -m pip install matplotlib ipykernel graphviz + + - name: Run Smoke Tests (Electron) + env: + POSITRON_PY_VER_SEL: 3.10.12 + POSITRON_R_VER_SEL: 4.4.0 + id: electron-smoke-tests + run: DISPLAY=:10 yarn smoketest-merge-to-main --tracing + + - name: Upload run artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: run-artifacts + path: .build/logs/smoke-tests-electron/ diff --git a/.github/workflows/positron-ci.yml b/.github/workflows/positron-pull-requests.yml similarity index 95% rename from .github/workflows/positron-ci.yml rename to .github/workflows/positron-pull-requests.yml index 85f6a15b0dd..8191cfdd400 100644 --- a/.github/workflows/positron-ci.yml +++ b/.github/workflows/positron-pull-requests.yml @@ -1,9 +1,6 @@ -name: "Positron: CI" +name: "Positron: CI - pull request" on: - push: - branches: - - main pull_request: branches: - main @@ -88,8 +85,8 @@ jobs: - name: Run Smoke Tests (Electron) env: - POSITRON_PY_VER_SEL: Python 3.10.12 - POSITRON_R_VER_SEL: R 4.4.0 + POSITRON_PY_VER_SEL: 3.10.12 + POSITRON_R_VER_SEL: 4.4.0 id: electron-smoke-tests run: DISPLAY=:10 yarn smoketest-pr --tracing diff --git a/package.json b/package.json index 034df471685..703bce745f0 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "smoketest": "node build/lib/preLaunch.js && cd test/smoke && BUILD_ARTIFACTSTAGINGDIRECTORY=../../.build/logs/smoke-tests-electron yarn compile && node test/index.js", "smoketest-no-compile": "cd test/smoke && BUILD_ARTIFACTSTAGINGDIRECTORY=../../.build/logs/smoke-tests-electron node test/index.js", "smoketest-pr": "cd test/smoke && TEST_FILTER=/#pr/ node test/index.js", + "smoketest-merge-to-main": "cd test/smoke && node test/index.js", "download-builtin-extensions": "node build/lib/builtInExtensions.js", "download-builtin-extensions-cg": "node build/lib/builtInExtensionsCG.js", "download-pandoc": "node build/lib/pandoc.js", diff --git a/test/smoke/test/index.js b/test/smoke/test/index.js index 02c75702a9f..0795116f37a 100644 --- a/test/smoke/test/index.js +++ b/test/smoke/test/index.js @@ -39,8 +39,13 @@ if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { const mocha = new Mocha(options); mocha.addFile('out/main.js'); // --- Start Positron --- -mocha.grep(process.env.TEST_FILTER || ''); -// mocha.dryRun(); // debugging +if (process.env.TEST_FILTER) { + mocha.grep(process.env.TEST_FILTER); +} else if (process.env.INVERSE_FILTER) { + mocha.grep(process.env.INVERSE_FILTER); + mocha.invert(); +} +// mocha.dryRun(); // debugging filters // --- End Positron --- mocha.run(failures => {