Reduce running time for CI tests (#956)

* Split unit test and benchark Signed-off-by: Georgi Zlatarev <[email protected]> * Use all threads for unit tests Signed-off-by: Georgi Zlatarev <[email protected]> * Use dev branch for CI test Signed-off-by: Georgi Zlatarev <[email protected]> * Fix yml Signed-off-by: Georgi Zlatarev <[email protected]> * Get rid of -dev tests Signed-off-by: Georgi Zlatarev <[email protected]> * Fix dependencies Signed-off-by: Georgi Zlatarev <[email protected]> * Use dev branch for test Signed-off-by: Georgi Zlatarev <[email protected]> * Use dev branch for test Signed-off-by: Georgi Zlatarev <[email protected]> * Check Signed-off-by: Georgi Zlatarev <[email protected]> * Check Signed-off-by: Georgi Zlatarev <[email protected]> * Revert for pr Signed-off-by: Georgi Zlatarev <[email protected]> Signed-off-by: Georgi Zlatarev <[email protected]>
Manta-Network · Jan 3, 2023 · aa34668 · aa34668
1 parent 7ec09a2
commit aa34668
Show file tree

Hide file tree

Showing 3 changed files with 90 additions and 23 deletions.
diff --git a/.github/workflows/check_tests.yml b/.github/workflows/check_tests.yml
@@ -81,8 +81,91 @@ jobs:
           SCCACHE_DIR: /home/runner/.cache/sccache
         run: |
           source ${HOME}/.cargo/env
-          RUSTC_BOOTSTRAP=1 cargo test --release --features=runtime-benchmarks,try-runtime -- --test-threads=2
-      - name: run live benchmarks test
+          RUSTC_BOOTSTRAP=1 cargo test --release --features=runtime-benchmarks,try-runtime
+      - name: stop sccache server
+        run: sccache --stop-server || true
+  stop-unit-test-checks:
+    needs: start-unit-test-checks
+    runs-on: ubuntu-20.04
+    if: ${{ always() }}
+    steps:
+      - continue-on-error: true
+        uses: audacious-network/[email protected]
+        with:
+          mode: stop
+          github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN  }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ needs.start-unit-test-checks.outputs.aws-region }}
+          runner-label: ${{ needs.start-unit-test-checks.outputs.runner-label }}
+          aws-instance-id: ${{ needs.start-unit-test-checks.outputs.aws-instance-id }}
+      - name: discard stopper success/failure
+        run: true
+  start-benchmark-checks:
+    runs-on: ubuntu-20.04
+    outputs:
+      runner-label: ${{ steps.start-self-hosted-runner.outputs.runner-label }}
+      aws-region: ${{ steps.start-self-hosted-runner.outputs.aws-region }}
+      aws-instance-id: ${{ steps.start-self-hosted-runner.outputs.aws-instance-id }}
+    steps:
+      - id: start-self-hosted-runner
+        uses: audacious-network/[email protected]
+        with:
+          mode: start
+          github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN  }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+          aws-instance-type: ${{ env.AWS_INSTANCE_TYPE }}
+          aws-instance-root-volume-size: ${{ env.AWS_INSTANCE_ROOT_VOLUME_SIZE }}
+          aws-image-search-pattern: ${{ env.AWS_IMAGE_SEARCH_PATTERN }}
+          aws-image-search-owners: ${{ env.AWS_IMAGE_SEARCH_OWNERS }}
+      - uses: actions/checkout@v2
+      - name: install sccache
+        env:
+          SCCACHE_RELEASE_URL: https://github.com/mozilla/sccache/releases/download
+          SCCACHE_VERSION: v0.2.15
+        run: |
+          SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl
+          mkdir -p $HOME/.local/bin
+          curl -L "$SCCACHE_RELEASE_URL/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz
+          mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache
+          chmod +x $HOME/.local/bin/sccache
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+      - name: cache cargo registry
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+          key: cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            cargo-
+      - name: cache sccache
+        uses: actions/cache@v2
+        continue-on-error: false
+        with:
+          path: /home/runner/.cache/sccache
+          key: sccache-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            sccache-
+      - name: start sccache server
+        run: sccache --start-server
+      - name: init
+        run: |
+          curl -s https://sh.rustup.rs -sSf | sh -s -- -y
+          source ${HOME}/.cargo/env
+          rustup toolchain install stable
+          rustup default stable
+          rustup update
+          rustup target add wasm32-unknown-unknown
+      - name: Run live benchmarks test
+        env:
+          RUST_BACKTRACE: full
+          RUSTC_WRAPPER: sccache
+          SCCACHE_CACHE_SIZE: 2G
+          SCCACHE_DIR: /home/runner/.cache/sccache
         run: |
           RUSTC_BOOTSTRAP=1 cargo run --release --features runtime-benchmarks,try-runtime \
           benchmark \
@@ -94,8 +177,8 @@ jobs:
             --steps=1
       - name: stop sccache server
         run: sccache --stop-server || true
-  stop-unit-test-checks:
-    needs: start-unit-test-checks
+  stop-benchmark-checks:
+    needs: start-benchmark-checks
     runs-on: ubuntu-20.04
     if: ${{ always() }}
     steps:
@@ -106,8 +189,8 @@ jobs:
           github-token: ${{ secrets.SELF_HOSTED_RUNNER_TOKEN  }}
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ needs.start-unit-test-checks.outputs.aws-region }}
-          runner-label: ${{ needs.start-unit-test-checks.outputs.runner-label }}
-          aws-instance-id: ${{ needs.start-unit-test-checks.outputs.aws-instance-id }}
+          aws-region: ${{ needs.start-benchmark-checks.outputs.aws-region }}
+          runner-label: ${{ needs.start-benchmark-checks.outputs.runner-label }}
+          aws-instance-id: ${{ needs.start-benchmark-checks.outputs.aws-instance-id }}
       - name: discard stopper success/failure
         run: true
diff --git a/.github/workflows/integration_test_calamari.yml b/.github/workflows/integration_test_calamari.yml
@@ -115,14 +115,6 @@ jobs:
       fail-fast: false
       matrix:
         chain-spec:
-          - id: calamari-dev
-            expected:
-              block-count:
-                relay: 25
-                para: 6
-              peer-count:
-                relay: 7
-                para: 4
           - id: calamari-local
             expected:
               block-count:

diff --git a/.github/workflows/integration_test_dolphin.yml b/.github/workflows/integration_test_dolphin.yml
@@ -115,14 +115,6 @@ jobs:
       fail-fast: false
       matrix:
         chain-spec:
-          - id: dolphin-dev
-            expected:
-              block-count:
-                relay: 25
-                para: 6
-              peer-count:
-                relay: 7
-                para: 4
           - id: dolphin-local
             expected:
               block-count: