fix mpi ci (#174)

* remove rust cache * try... * clean up * profiling * the old fashion * Update install.py * remove apt update * Update benchmarks.yml * enable sudo for CI
PolyhedraZK · Jan 14, 2025 · 5b5f8df · 5b5f8df
1 parent 8abc8e2
commit 5b5f8df
Show file tree

Hide file tree

Showing 5 changed files with 69 additions and 20 deletions.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -1,5 +1,10 @@
 name: Run Benchmarks
 
+env:
+  RUSTFLAGS: "-Dwarnings -C target-cpu=native"
+  RUST_BACKTRACE: 1
+  ACTIONS_RUNNER_DEBUG: true
+
 on:
   push:
   pull_request:
@@ -29,6 +34,8 @@ jobs:
       run: cargo install cargo-criterion
     - name: Install mpi
       run: python3 ./scripts/install.py
+    - name: Install build dependencies
+      run: sudo apt-get install -y build-essential  
     - name: Setup
       run: cargo run --bin=dev-setup --release
     - name: Run benchmark

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -6,6 +6,7 @@ on:
 env:
   RUSTFLAGS: "-Dwarnings -C target-cpu=native"
   RUST_BACKTRACE: 1
+  ACTIONS_RUNNER_DEBUG: true
 
 jobs:
   build:
@@ -23,18 +24,32 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
+
+      - name: Setup Dependencies
+        run: |
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            brew install gcc make
+          else
+            sudo apt-get install -y build-essential
+          fi
+        shell: bash
+
       - uses: dtolnay/rust-toolchain@stable
+
       - uses: Swatinem/rust-cache@v2
         with:
-          # The prefix cache key, this can be changed to start a new cache manually.
-          prefix-key: "mpi-v5.0.6" # update me if brew formula changes to a new version
+          prefix-key: "mpi-v5.0.6"
+
       - name: Install MPI
         run: python3 ./scripts/install.py
+
       - name: Set RUSTFLAGS for AVX
         if: matrix.feature != ''
         run: echo "RUSTFLAGS=$RUSTFLAGS -C target-feature=+${{ matrix.feature }}" >> $GITHUB_ENV
+
       - name: Prepare binary
         run: cargo build --release --bin expander-exec
+
       - name: Upload release asset
         uses: actions/github-script@v6
         with:
@@ -46,4 +61,4 @@ jobs:
               release_id: ${{ github.event.release.id }},
               name: '${{ matrix.binary_name }}',
               data: await fs.readFile('target/release/expander-exec')
-            });
+            });
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -5,6 +5,7 @@ on: [pull_request, push]
 env:
   RUSTFLAGS: "-Dwarnings -C target-cpu=native"
   RUST_BACKTRACE: 1
+  ACTIONS_RUNNER_DEBUG: true
 
 jobs:
   lint:
@@ -15,6 +16,9 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: rustfmt, clippy
+      - name: Setup Dependencies
+        run: |
+          sudo apt-get install -y build-essential
       - run: python3 ./scripts/install.py
       - run: cargo fmt --all -- --check
       - run: cargo clippy --all
@@ -35,18 +39,30 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
         with:
-          # The prefix cache key, this can be changed to start a new cache manually.
-          prefix-key: "mpi-v5.0.6" # update me if brew formula changes to a new version
+          prefix-key: "mpi-v5.0.6"
+      - name: Setup Dependencies
+        run: |
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            brew install gcc make
+          else
+            sudo apt-get install -y build-essential
+          fi
       - name: Set RUSTFLAGS for AVX
         if: matrix.feature != ''
         run: echo "RUSTFLAGS=$RUSTFLAGS -C target-feature=+${{ matrix.feature }}" >> $GITHUB_ENV  
-      - name: Build and Test
+      - name: Setup
         run: |
           python3 ./scripts/install.py
           mpiexec --version
+      - name: download data
+        run: |
           cargo run --bin=dev-setup --release
+      - name: Unit Tests
+        run: |
           cargo build --all-features --release
           cargo test --all-features --release --workspace
+      - name: E2E Test
+        run: |   
           ./scripts/test_recursion.py
 
   gkr-e2e:
@@ -84,13 +100,19 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
         with:
-          # The prefix cache key, this can be changed to start a new cache manually.
-          prefix-key: "mpi-v5.0.5" # update me if brew formula changes to a new version
+          prefix-key: "mpi-v5.0.6"
+      - name: Setup Dependencies
+        run: |
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            brew install gcc make
+          else
+            sudo apt-get install -y build-essential
+          fi
       - name: Set RUSTFLAGS for AVX
         if: matrix.feature != ''
         run: echo "RUSTFLAGS=$RUSTFLAGS -C target-feature=+${{ matrix.feature }}" >> $GITHUB_ENV
       - name: Run benchmark
         run: |
           python3 ./scripts/install.py
           cargo run --bin=dev-setup --release
-          cargo run --bin=gkr --release -- -t ${{ matrix.os == 'macos-latest' && 2 || 16 }} -f ${{ matrix.field }}
+          cargo run --bin=gkr --release -- -t ${{ matrix.os == 'macos-latest' && 2 || 16 }} -f ${{ matrix.field }}
diff --git a/.github/workflows/nightly_e2e.yml b/.github/workflows/nightly_e2e.yml
@@ -1,5 +1,10 @@
 name: E2E Tests
 
+env:
+  RUSTFLAGS: "-Dwarnings -C target-cpu=native"
+  RUST_BACKTRACE: 1
+  ACTIONS_RUNNER_DEBUG: true
+
 on:
   schedule:
     # This will run the workflow every day at 2:00 AM UTC
@@ -18,20 +23,26 @@ jobs:
     runs-on: 7950x3d
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4  # Updated to v4
+
+    - name: Setup Dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y build-essential
 
     - name: Set up Rust
       uses: dtolnay/rust-toolchain@master
       with:
         toolchain: nightly-2024-09-01
         components: rustfmt, clippy
+
     - uses: Swatinem/rust-cache@v2
       with:
         # The prefix cache key, this can be changed to start a new cache manually.
         prefix-key: "mpi-v5.0.6" # update me if brew formula changes to a new version
 
     - name: Set up Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5  # Updated to v5
       with:
         go-version: '^1.16'
 
@@ -65,10 +76,4 @@ jobs:
         RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo +nightly run --release --bin=gkr -- -s keccak -f gf2ext128 -t 16
         RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo +nightly run --release --bin=gkr -- -s keccak -f m31ext3 -t 16
         RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo +nightly run --release --bin=gkr -- -s keccak -f fr -t 16
-        RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo +nightly run --release --bin=gkr -- -s poseidon -f m31ext3 -t 16
-
-        # # Run MPI tests
-        # RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" mpiexec -n 2 cargo +nightly run --release --bin=gkr-mpi -- -s keccak -f gf2ext128
-        # RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" mpiexec -n 2 cargo +nightly run --release --bin=gkr-mpi -- -s keccak -f m31ext3
-        # RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" mpiexec -n 2 cargo +nightly run --release --bin=gkr-mpi -- -s keccak -f fr
-        # RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" mpiexec -n 2 cargo +nightly run --release --bin=gkr-mpi -- -s poseidon -f m31ext3
+        RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo +nightly run --release --bin=gkr -- -s poseidon -f m31ext3 -t 16
diff --git a/scripts/install.py b/scripts/install.py
@@ -3,6 +3,6 @@
 
 if __name__ == "__main__":
     if platform == "darwin": # mac os
-        subprocess.run(["brew", "install", "openmpi"])
+        subprocess.run(["brew", "install", "gcc", "make", "openmpi"])
     else:
-        pass # Do nothing, assuming mpi has already been installed
+        pass